From 23317c766c4a058b2fb6b88e7ef40234ddd3f0c5 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sat, 19 Sep 2026 02:10:34 +0400 Subject: [PATCH 1/6] feat: add versioning support for directory objects in posix Directory objects (keys ending in `/`) are now versioned in versioning-enabled posix buckets like regular objects. The directory is kept in place across versions and carries the `version-id` and `delete-marker` attributes, while older versions are stored as empty files in the versioning directory. Only the attributes listed in `dirObjectAttrs` are copied into versions, replaced on a new `PutObject`, and restored when the latest version is deleted, so attributes the directory carries as a parent of other objects are left untouched. `PutObject` on a directory object now snapshots the current version, starts the new version with a clean attribute set and returns a `VersionId`. `DeleteObject` creates delete markers for directory objects and, when deleting the latest version, restores the previous one onto the directory, or removes the directory (or keeps it as a plain parent of its children) when no versions are left. `ListObjectVersions` lists directory object versions and delete markers and uses the key with the trailing slash for `NextKeyMarker` and marker comparisons, `ListObjects` hides directory delete markers and `GetObject` returns the directory object version id. `PutObject` on directory objects now applies the object lock legal hold and retention headers, and `CopyObject` and `UploadPartCopy` accept a directory object as a 0-byte copy source. The versioning integration tests are extended to run their scenarios for directory objects as well, with new tests for directory-object-specific behavior. --- backend/posix/posix.go | 551 ++- backend/walk.go | 31 +- tests/integration/ListObjectVersions.go | 594 ++- tests/integration/group-tests.go | 14 + tests/integration/utils.go | 51 +- tests/integration/versioning.go | 5360 ++++++++++++++--------- 6 files changed, 4063 insertions(+), 2538 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 803e3977..018421be 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1264,6 +1264,81 @@ func isRemovableAttr(attr string) bool { } } +// dirObjectAttrs are the attributes that make up the state of a directory +// object. The directory is kept across versions and is also the parent of +// other objects, so it may carry attributes that don't belong to the object: +// only these are copied into, restored from and cleared between versions. +// An attribute stored on objects must also be listed here. +var dirObjectAttrs = []string{ + etagkey, + checksumsKey, + contentTypeHdr, + contentEncHdr, + contentLangHdr, + contentDispHdr, + cacheCtrlHdr, + expiresHdr, + websiteRedirectHdr, + metadataHdr, + tagHdr, + objectLegalHoldKey, + objectRetentionKey, + versionIdKey, + deleteMarkerKey, +} + +// isDirObject reports whether the directory at bucket/key is a directory +// object: only directories created with a put carry an etag +func (p *Posix) isDirObject(bucket, key string) (bool, error) { + _, err := p.meta.RetrieveAttribute(nil, bucket, key, etagkey) + if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get dir etag: %w", err) + } + return true, nil +} + +// isLiveDirObject reports whether fi, the entry at the path of the +// directory object key, is a directory object +func (p *Posix) isLiveDirObject(fi os.FileInfo, bucket, key string) (bool, error) { + if !fi.IsDir() { + return false, nil + } + return p.isDirObject(bucket, key) +} + +// clearDirObjectAttrs removes the directory object attributes, including +// legacy metadata attributes, from the directory at bucket/key. The etag is +// kept: it marks the directory as an object, so a failure before the new +// attributes are stored leaves the object and its versions visible. +func (p *Posix) clearDirObjectAttrs(bucket, key string) error { + attrs, err := p.meta.ListAttributes(bucket, key) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("list object attributes: %w", err) + } + for _, attr := range attrs { + if isValidMeta(attr) { + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + } + + for _, attr := range dirObjectAttrs { + if attr == etagkey { + continue + } + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + return nil +} + // Creates a new copy(version) of an object in the versioning directory func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account, removeAttributes bool) (versionPath string, err error) { sf, err := os.Open(p.ObjectPath(bucket, key)) @@ -1283,9 +1358,19 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun versionId = nullVersionId } - attrs, err := p.meta.ListAttributes(bucket, key) - if err != nil { - return versionPath, fmt.Errorf("load object attributes: %w", err) + // a directory object version is an empty file carrying the + // directory object attributes + isDir := strings.HasSuffix(key, "/") + attrs := dirObjectAttrs + if isDir { + size = 0 + // store legacy metadata attributes as the metadata attribute + p.loadObjectMetadata(sf, bucket, key) + } else { + attrs, err = p.meta.ListAttributes(bucket, key) + if err != nil { + return versionPath, fmt.Errorf("load object attributes: %w", err) + } } versionBucketPath := filepath.Join(p.versioningDir, bucket) @@ -1303,10 +1388,12 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun originalMTime = srcInfo.ModTime() } - // Prioritize copy_file_range for internal file-to-file version copies. - _, err = io.Copy(f.File(), sf) - if err != nil { - return versionPath, err + if !isDir { + // Prioritize copy_file_range for internal file-to-file version copies. + _, err = io.Copy(f.File(), sf) + if err != nil { + return versionPath, err + } } // Restore original mtime after copy @@ -1325,6 +1412,9 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun // Copy the object attributes(metadata) for _, attr := range attrs { data, err := p.meta.RetrieveAttribute(sf, bucket, key, attr) + if isDir && errors.Is(err, meta.ErrNoSuchKey) { + continue + } if err != nil { return versionPath, fmt.Errorf("list %v attribute: %w", attr, err) } @@ -1479,53 +1569,21 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { Truncated: true, }, nil } + // a directory is listed under the key with the trailing slash + key := path if d.IsDir() { - // directory object only happens if directory empty - // check to see if this is a directory object by checking etag - etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) - if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { - return nil, backend.ErrSkipObj - } - if err != nil { - return nil, fmt.Errorf("get etag: %w", err) - } - etag := string(etagBytes) - - fi, err := d.Info() - if errors.Is(err, fs.ErrNotExist) { - return nil, backend.ErrSkipObj - } - if err != nil { - return nil, fmt.Errorf("get fileinfo: %w", err) - } - - key := path + "/" - // Directory objects don't contain data - size := int64(0) - versionId := "null" - - objects = append(objects, s3response.ObjectVersion{ - ETag: &etag, - Key: &key, - LastModified: backend.GetTimePtr(fi.ModTime()), - IsLatest: getBoolPtr(true), - Size: &size, - VersionId: &versionId, - StorageClass: types.ObjectVersionStorageClassStandard, - }) - - return &backend.ObjVersionFuncResult{ - ObjectVersions: objects, - DelMarkers: delMarkers, - Truncated: availableObjCount == 1, - }, nil + key = path + "/" } - // file object, get object info and fill out object data + // get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { return nil, backend.ErrSkipObj } + if d.IsDir() && errors.Is(err, meta.ErrNoSuchKey) { + // a directory is listed only if it's a directory object + return nil, backend.ErrSkipObj + } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get etag: %w", err) } @@ -1552,6 +1610,10 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } size := fi.Size() + if d.IsDir() { + // directory objects don't contain data + size = 0 + } isDel, err := p.isObjDeleteMarker(bucket, path) if err != nil { @@ -1563,7 +1625,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { IsLatest: getBoolPtr(true), VersionId: &versionId, LastModified: backend.GetTimePtr(fi.ModTime()), - Key: &path, + Key: &key, }) } else { // Retrieve checksum @@ -1574,7 +1636,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { objects = append(objects, s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(fi.ModTime()), Size: &size, VersionId: &versionId, @@ -1604,7 +1666,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } // List all the versions of the object in the versioning directory - versionPath := p.genObjVersionPath(bucket, path) + versionPath := p.genObjVersionPath(bucket, key) dirEnts, err := os.ReadDir(versionPath) if errors.Is(err, fs.ErrNotExist) { return &backend.ObjVersionFuncResult{ @@ -1642,7 +1704,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { nullObjDelMarker = &types.DeleteMarkerEntry{ VersionId: backend.GetPtrFromString("null"), LastModified: backend.GetTimePtr(nf.ModTime()), - Key: &path, + Key: &key, IsLatest: getBoolPtr(false), } } else { @@ -1665,7 +1727,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { nullVersionIdObj = &s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(nf.ModTime()), Size: &size, VersionId: backend.GetPtrFromString("null"), @@ -1775,7 +1837,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { delMarkers = append(delMarkers, types.DeleteMarkerEntry{ VersionId: &versionId, LastModified: backend.GetTimePtr(f.ModTime()), - Key: &path, + Key: &key, IsLatest: getBoolPtr(false), }) } else { @@ -1786,7 +1848,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } objects = append(objects, s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(f.ModTime()), Size: &size, VersionId: &versionId, @@ -3889,8 +3951,23 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) if err != nil { return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) } + if strings.HasSuffix(srcObject, "/") != fi.IsDir() { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + // a directory object holds no data + srcSize := fi.Size() + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyPartResult{}, err + } + if !isObj { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + } - startOffset, length, err := backend.ParseCopySourceRange(fi.Size(), *upi.CopySourceRange) + startOffset, length, err := backend.ParseCopySourceRange(srcSize, *upi.CopySourceRange) if err != nil { return s3response.CopyPartResult{}, err } @@ -4128,11 +4205,22 @@ func (p *Posix) snapshotObjVersion(bucket, key string, vStatus types.BucketVersi return nil } + isDir := strings.HasSuffix(key, "/") d, err := os.Stat(p.ObjectPath(bucket, key)) - if err != nil || d.IsDir() { + if err != nil || d.IsDir() != isDir { // nothing to snapshot return nil } + if isDir { + isObj, err := p.isDirObject(bucket, key) + if err != nil { + return err + } + if !isObj { + // nothing to snapshot + return nil + } + } var isVersionIdMissing bool if p.isBucketVersioningSuspended(vStatus) { @@ -4268,6 +4356,37 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, err } + expectedSum := getEmptyChecksumValue(checksumAlgorithm) + if checksumValue != "" && expectedSum != checksumValue { + return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + + // reject object lock settings the bucket doesn't support before + // the directory object is changed + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn || po.ObjectLockMode != "" { + err = p.isBucketObjectLockEnabled(*po.Bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err + } + versioned := p.versioningEnabled() && vStatus != "" + + // In a versioned bucket the directory is kept across versions: its + // current version is copied to the versioning directory and its + // object attributes are then replaced with the new version's. + err = p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + if err != nil { + return s3response.PutObjectOutput{}, err + } + err = p.mkdirAll(name, uid, gid, doChown) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -4279,6 +4398,23 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, err } + var versionID string + if versioned { + err = p.clearDirObjectAttrs(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } else { + versionID = ulid.Make().String() + } + } + err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err) @@ -4292,8 +4428,6 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - expectedSum := getEmptyChecksumValue(checksumAlgorithm) - dirETag := emptyMD5 if p.dataIntegrityEtag { dirETag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksumAlgorithm)), expectedSum) @@ -4321,10 +4455,6 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - if checksumValue != "" && expectedSum != checksumValue { - return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) - } - // set empty checksum checksum := s3response.Checksum{ Type: types.ChecksumTypeFullObject, @@ -4338,9 +4468,29 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) } - // for directory object no version is created + if versionID != "" { + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + if versioned { + // The directory mtime is the version's last modified time. + // Setting it needs the directory to be owned by the gateway, + // otherwise it's left as is. + now := time.Now() + _ = os.Chtimes(name, now, now) + } + + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err + } + return s3response.PutObjectOutput{ ETag: dirETag, + VersionID: versionID, Size: &contentLength, ChecksumType: checksum.Type, ChecksumCRC32: checksum.CRC32, @@ -4614,34 +4764,9 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - // Set object legal hold - if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", true) - if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { - err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) - } - return s3response.PutObjectOutput{}, err - } - } - - // Set object retention - if po.ObjectLockMode != "" { - retention := types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), - RetainUntilDate: po.ObjectLockRetainUntilDate, - } - retParsed, err := json.Marshal(retention) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) - } - err = p.PutObjectRetention(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", retParsed) - if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { - err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) - } - return s3response.PutObjectOutput{}, err - } + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err } return s3response.PutObjectOutput{ @@ -4662,6 +4787,42 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje }, nil } +// putObjectLockSettings sets the legal hold and retention requested with +// the put on the object that was just published +func (p *Posix) putObjectLockSettings(ctx context.Context, po s3response.PutObjectInput) error { + // Set object legal hold + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + err := p.PutObjectLegalHold(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", true) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + // Set object retention + if po.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), + RetainUntilDate: po.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + return fmt.Errorf("parse object lock retention: %w", err) + } + err = p.PutObjectRetention(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + return nil +} + func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { release, err := p.acquireActionSlot(ctx) if err != nil { @@ -4721,8 +4882,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }) } - // Directory objects can't have versions - if !isDir && p.versioningEnabled() && vStatus != "" { + if p.versioningEnabled() && vStatus != "" { if getString(input.VersionId) == "" { // if the versionId is not specified, make the current version a delete marker fi, err := os.Stat(objpath) @@ -4736,6 +4896,16 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if isDir { + isObj, err := p.isLiveDirObject(fi, bucket, object) + if err != nil { + return nil, err + } + if !isObj { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + } err = evalPreconditions(fi, bucket, object) if err != nil { @@ -4792,6 +4962,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } else { versionPath := p.genObjVersionPath(bucket, object) + if isDir { + // the attributes at a directory object path may belong to a + // file or to a directory that isn't an object + fi, err := os.Stat(objpath) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + isObj, err := p.isLiveDirObject(fi, bucket, object) + if err != nil { + return nil, err + } + if !isObj { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + } + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist @@ -4829,6 +5023,16 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, err } + if isDir { + err = p.deleteDirObjectLatestVersion(bucket, object) + if err != nil { + return nil, err + } + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } err = os.Remove(objpath) if err != nil && !errors.Is(err, fs.ErrNotExist) && !isErrNotDir(err) { return nil, fmt.Errorf("remove obj version: %w", err) @@ -4862,7 +5066,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }, nil } - srcObjVersion, err := ents[len(ents)-1].Info() + srcObjVersion, err := latestObjVersion(ents).Info() if err != nil { return nil, fmt.Errorf("get file info: %w", err) } @@ -5033,6 +5237,93 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } +// latestObjVersion returns the entry of the version, among the version +// directory entries, that becomes the latest one when the latest version +// of the object is deleted +func latestObjVersion(ents []fs.DirEntry) fs.DirEntry { + return ents[len(ents)-1] +} + +// deleteDirObjectLatestVersion removes the latest version of the directory +// object at bucket/key. The newest remaining version is restored onto the +// directory. With no versions left the directory stops being an object and +// is removed, unless other objects are under it. +func (p *Posix) deleteDirObjectLatestVersion(bucket, key string) error { + objpath := p.ObjectPath(bucket, key) + versionPath := p.genObjVersionPath(bucket, key) + + ents, err := os.ReadDir(versionPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + err := os.Remove(objpath) + if isErrDirNotEmpty(err) { + // the directory stays as the parent of the objects under it + err = p.clearDirObjectAttrs(bucket, key) + if err != nil { + return err + } + err = p.meta.DeleteAttribute(bucket, key, etagkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove etag attribute: %w", err) + } + return nil + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove dir object: %w", err) + } + err = p.meta.DeleteAttributes(bucket, key) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("delete object attributes: %w", err) + } + p.removeParents(bucket, key) + return nil + } + + srcVersion := latestObjVersion(ents) + srcVersionId := srcVersion.Name() + srcInfo, err := srcVersion.Info() + if err != nil { + return fmt.Errorf("get file info: %w", err) + } + + // replace the attributes in place, so that the directory keeps its etag + for _, attr := range dirObjectAttrs { + data, err := p.meta.RetrieveAttribute(nil, versionPath, srcVersionId, attr) + if errors.Is(err, meta.ErrNoSuchKey) { + err = p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + continue + } + if err != nil { + return fmt.Errorf("load %v attribute: %w", attr, err) + } + + err = p.meta.StoreAttribute(nil, bucket, key, attr, data) + if err != nil { + return fmt.Errorf("store %v attribute: %w", attr, err) + } + } + + err = os.Remove(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return fmt.Errorf("remove obj version: %w", err) + } + + _ = p.meta.DeleteAttributes(versionPath, srcVersionId) + p.removeParents(filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), srcVersionId)) + + // The restored version keeps its last modified time. Setting it needs + // the directory to be owned by the gateway, otherwise it's left as is. + _ = os.Chtimes(objpath, time.Now(), srcInfo.ModTime()) + + return nil +} + func (p *Posix) removeParents(bucket, object string) { // this will remove all parent directories that were not // specifically uploaded with a put object. we detect @@ -5195,12 +5486,12 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge // in '/') have an etag attribute. Directories created incidentally on the // filesystem or as parent directories during object upload should not be // accessible via get-object. - _, derr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) - if errors.Is(derr, meta.ErrNoSuchKey) || errors.Is(derr, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err } - if derr != nil { - return nil, fmt.Errorf("get dir etag: %w", derr) + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } } @@ -5241,6 +5532,18 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge return nil, err } + // If versioning is configured get the object versionId + if p.versioningEnabled() && versionId == "" { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if errors.Is(err, meta.ErrNoSuchKey) { + versionId = nullVersionId + } else if err != nil { + return nil, err + } + + versionId = string(vId) + } + if fid.IsDir() { _, _, _, err := backend.ParseObjectRange(0, *input.Range) if err != nil { @@ -5299,18 +5602,6 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge }, nil } - // If versioning is configured get the object versionId - if p.versioningEnabled() && versionId == "" { - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, meta.ErrNoSuchKey) { - versionId = nullVersionId - } else if err != nil { - return nil, err - } - - versionId = string(vId) - } - // openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent // DeleteObject can call os.Remove on this file while the GET response body // is still being streamed. On POSIX, os.Open is sufficient. @@ -5535,12 +5826,12 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. // in '/') have an etag attribute. Directories created incidentally on the // filesystem or as parent directories during object upload should not be // accessible via head-object. - _, derr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) - if errors.Is(derr, meta.ErrNoSuchKey) || errors.Is(derr, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err } - if derr != nil { - return nil, fmt.Errorf("get dir etag: %w", derr) + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } } @@ -5878,7 +6169,21 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput if !strings.HasSuffix(srcObject, "/") && fi.IsDir() { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if fi.Size() > p.copyObjectThreshold { + // a directory object holds no data + srcSize := fi.Size() + var srcBody io.Reader = f + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if !isObj { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + srcBody = strings.NewReader("") + } + if srcSize > p.copyObjectThreshold { return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(p.copyObjectThreshold) } @@ -6042,7 +6347,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput } } } else { - contentLength := fi.Size() + contentLength := srcSize checksums, err := p.retrieveChecksums(f, srcBucket, srcObject) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6058,7 +6363,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput putObjectInput := s3response.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, - Body: f, + Body: srcBody, ContentLength: &contentLength, ChecksumAlgorithm: checksums.Algorithm, ContentType: input.ContentType, @@ -6237,7 +6542,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } } if d.IsDir() { - // directory object only happens if directory empty + // a directory is listed only if it's a directory object // check to see if this is a directory object by checking etag etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { @@ -6248,6 +6553,12 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } etag := string(etagBytes) + // If the directory object is a delete marker, skip + isDel, _ := p.isObjDeleteMarker(bucket, path) + if isDel { + return s3response.Object{}, backend.ErrSkipObj + } + fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { return s3response.Object{}, backend.ErrSkipObj diff --git a/backend/walk.go b/backend/walk.go index 767004fc..d0b9bfda 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -177,11 +177,18 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM return fs.SkipDir } + // A directory is compared to the markers as the key of its + // directory object, which has a trailing slash. + key := path + if d.IsDir() { + key += "/" + } + if !pastMarker { - if path == keyMarker { + if key == keyMarker { pastMarker = true } - if path < keyMarker { + if key < keyMarker { return nil } } @@ -195,28 +202,32 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM // building to match. So only skip if path isn't a prefix of prefix // and prefix isn't a prefix of path. if prefix != "" && - !strings.HasPrefix(path+"/", prefix) && - !strings.HasPrefix(prefix, path+"/") { + !strings.HasPrefix(key, prefix) && + !strings.HasPrefix(prefix, key) { return fs.SkipDir } // Don't recurse into subdirectories when listing with delimiter. + // The walk also enters the directories that sort before the key + // marker, so the common prefix is the part of the key up to the + // first delimiter after the prefix. if delimiter == "/" && - prefix != path+"/" && - strings.HasPrefix(path+"/", prefix) { - cpmap.Add(path + "/") + prefix != key && + strings.HasPrefix(key, prefix) { + before, _, _ := strings.Cut(strings.TrimPrefix(key, prefix), delimiter) + cpmap.Add(prefix + before + delimiter) return fs.SkipDir } // Skip ancestor directories of the specified prefix; only process // the directory that exactly matches the prefix. - // At this point we know strings.HasPrefix(prefix, path+"/") holds + // At this point we know strings.HasPrefix(prefix, key) holds // (i.e. path is an ancestor of the prefix directory). Skip it // unless it is the exact prefix directory. // Note: WalkVersions always walks from "." (unlike Walk, which // narrows the root) because versioning marker semantics require // visiting all entries in order, so this guard is needed instead. - if prefix != "" && strings.HasPrefix(prefix, path+"/") && path+"/" != prefix { + if prefix != "" && strings.HasPrefix(prefix, key) && key != prefix { return nil } @@ -231,7 +242,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM delMarkers = append(delMarkers, res.DelMarkers...) if res.Truncated { truncated = true - nextMarker = path + nextMarker = key nextVersionIdMarker = res.NextVersionIdMarker return fs.SkipAll } diff --git a/tests/integration/ListObjectVersions.go b/tests/integration/ListObjectVersions.go index 27a3932e..eaf9b51c 100644 --- a/tests/integration/ListObjectVersions.go +++ b/tests/integration/ListObjectVersions.go @@ -26,7 +26,7 @@ import ( func ListObjectVersions_VD_success(s *S3Conf) error { testName := "ListObjectVersions_VD_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - versions := []types.ObjectVersion{} + versions, dirVersions := []types.ObjectVersion{}, []types.ObjectVersion{} for i := range 5 { dLgth := int64(i * 100) key := fmt.Sprintf("my-obj-%v", i) @@ -46,8 +46,29 @@ func ListObjectVersions_VD_success(s *S3Conf) error { VersionId: getPtr("null"), StorageClass: types.ObjectVersionStorageClassStandard, }) + + dir := fmt.Sprintf("my-dir-%v/", i) + out, err = putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &dir, + }, s3client) + if err != nil { + return err + } + + dirVersions = append(dirVersions, types.ObjectVersion{ + ETag: out.res.ETag, + IsLatest: getBoolPtr(true), + Key: &dir, + Size: getPtr(int64(0)), + VersionId: getPtr("null"), + StorageClass: types.ObjectVersionStorageClassStandard, + }) } + // the directory objects are listed before the regular objects + versions = append(dirVersions, versions...) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, @@ -97,11 +118,68 @@ func ListObjectVersions_negative_max_keys(s *S3Conf) error { func ListObjectVersions_list_single_object_versions(s *S3Conf) error { testName := "ListObjectVersions_list_single_object_versions" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - object := "my-obj" - versions, err := createObjVersions(s3client, bucket, object, 5) + return forEachKey([]string{"my-obj", "my-dir/"}, func(object string) error { + versions, err := createObjVersions(s3client, bucket, object, 5) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &object, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, out.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, out.Versions) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { + testName := "ListObjectVersions_list_multiple_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2, obj3 := "foo", "bar", "baz" + // "qux-quux" sorts before the directory object "qux/", which + // sorts before the object under it + dir, beforeDir, inDir := "qux/", "qux-quux", "qux/quux" + + obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) if err != nil { return err } + obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + if err != nil { + return err + } + obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + if err != nil { + return err + } + // "qux/" is put over the existing parent directory of "qux/quux" + inDirVersions, err := createObjVersions(s3client, bucket, inDir, 2) + if err != nil { + return err + } + dirVersions, err := createObjVersions(s3client, bucket, dir, 3) + if err != nil { + return err + } + beforeDirVersions, err := createObjVersions(s3client, bucket, beforeDir, 2) + if err != nil { + return err + } + + versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + versions = append(append(append(versions, beforeDirVersions...), dirVersions...), inDirVersions...) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ @@ -121,38 +199,100 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } -func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { - testName := "ListObjectVersions_list_multiple_object_versions" +func ListObjectVersions_dir_object_with_children(s *S3Conf) error { + testName := "ListObjectVersions_dir_object_with_children" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj1, obj2, obj3 := "foo", "bar", "baz" + obj, dir, child, nestedObj := "my-obj", "my-dir/", "my-dir/child", "my-dir/sub/obj" - obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) + objVersions, err := createObjVersions(s3client, bucket, obj, 2) if err != nil { return err } - obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + dirVersions, err := createObjVersions(s3client, bucket, dir, 3) if err != nil { return err } - obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + childVersions, err := createObjVersions(s3client, bucket, child, 2) + if err != nil { + return err + } + nestedVersions, err := createObjVersions(s3client, bucket, nestedObj, 1) if err != nil { return err } - versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) - + // the objects under "my-dir/" stay listed after it's deleted ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, + Key: &dir, }) cancel() if err != nil { return err } - if !compareVersions(versions, out.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, out.Versions) + dirVersions[0].IsLatest = getBoolPtr(false) + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getBoolPtr(true), + Key: &dir, + VersionId: res.VersionId, + }, + } + + for i, test := range []struct { + prefix *string + delimiter *string + versions []types.ObjectVersion + delMarkers []types.DeleteMarkerEntry + prefixes []string + }{ + // "my-dir/" is rolled up into a common prefix + { + delimiter: getPtr("/"), + versions: objVersions, + prefixes: []string{dir}, + }, + // "my-dir/" itself and "my-dir/child" have no delimiter + // after the prefix + { + prefix: &dir, + delimiter: getPtr("/"), + versions: append(dirVersions, childVersions...), + delMarkers: delMarkers, + prefixes: []string{"my-dir/sub/"}, + }, + // "my-dir/sub" isn't a directory object + { + prefix: &dir, + versions: append(append(dirVersions, childVersions...), nestedVersions...), + delMarkers: delMarkers, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: test.prefix, + Delimiter: test.delimiter, + }) + cancel() + if err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + + if !compareVersions(test.versions, out.Versions) { + return fmt.Errorf("test case %d failed: expected the versions to be %v, instead got %v", + i, sprintVersions(test.versions), sprintVersions(out.Versions)) + } + if !compareDelMarkers(test.delMarkers, out.DeleteMarkers) { + return fmt.Errorf("test case %d failed: expected the delete markers to be %v, instead got %v", + i, test.delMarkers, out.DeleteMarkers) + } + if !comparePrefixes(test.prefixes, out.CommonPrefixes) { + return fmt.Errorf("test case %d failed: expected the common prefixes to be %v, instead got %v", + i, test.prefixes, sprintPrefixes(out.CommonPrefixes)) + } } return nil @@ -162,7 +302,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { testName := "ListObjectVersions_multiple_object_versions_truncated" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj1, obj2, obj3 := "foo", "bar", "baz" + obj1, obj2, obj3, dir, child := "foo", "bar", "baz", "dir/", "dir/child" obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) if err != nil { @@ -176,87 +316,85 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { if err != nil { return err } + dirVersions, err := createObjVersions(s3client, bucket, dir, 4) + if err != nil { + return err + } + childVersions, err := createObjVersions(s3client, bucket, child, 3) + if err != nil { + return err + } - versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + versions := append(append(obj2Versions, obj3Versions...), dirVersions...) + versions = append(append(versions, childVersions...), obj1Versions...) maxKeys := int32(5) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - MaxKeys: &maxKeys, - }) - cancel() - if err != nil { - return err - } + // the pages end on noncurrent versions of "baz", "dir/" and "dir/child" + var keyMarker, versionIdMarker *string + for page := 0; page*int(maxKeys) < len(versions); page++ { + start := page * int(maxKeys) + end := min(start+int(maxKeys), len(versions)) + truncated := end < len(versions) - if out.Name == nil { - return fmt.Errorf("expected the bucket name to be %v, instead got nil", - bucket) - } - if *out.Name != bucket { - return fmt.Errorf("expected the bucket name to be %v, instead got %v", - bucket, *out.Name) - } - if out.IsTruncated == nil || !*out.IsTruncated { - return fmt.Errorf("expected the output to be truncated") - } - if out.MaxKeys == nil { - return fmt.Errorf("expected the max-keys to be %v, instead got nil", - maxKeys) - } - if *out.MaxKeys != maxKeys { - return fmt.Errorf("expected the max-keys to be %v, instead got %v", - maxKeys, *out.MaxKeys) - } - if getString(out.NextKeyMarker) != getString(versions[maxKeys-1].Key) { - return fmt.Errorf("expected the NextKeyMarker to be %v, instead got %v", - getString(versions[maxKeys-1].Key), getString(out.NextKeyMarker)) - } - if getString(out.NextVersionIdMarker) != getString(versions[maxKeys-1].VersionId) { - return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", - getString(versions[maxKeys-1].VersionId), getString(out.NextVersionIdMarker)) - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }) + cancel() + if err != nil { + return fmt.Errorf("page %v: %w", page, err) + } - if !compareVersions(versions[:maxKeys], out.Versions) { - return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", - sprintVersions(versions[:maxKeys]), sprintVersions(out.Versions)) - } + if out.Name == nil { + return fmt.Errorf("page %v: expected the bucket name to be %v, instead got nil", + page, bucket) + } + if *out.Name != bucket { + return fmt.Errorf("page %v: expected the bucket name to be %v, instead got %v", + page, bucket, *out.Name) + } + isTruncated := out.IsTruncated != nil && *out.IsTruncated + if isTruncated != truncated { + return fmt.Errorf("page %v: expected the output truncation to be %v, instead got %v", + page, truncated, isTruncated) + } + if out.MaxKeys == nil { + return fmt.Errorf("page %v: expected the max-keys to be %v, instead got nil", + page, maxKeys) + } + if *out.MaxKeys != maxKeys { + return fmt.Errorf("page %v: expected the max-keys to be %v, instead got %v", + page, maxKeys, *out.MaxKeys) + } + if getString(out.KeyMarker) != getString(keyMarker) { + return fmt.Errorf("page %v: expected the KeyMarker to be %v, instead got %v", + page, getString(keyMarker), getString(out.KeyMarker)) + } + if getString(out.VersionIdMarker) != getString(versionIdMarker) { + return fmt.Errorf("page %v: expected the VersionIdMarker to be %v, instead got %v", + page, getString(versionIdMarker), getString(out.VersionIdMarker)) + } + if truncated { + last := versions[end-1] + if getString(out.NextKeyMarker) != getString(last.Key) { + return fmt.Errorf("page %v: expected the NextKeyMarker to be %v, instead got %v", + page, getString(last.Key), getString(out.NextKeyMarker)) + } + if getString(out.NextVersionIdMarker) != getString(last.VersionId) { + return fmt.Errorf("page %v: expected the NextVersionIdMarker to be %v, instead got %v", + page, getString(last.VersionId), getString(out.NextVersionIdMarker)) + } + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - KeyMarker: out.NextKeyMarker, - VersionIdMarker: out.NextVersionIdMarker, - }) - cancel() - if err != nil { - return err - } + if !compareVersions(versions[start:end], out.Versions) { + return fmt.Errorf("page %v: expected the resulting object versions to be %v, instead got %v", + page, sprintVersions(versions[start:end]), sprintVersions(out.Versions)) + } - if out.Name == nil { - return fmt.Errorf("expected the bucket name to be %v, instead got nil", - bucket) - } - if *out.Name != bucket { - return fmt.Errorf("expected the bucket name to be %v, instead got %v", - bucket, *out.Name) - } - if out.IsTruncated != nil && *out.IsTruncated { - return fmt.Errorf("expected the output not to be truncated") - } - if getString(out.KeyMarker) != getString(versions[maxKeys-1].Key) { - return fmt.Errorf("expected the KeyMarker to be %v, instead got %v", - getString(versions[maxKeys-1].Key), getString(out.KeyMarker)) - } - if getString(out.VersionIdMarker) != getString(versions[maxKeys-1].VersionId) { - return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", - getString(versions[maxKeys-1].VersionId), getString(out.VersionIdMarker)) - } - - if !compareVersions(versions[maxKeys:], out.Versions) { - return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", - sprintVersions(versions[:maxKeys]), sprintVersions(out.Versions)) + keyMarker, versionIdMarker = out.NextKeyMarker, out.NextVersionIdMarker } return nil @@ -266,170 +404,180 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { func ListObjectVersions_with_delete_markers(s *S3Conf) error { testName := "ListObjectVersions_with_delete_markers" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - versions[0].IsLatest = getBoolPtr(false) + versions[0].IsLatest = getBoolPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + delMarkers := []types.DeleteMarkerEntry{} + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + Key: &obj, + VersionId: out.VersionId, + IsLatest: getBoolPtr(true), + }) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(res.DeleteMarkers, delMarkers) { + return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + return nil }) - cancel() - if err != nil { - return err - } - - delMarkers := []types.DeleteMarkerEntry{} - delMarkers = append(delMarkers, types.DeleteMarkerEntry{ - Key: &obj, - VersionId: out.VersionId, - IsLatest: getBoolPtr(true), - }) - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, res.Versions) - } - if !compareDelMarkers(res.DeleteMarkers, delMarkers) { - return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", - delMarkers, res.DeleteMarkers) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error { testName := "ListObjectVersions_containing_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) - if err != nil { - return err - } + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } - objLgth := int64(543) - out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + objLgth := objDataLen(obj, 543) + out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - versions[0].IsLatest = getBoolPtr(false) + versions[0].IsLatest = getBoolPtr(false) - versions = append([]types.ObjectVersion{ - { - ETag: out.res.ETag, - IsLatest: getBoolPtr(false), - Key: &obj, - Size: &objLgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - }, - }, versions...) + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &objLgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + }, + }, versions...) - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } - newVersions, err := createObjVersions(s3client, bucket, obj, 4) - if err != nil { - return err - } + newVersions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } - versions = append(newVersions, versions...) + versions = append(newVersions, versions...) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed object versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed object versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { testName := "ListObjectVersions_single_null_versionId_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, objLgth := "my-obj", int64(890) - out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + // the objects are put before versioning is enabled and are + // listed in this order + objs, objLgth := []string{"my-dir/", "my-obj"}, int64(890) + versions := []types.ObjectVersion{} + for _, obj := range objs { + size := objDataLen(obj, objLgth) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - versions := []types.ObjectVersion{ - { + versions = append(versions, types.ObjectVersion{ ETag: out.res.ETag, Key: &obj, StorageClass: types.ObjectVersionStorageClassStandard, IsLatest: getBoolPtr(false), - Size: &objLgth, + Size: &size, VersionId: &nullVersionId, - }, + }) } - delMarkers := []types.DeleteMarkerEntry{ - { + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + delMarkers := []types.DeleteMarkerEntry{} + for _, obj := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ IsLatest: getBoolPtr(true), Key: &obj, VersionId: res.VersionId, - }, + }) } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, }) @@ -454,7 +602,7 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { func ListObjectVersions_checksum(s *S3Conf) error { testName := "ListObjectVersions_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - versions := []types.ObjectVersion{} + versions, dirVersions := []types.ObjectVersion{}, []types.ObjectVersion{} for i, algo := range types.ChecksumAlgorithmCrc32.Values() { vers, err := createObjVersions(s3client, bucket, fmt.Sprintf("obj-%v", i), 1, withChecksumAlgo(algo)) if err != nil { @@ -462,8 +610,18 @@ func ListObjectVersions_checksum(s *S3Conf) error { } versions = append(versions, vers...) + + vers, err = createObjVersions(s3client, bucket, fmt.Sprintf("dir-%v/", i), 1, withChecksumAlgo(algo)) + if err != nil { + return err + } + + dirVersions = append(dirVersions, vers...) } + // the directory objects are listed before the regular objects + versions = append(dirVersions, versions...) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 78b7e4c5..02b9e1a1 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -1941,6 +1941,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_PutObject_null_versionId_obj) ts.Run(Versioning_PutObject_overwrite_null_versionId_obj) ts.Run(Versioning_PutObject_success) + ts.Run(Versioning_PutObject_dir_object_new_version_resets_attributes) // CopyObject action ts.Run(Versioning_CopyObject_invalid_versionId) ts.Run(Versioning_CopyObject_encoded_versionid_separator_invalid_versionId) @@ -1957,6 +1958,7 @@ func TestVersioning(ts *TestState) { } ts.Run(Versioning_HeadObject_invalid_parent) ts.Run(Versioning_HeadObject_success) + ts.Run(Versioning_HeadObject_dir_object_versions) ts.Run(Versioning_HeadObject_without_versionId) ts.Run(Versioning_HeadObject_delete_marker) // GetObject action @@ -1982,9 +1984,12 @@ func TestVersioning(ts *TestState) { // DeleteObject actions ts.Run(Versioning_DeleteObject_invalid_versionId) ts.Run(Versioning_DeleteObject_delete_object_version) + ts.Run(Versioning_DeleteObject_dir_object_latest_version) ts.Run(Versioning_DeleteObject_non_existing_object) + ts.Run(Versioning_DeleteObject_implicit_dir) if !ts.conf.windowsTests { ts.Run(Versioning_DeleteObject_delete_a_delete_marker) + ts.Run(Versioning_DeleteObject_dir_object_with_children) } ts.Run(Versioning_Delete_null_versionId_object) ts.Run(Versioning_DeleteObject_nested_dir_object) @@ -1997,6 +2002,7 @@ func TestVersioning(ts *TestState) { ts.Run(ListObjectVersions_negative_max_keys) ts.Run(ListObjectVersions_list_single_object_versions) ts.Run(ListObjectVersions_list_multiple_object_versions) + ts.Run(ListObjectVersions_dir_object_with_children) ts.Run(ListObjectVersions_multiple_object_versions_truncated) ts.Run(ListObjectVersions_with_delete_markers) ts.Run(ListObjectVersions_containing_null_versionId_obj) @@ -2029,6 +2035,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_Put_GetObjectLegalHold_success) // WORM protection ts.Run(Versioning_WORM_obj_version_locked_with_legal_hold) + ts.Run(Versioning_WORM_dir_object_lock_headers) ts.Run(Versioning_WORM_obj_version_locked_with_governance_retention) ts.Run(Versioning_WORM_obj_version_locked_with_compliance_retention) ts.Run(Versioning_WORM_delete_marker_locked_object_legal_hold) @@ -3464,6 +3471,7 @@ func GetIntTests() IntTests { "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, "Versioning_PutObject_success": Versioning_PutObject_success, + "Versioning_PutObject_dir_object_new_version_resets_attributes": Versioning_PutObject_dir_object_new_version_resets_attributes, "Versioning_CopyObject_invalid_versionId": Versioning_CopyObject_invalid_versionId, "Versioning_CopyObject_encoded_versionid_separator_invalid_versionId": Versioning_CopyObject_encoded_versionid_separator_invalid_versionId, "Versioning_CopyObject_success": Versioning_CopyObject_success, @@ -3474,6 +3482,7 @@ func GetIntTests() IntTests { "Versioning_HeadObject_non_existing_object_version": Versioning_HeadObject_non_existing_object_version, "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, "Versioning_HeadObject_success": Versioning_HeadObject_success, + "Versioning_HeadObject_dir_object_versions": Versioning_HeadObject_dir_object_versions, "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, @@ -3495,8 +3504,11 @@ func GetIntTests() IntTests { "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, "Versioning_DeleteObject_invalid_versionId": Versioning_DeleteObject_invalid_versionId, "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_dir_object_latest_version": Versioning_DeleteObject_dir_object_latest_version, "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, + "Versioning_DeleteObject_implicit_dir": Versioning_DeleteObject_implicit_dir, "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_DeleteObject_dir_object_with_children": Versioning_DeleteObject_dir_object_with_children, "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, "Versioning_DeleteObject_nested_dir_object": Versioning_DeleteObject_nested_dir_object, "Versioning_DeleteObject_non_existing_objects": Versioning_DeleteObject_non_existing_objects, @@ -3507,6 +3519,7 @@ func GetIntTests() IntTests { "ListObjectVersions_negative_max_keys": ListObjectVersions_negative_max_keys, "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, + "ListObjectVersions_dir_object_with_children": ListObjectVersions_dir_object_with_children, "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, @@ -3534,6 +3547,7 @@ func GetIntTests() IntTests { "Versioning_PutGetObjectLegalHold_delete_marker": Versioning_PutGetObjectLegalHold_delete_marker, "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, + "Versioning_WORM_dir_object_lock_headers": Versioning_WORM_dir_object_lock_headers, "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, "Versioning_WORM_delete_marker_locked_object_legal_hold": Versioning_WORM_delete_marker_locked_object_legal_hold, diff --git a/tests/integration/utils.go b/tests/integration/utils.go index a04b4d02..e12976bb 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -1234,22 +1234,24 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, var csum [32]byte var data []byte - if input.Body == nil && lgth != 0 { - data = make([]byte, lgth) - rand.Read(data) + if input.Body == nil { + if lgth != 0 { + data = make([]byte, lgth) + rand.Read(data) - csum = sha256.Sum256(data) - if cfg.checksumAlgorithm != "" { - hasher, err := NewHasher(cfg.checksumAlgorithm) - if err != nil { - return nil, err + if cfg.checksumAlgorithm != "" { + hasher, err := NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, err + } + + hasher.Write(data) + sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) } - - hasher.Write(data) - sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) + input.Body = bytes.NewReader(data) } - input.Body = bytes.NewReader(data) + csum = sha256.Sum256(data) } ctx, cancel := context.WithTimeout(context.Background(), longTimeout) @@ -2298,10 +2300,10 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts versions := []types.ObjectVersion{} for i := range count { rNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) - dataLength := rNumber.Int64() if err != nil { return nil, err } + dataLength := objDataLen(object, rNumber.Int64()) r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, @@ -2373,6 +2375,27 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts return versions, nil } +// objDataLen returns the data length to upload for key: a directory +// object can't hold data +func objDataLen(key string, lgth int64) int64 { + if strings.HasSuffix(key, "/") { + return 0 + } + return lgth +} + +// forEachKey runs fn for each of the keys and prefixes a returned error +// with the key it failed for. Tests use it to run the same scenario for +// a regular object and a directory object. +func forEachKey(keys []string, fn func(key string) error) error { + for _, key := range keys { + if err := fn(key); err != nil { + return fmt.Errorf("%v: %w", key, err) + } + } + return nil +} + // ReverseSlice reverses a slice of any type func reverseSlice[T any](s []T) []T { for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 88bc015a..5af1e0ed 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -33,54 +33,88 @@ import ( func Versioning_DeleteBucket_not_empty(s *S3Conf) error { testName := "Versioning_DeleteBucket_not_empty" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty)); err != nil { + return err + } + + // delete the key versions, so that the next key + // is the only one in the bucket + for _, version := range versions { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err != nil { + return err + } + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutObject_suspended_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_suspended_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - out, err := putObjectWithData(1222, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, 1222), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - return nil + return nil + }) }, withVersioning(types.BucketVersioningStatusSuspended)) } func Versioning_PutObject_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, lgth := "my-obj", int64(1234) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + nullVersions := map[string]types.ObjectVersion{} + err := forEachKey(keys, func(obj string) error { + lgth := objDataLen(obj, 1234) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + nullVersions[obj] = types.ObjectVersion{ + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + } + return nil + }) if err != nil { return err } @@ -91,46 +125,45 @@ func Versioning_PutObject_null_versionId_obj(s *S3Conf) error { return err } - versions, err := createObjVersions(s3client, bucket, obj, 4) - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } - versions = append(versions, types.ObjectVersion{ - ETag: out.res.ETag, - IsLatest: getBoolPtr(false), - Key: &obj, - Size: &lgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, + versions = append(versions, nullVersions[obj]) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }) } func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_overwrite_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := putObjectWithData(int64(1233), &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + err := forEachKey(keys, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 1233), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + return err + }) if err != nil { return err } @@ -141,7 +174,16 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { return err } - versions, err := createObjVersions(s3client, bucket, obj, 4) + objVersions := map[string][]types.ObjectVersion{} + err = forEachKey(keys, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } + + objVersions[obj] = versions + return nil + }) if err != nil { return err } @@ -152,67 +194,221 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { return err } - lgth := int64(3200) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + lgth := objDataLen(obj, 3200) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - versions[0].IsLatest = getBoolPtr(false) + versions := objVersions[obj] + versions[0].IsLatest = getBoolPtr(false) - versions = append([]types.ObjectVersion{ - { - ETag: out.res.ETag, - IsLatest: getBoolPtr(true), - Key: &obj, - Size: &lgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: out.res.ChecksumType, - }, - }, versions...) + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(true), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.res.ChecksumType, + }, + }, versions...) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }) } func Versioning_PutObject_success(s *S3Conf) error { testName := "Versioning_PutObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected the versionId to be returned") + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_PutObject_dir_object_new_version_resets_attributes(s *S3Conf) error { + testName := "Versioning_PutObject_dir_object_new_version_resets_attributes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + tags := []types.Tag{ + {Key: getPtr("key1"), Value: getPtr("val1")}, + {Key: getPtr("key2"), Value: getPtr("val2")}, + } + meta := map[string]string{ + "foo": "bar", + "baz": "quxx", + } + redirect := "/some/redirect" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: getPtr("my-obj"), + first, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key1=val1&key2=val2"), + Metadata: meta, + WebsiteRedirectLocation: &redirect, }) cancel() if err != nil { return err } - if res.VersionId == nil || *res.VersionId == "" { - return fmt.Errorf("expected the versionId to be returned") + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + second, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(first.VersionId) == "" || getString(second.VersionId) == "" { + return fmt.Errorf("expected non empty versionIds, instead got %q and %q", + getString(first.VersionId), getString(second.VersionId)) + } + if *first.VersionId == *second.VersionId { + return fmt.Errorf("expected a new versionId, instead got %v twice", *first.VersionId) + } + + // the current version holds none of the first version attributes + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + cur, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(cur.VersionId) != *second.VersionId { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + *second.VersionId, getString(cur.VersionId)) + } + if len(cur.Metadata) != 0 { + return fmt.Errorf("expected empty metadata, instead got %v", cur.Metadata) + } + if cur.WebsiteRedirectLocation != nil { + return fmt.Errorf("expected nil website-redirect-location, instead got %v", + *cur.WebsiteRedirectLocation) + } + if cur.TagCount != nil { + return fmt.Errorf("expected nil tag count, instead got %v", *cur.TagCount) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + curTags, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if len(curTags.TagSet) != 0 { + return fmt.Errorf("expected empty tag set, instead got %v", curTags.TagSet) + } + + // the first version keeps its attributes + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + old, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: first.VersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(old.VersionId) != *first.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *first.VersionId, getString(old.VersionId)) + } + if old.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if *old.ContentLength != 0 { + return fmt.Errorf("expected zero content-length, instead got %v", + *old.ContentLength) + } + if getString(old.ContentType) != directoryContentType { + return fmt.Errorf("expected the content-type to be %v, instead got %v", + directoryContentType, getString(old.ContentType)) + } + if !areMapsSame(meta, old.Metadata) { + return fmt.Errorf("expected the metadata to be %v, instead got %v", + meta, old.Metadata) + } + if getString(old.WebsiteRedirectLocation) != redirect { + return fmt.Errorf("expected the website-redirect-location to be %v, instead got %v", + redirect, getString(old.WebsiteRedirectLocation)) + } + if old.TagCount == nil { + return fmt.Errorf("expected non nil TagCount") + } + if *old.TagCount != int32(len(tags)) { + return fmt.Errorf("expected the tag count to be %v, instead got %v", + len(tags), *old.TagCount) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + oldTags, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: first.VersionId, + }) + cancel() + if err != nil { + return err + } + + if !areTagsSame(tags, oldTags.TagSet) { + return fmt.Errorf("expected the tag set to be %v, instead got %v", + tags, oldTags.TagSet) } return nil @@ -272,101 +468,117 @@ func Versioning_CopyObject_encoded_versionid_separator_invalid_versionId(s *S3Co func Versioning_CopyObject_success(s *S3Conf) error { testName := "Versioning_CopyObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstObj := "dst-obj" - srcBucket, srcObj := getBucketName(), "src-obj" - + srcBucket := getBucketName() if err := setup(s, srcBucket); err != nil { return err } - dstObjVersions, err := createObjVersions(s3client, bucket, dstObj, 1) - if err != nil { - return err + srcObjLens := map[string]int64{ + "src-obj": 2345, + "src-dir/": 0, + } + for srcObj, srcObjLen := range srcObjLens { + _, err := putObjectWithData(srcObjLen, &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } } - srcObjLen := int64(2345) - _, err = putObjectWithData(srcObjLen, &s3.PutObjectInput{ - Bucket: &srcBucket, - Key: &srcObj, - }, s3client) - if err != nil { - return err + // destination object -> source object + srcObjs := map[string]string{ + "dst-obj": "src-obj", + "dst-dir/": "src-dir/", + "dst-file": "src-dir/", } + err := forEachKey([]string{"dst-obj", "dst-dir/", "dst-file"}, func(dstObj string) error { + srcObj := srcObjs[dstObj] + dstObjVersions, err := createObjVersions(s3client, bucket, dstObj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v", srcBucket, srcObj)), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", srcBucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId in the result") + } + + srcObjLen := srcObjLens[srcObj] + dstObjVersions[0].IsLatest = getBoolPtr(false) + versions := append([]types.ObjectVersion{ + { + ETag: out.CopyObjectResult.ETag, + IsLatest: getBoolPtr(true), + Key: &dstObj, + Size: &srcObjLen, + VersionId: out.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.CopyObjectResult.ChecksumType, + }, + }, dstObjVersions...) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &dstObj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() if err != nil { return err } - if err := teardown(s, srcBucket); err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId in the result") - } - - dstObjVersions[0].IsLatest = getBoolPtr(false) - versions := append([]types.ObjectVersion{ - { - ETag: out.CopyObjectResult.ETag, - IsLatest: getBoolPtr(true), - Key: &dstObj, - Size: &srcObjLen, - VersionId: out.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: out.CopyObjectResult.ChecksumType, - }, - }, dstObjVersions...) - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil + return teardown(s, srcBucket) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { testName := "Versioning_CopyObject_non_existing_version_id" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstBucket, dstObj := getBucketName(), "my-obj" - srcObj := "my-obj" - + dstBucket := getBucketName() if err := setup(s, dstBucket); err != nil { return err } - _, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &dstBucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVRZ", - bucket, srcObj)), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVRZ", + bucket, obj)), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + if err != nil { return err } @@ -381,135 +593,158 @@ func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { testName := "Versioning_CopyObject_from_an_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcBucket, srcObj, dstObj := getBucketName(), "my-obj", "my-dst-obj" + srcBucket := getBucketName() if err := setup(s, srcBucket, withVersioning(types.BucketVersioningStatusEnabled)); err != nil { return err } - srcObjVersions, err := createObjVersions(s3client, srcBucket, srcObj, 1) - if err != nil { - return err + // copy the noncurrent and then the current source object version + dstObjs := map[string]string{ + "my-obj": "my-dst-obj", + "my-dir/": "my-dst-dir/", } - srcObjVersion := srcObjVersions[0] + err := forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstObj := dstObjs[srcObj] + srcObjVersions, err := createObjVersions(s3client, srcBucket, srcObj, 2) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", srcBucket, srcObj, *srcObjVersion.VersionId)), + for _, srcObjVersion := range reverseSlice(srcObjVersions) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", srcBucket, srcObj, *srcObjVersion.VersionId)), + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if out.CopySourceVersionId == nil { + return fmt.Errorf("expected non nil CopySourceVersionId") + } + if *out.CopySourceVersionId != *srcObjVersion.VersionId { + return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", + *srcObjVersion.VersionId, *out.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *res.ContentLength != *srcObjVersion.Size { + return fmt.Errorf("expected the copied object size to be %v, instead got %v", + *srcObjVersion.Size, *res.ContentLength) + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", + *out.VersionId, *res.VersionId) + } + } + + return nil }) - cancel() if err != nil { return err } - if err := teardown(s, srcBucket); err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - if out.CopySourceVersionId == nil { - return fmt.Errorf("expected non nil CopySourceVersionId") - } - if *out.CopySourceVersionId != *srcObjVersion.VersionId { - return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", - *srcObjVersion.VersionId, *out.CopySourceVersionId) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &dstObj, - VersionId: out.VersionId, - }) - cancel() - if err != nil { - return err - } - - if res.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *res.ContentLength != *srcObjVersion.Size { - return fmt.Errorf("expected the copied object size to be %v, instead got %v", - *srcObjVersion.Size, *res.ContentLength) - } - if *res.VersionId != *out.VersionId { - return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", - *out.VersionId, *res.VersionId) - } - - return nil + return teardown(s, srcBucket) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_special_chars(s *S3Conf) error { testName := "Versioning_CopyObject_special_chars" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcObj, dstBucket, dstObj := "foo?bar", getBucketName(), "bar&foo" + dstBucket := getBucketName() err := setup(s, dstBucket) if err != nil { return err } - srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err + // source object -> destination object + dstObjs := map[string]string{ + "foo?bar": "bar&foo", + "baz?dir/": "dir&baz/", } + err = forEachKey([]string{"foo?bar", "baz?dir/"}, func(srcObj string) error { + dstObj := dstObjs[srcObj] + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } - srcObjVersionId := *srcObjVersions[0].VersionId - copySource := fmt.Sprintf("%v/%v?versionId=%v", - bucket, - url.PathEscape(srcObj), - url.QueryEscape(srcObjVersionId), - ) + srcObjVersionId := *srcObjVersions[0].VersionId + copySource := fmt.Sprintf("%v/%v?versionId=%v", + bucket, + url.PathEscape(srcObj), + url.QueryEscape(srcObjVersionId), + ) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(copySource), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(copySource), + }) + cancel() + if err != nil { + return err + } + + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if res.CopySourceVersionId == nil { + return fmt.Errorf("expected non nil CopySourceVersionId") + } + if *res.CopySourceVersionId != srcObjVersionId { + return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", + srcObjVersionId, *res.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + VersionId: res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.VersionId != *res.VersionId { + return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", + *res.VersionId, *out.VersionId) + } + + return nil }) - cancel() if err != nil { return err } - if res.VersionId == nil || *res.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - if res.CopySourceVersionId == nil { - return fmt.Errorf("expected non nil CopySourceVersionId") - } - if *res.CopySourceVersionId != srcObjVersionId { - return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", - srcObjVersionId, *res.CopySourceVersionId) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &dstObj, - VersionId: res.VersionId, - }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.VersionId != *res.VersionId { - return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", - *res.VersionId, *out.VersionId) - } - err = teardown(s, dstBucket) if err != nil { return err @@ -548,27 +783,28 @@ func Versioning_HeadObject_invalid_versionId(s *S3Conf) error { func Versioning_HeadObject_non_existing_object_version(s *S3Conf) error { testName := "Versioning_HeadObject_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - return nil }) } @@ -585,58 +821,169 @@ func Versioning_HeadObject_invalid_parent(s *S3Conf) error { return err } - obj = "not-a-dir/bad-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + return forEachKey([]string{"not-a-dir/bad-obj", "not-a-dir/bad-dir/"}, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - return nil }) } func Versioning_HeadObject_success(s *S3Conf) error { testName := "Versioning_HeadObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - r, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", + dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *r.res.VersionId, *out.VersionId) + } + if strings.HasSuffix(obj, "/") && getString(out.ContentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(out.ContentType)) + } + + return nil }) - cancel() - if err != nil { - return err + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_HeadObject_dir_object_versions(s *S3Conf) error { + testName := "Versioning_HeadObject_dir_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + + type dirVersion struct { + versionId *string + metadata map[string]string } - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") + // each version gets its own metadata key + versions := []dirVersion{} + for i := range 3 { + metadata := map[string]string{ + fmt.Sprintf("key%v", i): fmt.Sprintf("value%v", i), + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: metadata, + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + versions = append(versions, dirVersion{ + versionId: res.VersionId, + metadata: metadata, + }) } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") + + checkVersion := func(v dirVersion, contentLength *int64, contentType, versionId *string, metadata map[string]string) error { + if contentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if *contentLength != 0 { + return fmt.Errorf("expected the object content-length to be 0, instead got %v", + *contentLength) + } + if getString(contentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(contentType)) + } + if getString(versionId) != *v.versionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *v.versionId, getString(versionId)) + } + if !areMapsSame(metadata, v.metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + v.metadata, metadata) + } + return nil } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) + + for _, v := range versions { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + head, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: v.versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("head version %v: %w", *v.versionId, err) + } + err = checkVersion(v, head.ContentLength, head.ContentType, head.VersionId, head.Metadata) + if err != nil { + return fmt.Errorf("head version %v: %w", *v.versionId, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + get, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: v.versionId, + }) + if err != nil { + cancel() + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } + bdy, err := io.ReadAll(get.Body) + get.Body.Close() + cancel() + if err != nil { + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } + if len(bdy) != 0 { + return fmt.Errorf("get version %v: expected empty body, instead got %v bytes", + *v.versionId, len(bdy)) + } + err = checkVersion(v, get.ContentLength, get.ContentType, get.VersionId, get.Metadata) + if err != nil { + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } } return nil @@ -646,82 +993,84 @@ func Versioning_HeadObject_success(s *S3Conf) error { func Versioning_HeadObject_without_versionId(s *S3Conf) error { testName := "Versioning_HeadObject_without_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - lastVersion := versions[0] + lastVersion := versions[0] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != *lastVersion.VersionId { + return fmt.Errorf("expected versionId to be %v, instead got %v", + *lastVersion.VersionId, getString(res.VersionId)) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if getString(res.VersionId) != *lastVersion.VersionId { - return fmt.Errorf("expected versionId to be %v, instead got %v", - *lastVersion.VersionId, getString(res.VersionId)) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_HeadObject_delete_marker(s *S3Conf) error { testName := "Versioning_HeadObject_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, "MethodNotAllowed"); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkSdkApiErr(err, "MethodNotAllowed"); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -752,131 +1101,124 @@ func Versioning_GetObject_invalid_versionId(s *S3Conf) error { func Versioning_GetObject_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObject_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_success(s *S3Conf) error { testName := "Versioning_GetObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - r, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - // Get the object by versionId - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + // getObject checks that the object read with versionId is + // the uploaded version + getObject := func(versionId *string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + defer cancel() + if err != nil { + return err + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", + dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *r.res.VersionId, *out.VersionId) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + out.Body.Close() + + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("incorrect output content") + } + + return nil + } + + // Get the object by versionId + if err := getObject(r.res.VersionId); err != nil { + return err + } + + // Get the object without versionId + if err := getObject(nil); err != nil { + return err + } + + // Get the noncurrent object version by versionId + _, err = putObjectWithData(objDataLen(obj, 1000), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + return getObject(r.res.VersionId) }) - defer cancel() - if err != nil { - return err - } - - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) - } - - bdy, err := io.ReadAll(out.Body) - if err != nil { - return err - } - out.Body.Close() - - outCsum := sha256.Sum256(bdy) - if outCsum != r.csum { - return fmt.Errorf("incorrect output content") - } - - // Get the object without versionId - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - defer cancel() - if err != nil { - return err - } - - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) - } - - bdy, err = io.ReadAll(out.Body) - if err != nil { - return err - } - out.Body.Close() - - outCsum = sha256.Sum256(bdy) - if outCsum != r.csum { - return fmt.Errorf("incorrect output content") - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_delete_marker_without_versionId(s *S3Conf) error { testName := "Versioning_GetObject_delete_marker_without_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := putObjectWithData(1234, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + // the objects are put before versioning is enabled + err := forEachKey(keys, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 1234), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + return err + }) if err != nil { return err } @@ -886,80 +1228,92 @@ func Versioning_GetObject_delete_marker_without_versionId(s *S3Conf) error { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } - return nil + return nil + }) }) } func Versioning_GetObject_delete_marker(s *S3Conf) error { testName := "Versioning_GetObject_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_null_versionId_obj(s *S3Conf) error { testName := "Versioning_GetObject_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, lgth := "my-obj", int64(234) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys, dataLen := []string{"my-obj", "my-dir/"}, int64(234) + // the objects are put before versioning is enabled + etags := make(map[string]string, len(keys)) + err := forEachKey(keys, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, dataLen), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + etags[obj] = getString(out.res.ETag) + return nil + }) if err != nil { return err } @@ -969,40 +1323,43 @@ func Versioning_GetObject_null_versionId_obj(s *S3Conf) error { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: &nullVersionId, + return forEachKey(keys, func(obj string) error { + lgth, etag := objDataLen(obj, dataLen), etags[obj] + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &nullVersionId, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if res.ETag == nil { + return fmt.Errorf("expected non nil ETag") + } + if *res.ContentLength != lgth { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + lgth, *res.ContentLength) + } + if *res.VersionId != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, insted got %v", + nullVersionId, *res.VersionId) + } + if *res.ETag != etag { + return fmt.Errorf("expecte the ETag to be %v, instead got %v", + etag, *res.ETag) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if res.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if res.ETag == nil { - return fmt.Errorf("expected non nil ETag") - } - if *res.ContentLength != lgth { - return fmt.Errorf("expected the Content-Length to be %v, instead got %v", - lgth, *res.ContentLength) - } - if *res.VersionId != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, insted got %v", - nullVersionId, *res.VersionId) - } - if *res.ETag != *out.res.ETag { - return fmt.Errorf("expecte the ETag to be %v, instead got %v", - *out.res.ETag, *res.ETag) - } - - return nil }) } @@ -1032,95 +1389,97 @@ func Versioning_GetObjectAttributes_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectAttributes_object_version(s *S3Conf) error { testName := "Versioning_GetObjectAttributes_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := versions[0] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := versions[0] - getObjAttrs := func(versionId *string) (*s3.GetObjectAttributesOutput, error) { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - ObjectAttributes: []types.ObjectAttributes{ - types.ObjectAttributesEtag, - }, - }) - cancel() - return res, err - } + getObjAttrs := func(versionId *string) (*s3.GetObjectAttributesOutput, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesEtag, + }, + }) + cancel() + return res, err + } - // By specifying the versionId - res, err := getObjAttrs(version.VersionId) - if err != nil { - return err - } + // By specifying the versionId + res, err := getObjAttrs(version.VersionId) + if err != nil { + return err + } - if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { - return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", - strings.Trim(*version.ETag, "\""), getString(res.ETag)) - } - if getString(res.VersionId) != *version.VersionId { - return fmt.Errorf("expected the uploaded versionId to be %v, instead got %v", - *version.VersionId, getString(res.VersionId)) - } + if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { + return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", + strings.Trim(*version.ETag, "\""), getString(res.ETag)) + } + if getString(res.VersionId) != *version.VersionId { + return fmt.Errorf("expected the uploaded versionId to be %v, instead got %v", + *version.VersionId, getString(res.VersionId)) + } - // Without versionId - res, err = getObjAttrs(nil) - if err != nil { - return err - } + // Without versionId + res, err = getObjAttrs(nil) + if err != nil { + return err + } - if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { - return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", - strings.Trim(*version.ETag, "\""), getString(res.ETag)) - } - if getString(res.VersionId) != *version.VersionId { - return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", - *version.VersionId, getString(res.VersionId)) - } + if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { + return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", + strings.Trim(*version.ETag, "\""), getString(res.ETag)) + } + if getString(res.VersionId) != *version.VersionId { + return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", + *version.VersionId, getString(res.VersionId)) + } - return nil + return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObjectAttributes_delete_marker(s *S3Conf) error { testName := "Versioning_GetObjectAttributes_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: res.VersionId, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesEtag, + }, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: res.VersionId, - ObjectAttributes: []types.ObjectAttributes{ - types.ObjectAttributesEtag, - }, - }) - cancel() - if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1150,43 +1509,204 @@ func Versioning_DeleteObject_invalid_versionId(s *S3Conf) error { func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { testName := "Versioning_DeleteObject_delete_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - oLen := int64(1000) - obj := "my-obj" - r, err := putObjectWithData(oLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + oLen := objDataLen(obj, 1000) + r, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + versionId := r.res.VersionId + if versionId == nil || *versionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + _, err = putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil { + return fmt.Errorf("expected non nil versionId") + } + if *out.VersionId != *versionId { + return fmt.Errorf("expected deleted object versionId to be %v, instead got %v", + *versionId, *out.VersionId) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_dir_object_latest_version(s *S3Conf) error { + testName := "Versioning_DeleteObject_dir_object_latest_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + versionIds := []string{} + for i := range 3 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: map[string]string{ + "version": fmt.Sprint(i), + }, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + versionIds = append(versionIds, *out.VersionId) } - versionId := r.res.VersionId - if versionId == nil || *versionId == "" { - return fmt.Errorf("expected non empty versionId") - } + // deleting the latest version makes the previous one the current + for i := 2; i > 0; i-- { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &versionIds[i], + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) != versionIds[i] { + return fmt.Errorf("expected the deleted versionId to be %v, instead got %v", + versionIds[i], getString(out.VersionId)) + } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be false") + } - _, err = putObjects(s3client, []string{obj}, bucket) - if err != nil { - return err + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != versionIds[i-1] { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + versionIds[i-1], getString(res.VersionId)) + } + expectedMeta := map[string]string{ + "version": fmt.Sprint(i - 1), + } + if !areMapsSame(res.Metadata, expectedMeta) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + expectedMeta, res.Metadata) + } + if getString(res.ContentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(res.ContentType)) + } + if res.ContentLength == nil || *res.ContentLength != 0 { + return fmt.Errorf("expected zero content length, instead got %v", + res.ContentLength) + } } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, Key: &obj, - VersionId: versionId, + VersionId: &versionIds[0], }) cancel() if err != nil { return err } - if out.VersionId == nil { - return fmt.Errorf("expected non nil versionId") + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err == nil { + return fmt.Errorf("expected NotFound, instead got nil") } - if *out.VersionId != *versionId { - return fmt.Errorf("expected deleted object versionId to be %v, instead got %v", - *versionId, *out.VersionId) + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Versions) != 0 { + return fmt.Errorf("expected empty object versions, instead got %v", res.Versions) + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", res.DeleteMarkers) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Versions) != 1 { + return fmt.Errorf("expected 1 object version, instead got %v", len(res.Versions)) + } + if getString(res.Versions[0].Key) != obj { + return fmt.Errorf("expected the version key to be %v, instead got %v", + obj, getString(res.Versions[0].Key)) + } + if getString(res.Versions[0].VersionId) != *out.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *out.VersionId, getString(res.Versions[0].VersionId)) + } + if res.Versions[0].IsLatest == nil || !*res.Versions[0].IsLatest { + return fmt.Errorf("expected the version to be the latest") + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", res.DeleteMarkers) } return nil @@ -1196,29 +1716,79 @@ func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { testName := "Versioning_DeleteObject_non_existing_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + ctx, canel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + canel() + if err != nil { + return err + } - ctx, canel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, canel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("non_existing_version_id"), + }) + canel() + if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, "non_existing_version_id")); err != nil { + return err + } + + return nil }) - canel() + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_implicit_dir(s *S3Conf) error { + testName := "Versioning_DeleteObject_implicit_dir" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "my-dir/", "my-dir/obj" + versions, err := createObjVersions(s3client, bucket, obj, 1) if err != nil { return err } - ctx, canel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("non_existing_version_id"), + // "my-dir/" is only the parent directory of "my-dir/obj", not an object + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, }) - canel() - if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, "non_existing_version_id")); err != nil { + cancel() + if err != nil { return err } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be false") + } + if getString(out.VersionId) != "" { + return fmt.Errorf("expected empty versionId, instead got %v", + getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", + res.DeleteMarkers) + } + return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1226,20 +1796,113 @@ func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { testName := "Versioning_DeleteObject_delete_a_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - oLen := int64(1000) - obj := "my-obj" - _, err := putObjectWithData(oLen, &s3.PutObjectInput{ + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + oLen := objDataLen(obj, 1000) + _, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.DeleteMarker == nil || !*out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be true") + } + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.DeleteMarker == nil || !*res.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be true") + } + if res.VersionId == nil { + return fmt.Errorf("expected non empty versionId") + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *out.VersionId, *res.VersionId) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_dir_object_with_children(s *S3Conf) error { + testName := "Versioning_DeleteObject_dir_object_with_children" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, child := "my-dir/", "my-dir/child" + dirVersions, err := createObjVersions(s3client, bucket, dir, 1) + if err != nil { + return err + } + childData, err := putObjectWithData(100, &s3.PutObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &child, }, s3client) if err != nil { return err } + // checkObjects checks that the child object is readable and that + // the bucket lists exactly keys + checkObjects := func(keys ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &child, + }) + cancel() + if err != nil { + return err + } + body, err := io.ReadAll(out.Body) + out.Body.Close() + if err != nil { + return err + } + if sha256.Sum256(body) != childData.csum { + return fmt.Errorf("expected the %v data checksum to match", child) + } + + objs, _, err := listObjects(s3client, bucket, "", "", 1000) + if err != nil { + return err + } + if !hasObjNames(objs, keys) { + return fmt.Errorf("expected the listed objects to be %v, instead got %v", + keys, objs) + } + + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &dir, }) cancel() if err != nil { @@ -1249,14 +1912,19 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { if out.DeleteMarker == nil || !*out.DeleteMarker { return fmt.Errorf("expected the response DeleteMarker to be true") } - if out.VersionId == nil || *out.VersionId == "" { + if getString(out.VersionId) == "" { return fmt.Errorf("expected non empty versionId") } + err = checkObjects(child) + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &dir, VersionId: out.VersionId, }) cancel() @@ -1267,12 +1935,81 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { if res.DeleteMarker == nil || !*res.DeleteMarker { return fmt.Errorf("expected the response DeleteMarker to be true") } - if res.VersionId == nil { - return fmt.Errorf("expected non empty versionId") - } - if *res.VersionId != *out.VersionId { + if getString(res.VersionId) != *out.VersionId { return fmt.Errorf("expected the versionId to be %v, instead got %v", - *out.VersionId, *res.VersionId) + *out.VersionId, getString(res.VersionId)) + } + + // removing the delete marker restores the directory object + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + head, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + if getString(head.VersionId) != *dirVersions[0].VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *dirVersions[0].VersionId, getString(head.VersionId)) + } + + err = checkObjects(dir, child) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, + VersionId: dirVersions[0].VersionId, + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) != *dirVersions[0].VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *dirVersions[0].VersionId, getString(res.VersionId)) + } + + // with no versions left the directory is only the parent of the child + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err == nil { + return fmt.Errorf("expected NotFound, instead got nil") + } + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + err = checkObjects(child) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + versions, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(versions.Versions) != 1 || getString(versions.Versions[0].Key) != child { + return fmt.Errorf("expected only %v versions, instead got %v", + child, versions.Versions) + } + if len(versions.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", + versions.DeleteMarkers) } return nil @@ -1282,87 +2019,93 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { func Versioning_Delete_null_versionId_object(s *S3Conf) error { testName := "Versioning_Delete_null_versionId_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, nObjLgth := "my-obj", int64(3211) - _, err := putObjectWithData(nObjLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + objs, nObjLgth := []string{"my-obj", "my-dir/"}, int64(3211) + // the null versions are created before versioning is enabled + for _, obj := range objs { + _, err := putObjectWithData(objDataLen(obj, nObjLgth), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + } + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) if err != nil { return err } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } + return forEachKey(objs, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - _, err = createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr(nullVersionId), + return nil }) - cancel() - if err != nil { - return err - } - if getString(res.VersionId) != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - nullVersionId, getString(res.VersionId)) - } - - return nil }) } func Versioning_DeleteObject_nested_dir_object(s *S3Conf) error { testName := "Versioning_DeleteObject_nested_dir_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "foo/bar/baz" - out, err := putObjectWithData(1000, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"foo/bar/baz", "foo/bar/baz/"}, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, 1000), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.res.VersionId, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != getString(out.res.VersionId) { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + getString(out.res.VersionId), getString(res.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + // Then create the bucket back to not get error on teardown + if err := setup(s, bucket, withLock()); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if getString(res.VersionId) != getString(out.res.VersionId) { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - getString(out.res.VersionId), getString(res.VersionId)) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - // Then create the bucket back to not get error on teardown - if err := setup(s, bucket); err != nil { - return err - } - - return nil }, withLock()) } @@ -1378,6 +2121,15 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { } versionId := getString(out.res.VersionId) + out, err = putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("my-dir/"), + }, s3client) + if err != nil { + return err + } + dirVersionId := getString(out.res.VersionId) + for _, test := range []struct { key string versionId string @@ -1386,8 +2138,14 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { {"foo/bar/baz", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"hello", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"hello/world", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"foo/bar/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"hello/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"hello/world/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"foo/bar/baz/quxx", versionId}, + {"foo/bar/baz/quxx/", versionId}, + {"my-dir/hello/", dirVersionId}, {"foo", versionId}, + {"my-dir/", dirVersionId}, } { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ @@ -1428,72 +2186,80 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { func Versioning_DeleteObject_suspended(s *S3Conf) error { testName := "Versioning_DeleteObject_suspended" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - versions[0].IsLatest = getBoolPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // the object version is created while versioning is enabled + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) - if err != nil { - return err - } + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + versions[0].IsLatest = getBoolPtr(false) + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + for range 5 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.DeleteMarker == nil { + return fmt.Errorf("expected the delete marker to be true") + } + if !*res.DeleteMarker { + return fmt.Errorf("expected the delete marker to be true, instead got %v", + *res.DeleteMarker) + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil versionId") + } + if *res.VersionId != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, *res.VersionId) + } + } - for range 5 { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, - Key: &obj, + Prefix: &obj, }) cancel() if err != nil { return err } - if res.DeleteMarker == nil { - return fmt.Errorf("expected the delete marker to be true") + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getBoolPtr(true), + Key: &obj, + VersionId: &nullVersionId, + }, } - if !*res.DeleteMarker { - return fmt.Errorf("expected the delete marker to be true, instead got %v", - *res.DeleteMarker) - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil versionId") - } - if *res.VersionId != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - nullVersionId, *res.VersionId) - } - } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(res.DeleteMarkers, delMarkers) { + return fmt.Errorf("expected the delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + return nil }) - cancel() - if err != nil { - return err - } - - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getBoolPtr(true), - Key: &obj, - VersionId: &nullVersionId, - }, - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the versions to be %v, instead got %v", - versions, res.Versions) - } - if !compareDelMarkers(res.DeleteMarkers, delMarkers) { - return fmt.Errorf("expected the delete markers to be %v, instead got %v", - delMarkers, res.DeleteMarkers) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1501,6 +2267,7 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { testName := "Versioning_DeleteObjects_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj1, obj2, obj3 := "foo", "bar", "baz" + dir1, dir2 := "foo-dir/", "baz-dir/" obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) if err != nil { @@ -1514,6 +2281,14 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { if err != nil { return err } + dir1Version, err := createObjVersions(s3client, bucket, dir1, 1) + if err != nil { + return err + } + dir2Version, err := createObjVersions(s3client, bucket, dir2, 1) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ @@ -1530,6 +2305,13 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { { Key: obj3Version[0].Key, }, + { + Key: dir1Version[0].Key, + VersionId: dir1Version[0].VersionId, + }, + { + Key: dir2Version[0].Key, + }, }, }, }) @@ -1552,6 +2334,15 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { Key: obj3Version[0].Key, DeleteMarker: getBoolPtr(true), }, + { + Key: dir1Version[0].Key, + VersionId: dir1Version[0].VersionId, + DeleteMarker: getBoolPtr(false), + }, + { + Key: dir2Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, } if len(out.Errors) != 0 { @@ -1574,7 +2365,9 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { obj2Version[0].IsLatest = getBoolPtr(false) obj3Version[0].IsLatest = getBoolPtr(false) + dir2Version[0].IsLatest = getBoolPtr(false) versions := append(obj2Version, obj3Version...) + versions = append(versions, dir2Version...) delMarkers := []types.DeleteMarkerEntry{ { @@ -1587,6 +2380,11 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { Key: out.Deleted[2].Key, VersionId: out.Deleted[2].DeleteMarkerVersionId, }, + { + IsLatest: getBoolPtr(true), + Key: out.Deleted[4].Key, + VersionId: out.Deleted[4].DeleteMarkerVersionId, + }, } if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", @@ -1605,6 +2403,7 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { testName := "Versioning_DeleteObjects_delete_deleteMarkers" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj1, obj2 := "foo", "bar" + dir1, dir2 := "foo-dir/", "bar-dir/" obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) if err != nil { @@ -1614,6 +2413,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { if err != nil { return err } + dir1Version, err := createObjVersions(s3client, bucket, dir1, 1) + if err != nil { + return err + } + dir2Version, err := createObjVersions(s3client, bucket, dir2, 1) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ @@ -1626,6 +2433,12 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { { Key: obj2Version[0].Key, }, + { + Key: dir1Version[0].Key, + }, + { + Key: dir2Version[0].Key, + }, }, }, }) @@ -1643,6 +2456,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { Key: obj2Version[0].Key, DeleteMarker: getBoolPtr(true), }, + { + Key: dir1Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, + { + Key: dir2Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, } if len(out.Errors) != 0 { @@ -1667,6 +2488,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { Key: out.Deleted[1].Key, VersionId: out.Deleted[1].VersionId, }, + { + Key: out.Deleted[2].Key, + VersionId: out.Deleted[2].VersionId, + }, + { + Key: out.Deleted[3].Key, + VersionId: out.Deleted[3].VersionId, + }, }, }, }) @@ -1692,6 +2521,18 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { DeleteMarkerVersionId: out.Deleted[1].VersionId, VersionId: out.Deleted[1].VersionId, }, + { + Key: out.Deleted[2].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[2].VersionId, + VersionId: out.Deleted[2].VersionId, + }, + { + Key: out.Deleted[3].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[3].VersionId, + VersionId: out.Deleted[3].VersionId, + }, } if !compareDelObjects(delResult, res.Deleted) { @@ -1961,127 +2802,133 @@ func Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId(s * func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error { testName := "Versioning_UploadPartCopy_non_existing_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstBucket, dstObj, srcObj := getBucketName(), "dst-obj", "src-obj" + return forEachKey([]string{"src-obj", "src-dir/"}, func(srcObj string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" - lgth := int64(100) - _, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &srcObj, - }, s3client) - if err != nil { - return err - } + lgth := objDataLen(srcObj, 100) + _, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } - if err := setup(s, dstBucket); err != nil { - return err - } + if err := setup(s, dstBucket); err != nil { + return err + } - mp, err := createMp(s3client, dstBucket, dstObj) - if err != nil { - return err - } + mp, err := createMp(s3client, dstBucket, dstObj) + if err != nil { + return err + } - pNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &dstBucket, - Key: &dstObj, - UploadId: mp.UploadId, - PartNumber: &pNumber, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVS0", - bucket, srcObj)), + pNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &pNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVS0", + bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - if err := teardown(s, dstBucket); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { testName := "Versioning_UploadPartCopy_from_an_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcObj, dstBucket, obj := "my-obj", getBucketName(), "dst-obj" - err := setup(s, dstBucket) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstBucket, obj := getBucketName(), "dst-obj" + err := setup(s, dstBucket) + if err != nil { + return err + } - srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } - srcObjVersion := srcObjVersions[0] + // the latest version and a noncurrent one + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 2) + if err != nil { + return err + } - out, err := createMp(s3client, dstBucket, obj) - if err != nil { - return err - } + for _, srcObjVersion := range srcObjVersions { + out, err := createMp(s3client, dstBucket, obj) + if err != nil { + return err + } - partNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &dstBucket, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, *srcObjVersion.VersionId)), - UploadId: out.UploadId, - Key: &obj, - PartNumber: &partNumber, + partNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, *srcObjVersion.VersionId)), + UploadId: out.UploadId, + Key: &obj, + PartNumber: &partNumber, + }) + cancel() + if err != nil { + return err + } + + if getString(copyOut.CopySourceVersionId) != getString(srcObjVersion.VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersion.VersionId), getString(copyOut.CopySourceVersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &dstBucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", + len(res.Parts)) + } + if res.Parts[0].PartNumber == nil { + return fmt.Errorf("expected part-number to be non nil") + } + if *res.Parts[0].PartNumber != partNumber { + return fmt.Errorf("expected part-number to be %v, instead got %v", + partNumber, res.Parts[0].PartNumber) + } + if res.Parts[0].Size == nil { + return fmt.Errorf("expected part size to be non nil") + } + if *res.Parts[0].Size != *srcObjVersion.Size { + return fmt.Errorf("expected part size to be %v, instead got %v", + *srcObjVersion.Size, res.Parts[0].Size) + } + if getString(res.Parts[0].ETag) != getString(copyOut.CopyPartResult.ETag) { + return fmt.Errorf("expected part etag to be %v, instead got %v", + getString(copyOut.CopyPartResult.ETag), getString(res.Parts[0].ETag)) + } + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if getString(copyOut.CopySourceVersionId) != getString(srcObjVersion.VersionId) { - return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", - getString(srcObjVersion.VersionId), getString(copyOut.CopySourceVersionId)) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &dstBucket, - Key: &obj, - UploadId: out.UploadId, - }) - cancel() - if err != nil { - return err - } - - if len(res.Parts) != 1 { - return fmt.Errorf("expected parts to be 1, instead got %v", - len(res.Parts)) - } - if res.Parts[0].PartNumber == nil { - return fmt.Errorf("expected part-number to be non nil") - } - if *res.Parts[0].PartNumber != partNumber { - return fmt.Errorf("expected part-number to be %v, instead got %v", - partNumber, res.Parts[0].PartNumber) - } - if res.Parts[0].Size == nil { - return fmt.Errorf("expected part size to be non nil") - } - if *res.Parts[0].Size != *srcObjVersion.Size { - return fmt.Errorf("expected part size to be %v, instead got %v", - *srcObjVersion.Size, res.Parts[0].Size) - } - if getString(res.Parts[0].ETag) != getString(copyOut.CopyPartResult.ETag) { - return fmt.Errorf("expected part etag to be %v, instead got %v", - getString(copyOut.CopyPartResult.ETag), getString(res.Parts[0].ETag)) - } - - if err := teardown(s, dstBucket); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2167,29 +3014,30 @@ func Versioning_PutObjectRetention_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectRetention_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectRetention_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2216,121 +3064,124 @@ func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectRetention_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectRetention_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Put_GetObjectRetention_delete_marker(s *S3Conf) error { testName := "Versioning_Put_GetObjectRetention_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectRetention + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectRetention + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectRetention - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectRetention - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } func Versioning_Put_GetObjectRetention_success(s *S3Conf) error { testName := "Versioning_Put_GetObjectRetention_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } - objVersion := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + objVersion := objVersions[1] - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.Retention.Mode != types.ObjectLockRetentionModeGovernance { + return fmt.Errorf("expected the object retention mode to be %v, instead got %v", + types.ObjectLockRetentionModeGovernance, res.Retention.Mode) + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: getString(objVersion.Key), versionId: getString(objVersion.VersionId)}}) }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - }) - cancel() - if err != nil { - return err - } - - if res.Retention.Mode != types.ObjectLockRetentionModeGovernance { - return fmt.Errorf("expected the object retention mode to be %v, instead got %v", - types.ObjectLockRetentionModeGovernance, res.Retention.Mode) - } - - return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: getString(objVersion.Key), versionId: getString(objVersion.VersionId)}}) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2360,27 +3211,28 @@ func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectLegalHold_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectLegalHold_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2407,123 +3259,126 @@ func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectLegalHold_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectLegalHold_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetObjectLegalHold_delete_marker(s *S3Conf) error { testName := "Versioning_PutGetObjectLegalHold_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectLegalHold + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectLegalHold + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectLegalHold - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectLegalHold - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { testName := "Versioning_Put_GetObjectLegalHold_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } - objVersion := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + objVersion := objVersions[1] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + }) + cancel() + if err != nil { + return err + } - if res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { - return fmt.Errorf("expected the object version legal hold status to be %v, instead got %v", - types.ObjectLockLegalHoldStatusOn, res.LegalHold.Status) - } + if res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the object version legal hold status to be %v, instead got %v", + types.ObjectLockLegalHoldStatusOn, res.LegalHold.Status) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: getString(objVersion.Key), - versionId: getString(objVersion.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: getString(objVersion.Key), + versionId: getString(objVersion.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2531,90 +3386,200 @@ func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_legal_hold" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[1] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } +func Versioning_WORM_dir_object_lock_headers(s *S3Conf) error { + testName := "Versioning_WORM_dir_object_lock_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + rDate := time.Now().Add(time.Hour * 48) + lockedObjs := []objToDelete{} + + for i, test := range []struct { + legalHold types.ObjectLockLegalHoldStatus + mode types.ObjectLockMode + retainUntilDate *time.Time + }{ + {legalHold: types.ObjectLockLegalHoldStatusOn}, + {mode: types.ObjectLockModeGovernance, retainUntilDate: &rDate}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: test.legalHold, + ObjectLockMode: test.mode, + ObjectLockRetainUntilDate: test.retainUntilDate, + }, s3client) + if err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: getString(res.res.VersionId), + removeOnlyLeglHold: test.legalHold == types.ObjectLockLegalHoldStatusOn, + }) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + + if out.ObjectLockLegalHoldStatus != test.legalHold { + return fmt.Errorf("test %v: expected the object legal hold status to be %q, instead got %q", + i+1, test.legalHold, out.ObjectLockLegalHoldStatus) + } + if out.ObjectLockMode != test.mode { + return fmt.Errorf("test %v: expected the object lock mode to be %q, instead got %q", + i+1, test.mode, out.ObjectLockMode) + } + if test.retainUntilDate == nil { + if out.ObjectLockRetainUntilDate != nil { + return fmt.Errorf("test %v: expected nil object lock retain until date, instead got %v", + i+1, *out.ObjectLockRetainUntilDate) + } + } else if out.ObjectLockRetainUntilDate == nil || + out.ObjectLockRetainUntilDate.Unix() != test.retainUntilDate.Unix() { + return fmt.Errorf("test %v: expected the object lock retain until date to be %v, instead got %v", + i+1, test.retainUntilDate.Format(time.RFC3339), out.ObjectLockRetainUntilDate) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: res.res.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.DeleteMarker == nil || !*out.DeleteMarker { + return fmt.Errorf("expected the delete marker to be true, instead got %v", out.DeleteMarker) + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty delete marker versionId") + } + + // the noncurrent versions keep their lock settings + for _, lockedObj := range lockedObjs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &lockedObj.versionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("version %v: %w", lockedObj.versionId, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_governance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[0] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[0] - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, - }) - cancel() - if err != nil { - return err - } + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2622,114 +3587,124 @@ func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) err func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_compliance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[0] + // COMPLIANCE retentions can only be waited out, so the locked + // versions of all the keys are cleaned up together + lockedObjs := []objToDelete{} + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[0] - rDate := time.Now().Add(2 * complianceTestRetention) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &rDate, - }, - }) - cancel() - if err != nil { - return err - } + rDate := time.Now().Add(2 * complianceTestRetention) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { + lockedObjs = append(lockedObjs, objToDelete{ key: obj, versionId: getString(version.VersionId), isCompliance: true, - }, + }) + return nil }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_WORM_delete_marker_locked_object_legal_hold(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_legal_hold" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -2737,68 +3712,70 @@ func Versioning_WORM_delete_marker_locked_object_legal_hold(s *S3Conf) error { func Versioning_WORM_delete_marker_locked_object_governance_retention(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_governance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - isCompliance: false, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + isCompliance: false, + }, + }) }) }, withLock()) } @@ -2806,140 +3783,151 @@ func Versioning_WORM_delete_marker_locked_object_governance_retention(s *S3Conf) func Versioning_WORM_delete_marker_locked_object_compliance_retention(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_compliance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + // COMPLIANCE retentions can only be waited out, so the locked + // versions of all the keys are cleaned up together + lockedObjs := []objToDelete{} + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: getPtr(time.Now().Add(complianceTestRetention)), - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: getPtr(time.Now().Add(complianceTestRetention)), + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { + lockedObjs = append(lockedObjs, objToDelete{ key: obj, versionId: getString(version.VersionId), isCompliance: true, - }, + }) + return nil }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) }, withLock()) } func Versioning_WORM_PutObject_overwrite_locked_object(s *S3Conf) error { testName := "Versioning_WORM_PutObject_overwrite_locked_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - v := versions[0] - v.IsLatest = getPtr(false) + v := versions[0] + v.IsLatest = getPtr(false) - // lock the object with legal hold - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + // lock the object with legal hold + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - dataLen := int64(10) + dataLen := objDataLen(obj, 10) - // overwrite the locked object with a new version - r, err := putObjectWithData(dataLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + // overwrite the locked object with a new version + r, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - version := types.ObjectVersion{ - ETag: r.res.ETag, - IsLatest: getPtr(true), - Key: &obj, - Size: &dataLen, - VersionId: r.res.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: r.res.ChecksumType, - } + version := types.ObjectVersion{ + ETag: r.res.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: &dataLen, + VersionId: r.res.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: r.res.ChecksumType, + } - result := []types.ObjectVersion{version, v} + result := []types.ObjectVersion{version, v} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(result, out.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) - } + if !compareVersions(result, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(v.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(v.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -2947,81 +3935,89 @@ func Versioning_WORM_PutObject_overwrite_locked_object(s *S3Conf) error { func Versioning_WORM_CopyObject_overwrite_locked_object(s *S3Conf) error { testName := "Versioning_WORM_CopyObject_overwrite_locked_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - v := versions[0] - v.IsLatest = getPtr(false) + v := versions[0] + v.IsLatest = getPtr(false) - // lock the object with legal hold - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + // lock the object with legal hold + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - // create a source object version - srcObj := "source-object" - srcVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } + // create a source object version, a directory object for a + // directory object destination + srcObj := "source-object" + if strings.HasSuffix(obj, "/") { + srcObj = "source-dir/" + } + srcVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } - srcVersion := srcVersions[0] + srcVersion := srcVersions[0] - // overwrite the locked object with a new version with CopyObject - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &obj, - CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)), - }) - cancel() - if err != nil { - return err - } + // overwrite the locked object with a new version with CopyObject + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } - version := types.ObjectVersion{ - ETag: copyResult.CopyObjectResult.ETag, - IsLatest: getPtr(true), - Key: &obj, - Size: srcVersion.Size, - VersionId: copyResult.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: copyResult.CopyObjectResult.ChecksumType, - } + version := types.ObjectVersion{ + ETag: copyResult.CopyObjectResult.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: srcVersion.Size, + VersionId: copyResult.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: copyResult.CopyObjectResult.ChecksumType, + } - result := []types.ObjectVersion{version, v, srcVersion} + // the destination and the source versions are listed separately, + // as the bucket also holds the versions of the other keys + for _, result := range [][]types.ObjectVersion{{version, v}, {srcVersion}} { + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: result[0].Key, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + if !compareVersions(result, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) + } + } - if !compareVersions(result, out.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) - } - - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(v.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(v.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -3140,328 +4136,69 @@ func Versioning_WORM_remove_delete_marker_under_bucket_default_retention(s *S3Co return err } - obj := "my-object" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - // Create a delete marker - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - // Delete the delete marker - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, resp.Versions) { - return fmt.Errorf("expected the object vresions to be %v, instead got %v", versions, resp.Versions) - } - if len(resp.DeleteMarkers) != 0 { - return fmt.Errorf("expected empty delete markers list, instead got %v", resp.DeleteMarkers) - } - - // - lockedVersions := make([]objToDelete, 0, len(versions)) - for _, v := range versions { - lockedVersions = append(lockedVersions, objToDelete{ - key: obj, - versionId: getString(v.VersionId), - isCompliance: false, + // Create a delete marker + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, }) - } - return cleanupLockedObjects(s3client, bucket, lockedVersions) + cancel() + if err != nil { + return err + } + + // Delete the delete marker + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, resp.Versions) { + return fmt.Errorf("expected the object vresions to be %v, instead got %v", versions, resp.Versions) + } + if len(resp.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", resp.DeleteMarkers) + } + + // + lockedVersions := make([]objToDelete, 0, len(versions)) + for _, v := range versions { + lockedVersions = append(lockedVersions, objToDelete{ + key: obj, + versionId: getString(v.VersionId), + isCompliance: false, + }) + } + return cleanupLockedObjects(s3client, bucket, lockedVersions) + }) }, withLock()) } func Versioning_AccessControl_GetObjectVersion(s *S3Conf) error { testName := "Versioning_AccessControl_GetObjectVersion" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objData, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - // querying with versionId should return access denied - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - defer cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - - // grant the user s3:GetObjectVersion - doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - defer cancel() - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_HeadObjectVersion(s *S3Conf) error { - testName := "Versioning_AccessControl_HeadObjectVersion" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objData, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - // querying with versionId should return access denied - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - cancel() - if err := checkSdkApiErr(err, http.StatusText(http.StatusForbidden)); err != nil { - return err - } - - // grant the user s3:GetObjectVersion - doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - cancel() - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_object_tagging_policy(s *S3Conf) error { - testName := "Versioning_AccessControl_PutObjectTagging_policy" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - object := "my-object" - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &object, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - putGetDeleteObjectTagging := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - Tagging: &types.Tagging{ - TagSet: []types.Tag{ - {Key: getPtr("key"), Value: getPtr("value")}, - }, - }, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - return nil - } - - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectVersionTagging", "s3:GetObjectVersionTagging", "s3:DeleteObjectVersionTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny without versionId - err = putGetDeleteObjectTagging(nil, true) - if err != nil { - return err - } - - // allow with versionId - err = putGetDeleteObjectTagging(res.res.VersionId, false) - if err != nil { - return err - } - - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectTagging", "s3:GetObjectTagging", "s3:DeleteObjectTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // allow without versionId - err = putGetDeleteObjectTagging(nil, false) - if err != nil { - return err - } - - // deny with versionId - err = putGetDeleteObjectTagging(res.res.VersionId, true) - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { - testName := "Versioning_AccessControl_DeleteObject_policy" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" testuser := getUser("user") err := createUsers(s, []user{testuser}) if err != nil { @@ -3470,150 +4207,416 @@ func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { userClient := s.getUserClient(testuser) - delObject := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if denyAccess { - return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objData, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err } - return err - } + doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + // querying with versionId should return access denied + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + defer cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + + // grant the user s3:GetObjectVersion + doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + defer cancel() + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_HeadObjectVersion(s *S3Conf) error { + testName := "Versioning_AccessControl_HeadObjectVersion" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) + userClient := s.getUserClient(testuser) + + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objData, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + // querying with versionId should return access denied + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, http.StatusText(http.StatusForbidden)); err != nil { + return err + } + + // grant the user s3:GetObjectVersion + doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_object_tagging_policy(s *S3Conf) error { + testName := "Versioning_AccessControl_PutObjectTagging_policy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - // deny with versionId - err = delObject(res.res.VersionId, true) + userClient := s.getUserClient(testuser) + + return forEachKey([]string{"my-object", "my-dir/"}, func(object string) error { + res, err := putObjectWithData(objDataLen(object, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &object, + }, s3client) + if err != nil { + return err + } + + putGetDeleteObjectTagging := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + {Key: getPtr("key"), Value: getPtr("value")}, + }, + }, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + return nil + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectVersionTagging", "s3:GetObjectVersionTagging", "s3:DeleteObjectVersionTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny without versionId + err = putGetDeleteObjectTagging(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = putGetDeleteObjectTagging(res.res.VersionId, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectTagging", "s3:GetObjectTagging", "s3:DeleteObjectTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // allow without versionId + err = putGetDeleteObjectTagging(nil, false) + if err != nil { + return err + } + + // deny with versionId + err = putGetDeleteObjectTagging(res.res.VersionId, true) + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { + testName := "Versioning_AccessControl_DeleteObject_policy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - // allow without versionId - err = delObject(nil, false) - if err != nil { - return err - } + userClient := s.getUserClient(testuser) - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + delObject := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if denyAccess { + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + } - // recreate the object - res, err = putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return err + } - // deny without versionId - err = delObject(nil, true) - if err != nil { - return err - } + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - // allow with versionId - err = delObject(res.res.VersionId, false) - if err != nil { - return err - } + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } - return nil + // deny with versionId + err = delObject(res.res.VersionId, true) + if err != nil { + return err + } + + // allow without versionId + err = delObject(nil, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // recreate the object + res, err = putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + // deny without versionId + err = delObject(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = delObject(res.res.VersionId, false) + if err != nil { + return err + } + + return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_AccessControl_GetObjectAttributes_policy(s *S3Conf) error { testName := "Versioning_AccessControl_GetObjectAttributes_policy" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - testuser := getUser("user") - err = createUsers(s, []user{testuser}) + err := createUsers(s, []user{testuser}) if err != nil { return err } userClient := s.getUserClient(testuser) - getObjectAttr := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - ObjectAttributes: types.ObjectAttributesChecksum.Values(), - }) - cancel() - if denyAccess { - return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + getObjectAttr := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + ObjectAttributes: types.ObjectAttributesChecksum.Values(), + }) + cancel() + if denyAccess { + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + } + + return nil + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny with versionId + err = getObjectAttr(res.res.VersionId, true) + if err != nil { + return err + } + + // allow without versionId + err = getObjectAttr(nil, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersionAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny without versionId + err = getObjectAttr(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = getObjectAttr(res.res.VersionId, false) + if err != nil { + return err } return nil - } - - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny with versionId - err = getObjectAttr(res.res.VersionId, true) - if err != nil { - return err - } - - // allow without versionId - err = getObjectAttr(nil, false) - if err != nil { - return err - } - - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersionAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny without versionId - err = getObjectAttr(nil, true) - if err != nil { - return err - } - - // allow with versionId - err = getObjectAttr(res.res.VersionId, false) - if err != nil { - return err - } - - return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -3754,88 +4757,90 @@ func Versioning_GetObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &types.Tagging{ - TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, - }, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, + }, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetDeleteObjectTagging_delete_marker(s *S3Conf) error { testName := "Versioning_PutGetDeleteObjectTagging_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // DeleteObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - Tagging: &types.Tagging{ - TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // DeleteObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } @@ -3868,23 +4873,24 @@ func Versioning_PutObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -3914,113 +4920,115 @@ func Versioning_DeleteObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_DeleteObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_DeleteObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetDeleteObjectTagging_success(s *S3Conf) error { testName := "Versioning_PutGetDeleteObjectTagging_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - versions, err := createObjVersions(s3client, bucket, obj, 5) - if err != nil { - return err - } - versionId := versions[2].VersionId + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 5) + if err != nil { + return err + } + versionId := versions[2].VersionId - tagging := types.Tagging{ - TagSet: []types.Tag{ - {Key: getPtr("key"), Value: getPtr("value")}, - }, - } + tagging := types.Tagging{ + TagSet: []types.Tag{ + {Key: getPtr("key"), Value: getPtr("value")}, + }, + } - compareVersionId := func(expected, input *string) error { - if getString(expected) != getString(input) { - return fmt.Errorf("expected the response versionId to be %s, instead got %s", getString(expected), getString(input)) + compareVersionId := func(expected, input *string) error { + if getString(expected) != getString(input) { + return fmt.Errorf("expected the response versionId to be %s, instead got %s", getString(expected), getString(input)) + } + + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if err := compareVersionId(versionId, res.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + if !areTagsSame(tagging.TagSet, out.TagSet) { + return fmt.Errorf("expected the object version tags to be %v, instead got %v", tagging.TagSet, out.TagSet) + } + if err := compareVersionId(versionId, out.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if err := compareVersionId(versionId, resp.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + r, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if len(r.TagSet) != 0 { + return fmt.Errorf("expected empty tag set, instead got %v", r.TagSet) } return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &tagging, - VersionId: versionId, }) - cancel() - if err != nil { - return err - } - - if err := compareVersionId(versionId, res.VersionId); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } - if !areTagsSame(tagging.TagSet, out.TagSet) { - return fmt.Errorf("expected the object version tags to be %v, instead got %v", tagging.TagSet, out.TagSet) - } - if err := compareVersionId(versionId, out.VersionId); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } - - if err := compareVersionId(versionId, resp.VersionId); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - r, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } - - if len(r.TagSet) != 0 { - return fmt.Errorf("expected empty tag set, instead got %v", r.TagSet) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } From 729ced91e91dadd64222c37715dfa755890e2926 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sat, 19 Sep 2026 03:11:40 +0400 Subject: [PATCH 2/6] fix: stop posix publish lock serializing the same key across buckets `lockObjectPublish` picked the process-local slot by the key's shard alone, while the shared lock file is per bucket and shard. As a result, writes to the same key name in different buckets waited on each other even though their lock files never conflict. The slot is now chosen from the bucket hash and the shard (`bucketHash[0] ^ shard`). It still depends only on the lock file, never on the full key, so all requests for one lock file keep sharing a slot, which fcntl locks need because they don't exclude within a process. --- backend/posix/objlock.go | 21 +++++++++++++---- backend/posix/posix_conditional_put_test.go | 26 ++++++++++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/backend/posix/objlock.go b/backend/posix/objlock.go index 13098e37..8d507d12 100644 --- a/backend/posix/objlock.go +++ b/backend/posix/objlock.go @@ -49,6 +49,11 @@ import ( // file takes its place, which is unsafe to detect reliably on NFS due to // attribute caching. // +// The process-local slot is picked from the bucket hash and the shard, so all +// requests for one lock file take the same slot (fcntl locks don't exclude +// within a process), while the same key in different buckets usually takes +// different slots and doesn't wait on an unrelated lock. +// // The lock is held only for the commit phase (condition re-check, metadata // stores, final link/rename) — request bodies are staged to a temp file // before the lock is taken. The OS releases advisory locks automatically when @@ -131,15 +136,22 @@ func objLockShard(object string) uint8 { return sum[0] } +// objLockSlot returns the process-local slot index for the lock file of the +// shard in the bucket with bucketHash. +func objLockSlot(bucketHash [sha256.Size]byte, shard uint8) uint8 { + return bucketHash[0] ^ shard +} + // lockObjectPublish acquires the publish lock for bucket/object. It returns a // release function that must be called (typically deferred) once the new // object state is visible. All code paths that create or replace an object at // its final key must hold this lock across condition evaluation and // publication. func (p *Posix) lockObjectPublish(ctx context.Context, bucket, object string) (func(), error) { + bucketHash := sha256.Sum256([]byte(bucket)) shard := objLockShard(object) - slot := p.objLockSlots[shard] + slot := p.objLockSlots[objLockSlot(bucketHash, shard)] select { case <-ctx.Done(): return nil, ctx.Err() @@ -154,7 +166,7 @@ func (p *Posix) lockObjectPublish(ctx context.Context, bucket, object string) (f return releaseLocal, nil } - f, err := p.openObjLockFile(bucket, shard) + f, err := p.openObjLockFile(bucketHash, shard) if err != nil { releaseLocal() return nil, err @@ -188,9 +200,8 @@ func newObjLockSlots() [objLockShards]chan struct{} { } // openObjLockFile opens (creating as needed) the lock file for the shard in -// the given bucket. -func (p *Posix) openObjLockFile(bucket string, shard uint8) (*os.File, error) { - bucketHash := sha256.Sum256([]byte(bucket)) +// the bucket with bucketHash. +func (p *Posix) openObjLockFile(bucketHash [sha256.Size]byte, shard uint8) (*os.File, error) { lockDir := filepath.Join(p.rootdir, objLockDir, fmt.Sprintf("%x", bucketHash)) name := filepath.Join(lockDir, fmt.Sprintf("%02x", shard)) diff --git a/backend/posix/posix_conditional_put_test.go b/backend/posix/posix_conditional_put_test.go index 9e392fae..cce428c1 100644 --- a/backend/posix/posix_conditional_put_test.go +++ b/backend/posix/posix_conditional_put_test.go @@ -17,6 +17,7 @@ package posix import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "io" @@ -85,9 +86,9 @@ func TestObjectPublishLockHonorsContextWhileWaiting(t *testing.T) { bucket := "testbucket" createTestBucket(t, p, bucket) - shard := objLockShard("cancel-wait") - <-p.objLockSlots[shard] - defer func() { p.objLockSlots[shard] <- struct{}{} }() + slot := objLockSlot(sha256.Sum256([]byte(bucket)), objLockShard("cancel-wait")) + <-p.objLockSlots[slot] + defer func() { p.objLockSlots[slot] <- struct{}{} }() if _, err := os.Stat(p.ObjectPath(bucket, objLockDir)); !errors.Is(err, fs.ErrNotExist) { t.Fatalf("bucket contains publish lock directory: %v", err) } @@ -134,6 +135,25 @@ func TestObjectPublishLockHonorsCancellationAfterSlotAcquired(t *testing.T) { } } +func TestObjectPublishLockDoesNotBlockOtherBuckets(t *testing.T) { + p := newTestPosix(t, metaModes(t)["xattr"]) + + unlock, err := p.lockObjectPublish(context.Background(), "bucket-a", "my-obj") + if err != nil { + t.Fatalf("lock object publish: %v", err) + } + defer unlock() + + // the same key in another bucket has its own lock file + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + unlockOther, err := p.lockObjectPublish(ctx, "bucket-b", "my-obj") + if err != nil { + t.Fatalf("lock object publish in another bucket: %v", err) + } + unlockOther() +} + func TestObjectPublishLockModes(t *testing.T) { tests := []struct { name string From bd6cfe64a30b903d8b84c62a063d043c468352c3 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sat, 19 Sep 2026 22:26:47 +0400 Subject: [PATCH 3/6] fix: stop posix mixing up keys that differ only by a trailing slash In posix a key and the same key with a trailing slash, such as `foo` and `foo/`, map to one path and share one set of attributes: a file there is the object of `foo`, a directory object is the object of `foo/`. Several operations used the entry at that path without checking that it belongs to the requested key. They now check it through the new `isLiveObject`, `statLiveObject` and `objVersionAttrPath` helpers. `DeleteObject` of `foo` with a version id of `foo/` treated the directory as the current version of `foo` and removed it. That left the noncurrent versions of `foo/` unlisted and undeletable, so `DeleteBucket` failed with `BucketNotEmpty`. A delete of `foo` without a version id failed with an internal error while trying to version the directory as a file. Both now succeed without touching `foo/`, as for any object that doesn't exist. `CompleteMultipartUpload` of `foo` cleared the `delete-marker` attribute of `foo/` and then failed with an internal error when linking the object onto the directory, which turned the delete marker back into a live version. It now returns `ExistingObjectIsDirectory` before any attribute is changed, both before the parts are assembled and again under the object publish lock. The idempotent completion path also no longer reports a missing upload as completed just because `foo/` exists. The object tagging, legal hold and retention APIs read and wrote the attributes of the other key. For example, `PutObjectLegalHold` on `foo` could turn off the legal hold of `foo/`, and the object lock check could block a delete because of the other key's retention. When the requested key has no object, they now return `NoSuchKey` for the current version and `NoSuchVersion` for a specific version. They also resolve the `null` version id to a current null version instead of looking for it in the versioning directory. The object lock check treats `NoSuchVersion` like `NoSuchKey`, since a version that doesn't exist has nothing to protect. Conditional writes no longer evaluate `If-Match` and `If-None-Match` against the other key's ETag. `CreateMultipartUpload` now stores the upload's tagging and object lock settings directly on the upload directory, and `PutObject` of a directory object sets its tagging after the directory gets its ETag. --- auth/object_lock.go | 7 +- backend/posix/posix.go | 357 +++++++++++++++---------------- tests/integration/group-tests.go | 21 ++ tests/integration/posix.go | 250 ++++++++++++++++++++++ tests/integration/utils.go | 167 +++++++++++++++ tests/integration/versioning.go | 253 ++++++++++++++++++++++ 6 files changed, 873 insertions(+), 182 deletions(-) diff --git a/auth/object_lock.go b/auth/object_lock.go index 77083cb2..ba502e50 100644 --- a/auth/object_lock.go +++ b/auth/object_lock.go @@ -495,7 +495,9 @@ func (s objectLockState) checkObject(ctx context.Context, be backend.Backend, ia checkRetention := true retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId) - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + // an object or version that doesn't exist has nothing to protect + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) || + errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) { return nil } // the object is a delete marker, if a `MethodNotAllowed` error is returned @@ -538,7 +540,8 @@ func (s objectLockState) checkObject(ctx context.Context, be backend.Backend, ia status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId) if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) || + errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) { return nil } if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 018421be..a93ad686 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -631,21 +631,20 @@ func (p *Posix) doesBucketExist(bucket string) error { return nil } -func (p *Posix) doesBucketAndObjectExist(bucket, object string) error { +// doesBucketAndObjectExist checks that bucket exists and, when versionId is +// empty, that object has a current version. A specific version is looked up +// once versionId is validated. +func (p *Posix) doesBucketAndObjectExist(bucket, object, versionId string) error { err := p.doesBucketExist(bucket) if err != nil { return err } - - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil { - return fmt.Errorf("stat object: %w", err) + if versionId != "" { + return nil } - return nil + _, _, err = p.objVersionAttrPath(bucket, object, "") + return err } func (p *Posix) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { @@ -1300,15 +1299,79 @@ func (p *Posix) isDirObject(bucket, key string) (bool, error) { return true, nil } -// isLiveDirObject reports whether fi, the entry at the path of the -// directory object key, is a directory object -func (p *Posix) isLiveDirObject(fi os.FileInfo, bucket, key string) (bool, error) { - if !fi.IsDir() { +// isLiveObject reports whether fi, the entry at the path of key, is the +// current version of key. A key and the same key with a trailing slash +// share one path and one set of attributes: a file there is the object of +// the key without the slash, a directory object the object of the key with +// it. +func (p *Posix) isLiveObject(fi os.FileInfo, bucket, key string) (bool, error) { + if fi.IsDir() != strings.HasSuffix(key, "/") { return false, nil } + if !fi.IsDir() { + return true, nil + } return p.isDirObject(bucket, key) } +// statLiveObject returns the file info of the current version of key. An +// error matching fs.ErrNotExist is returned when key has no current +// version, including when the entry at its path is another key's object. +func (p *Posix) statLiveObject(bucket, key string) (os.FileInfo, error) { + fi, err := os.Stat(p.ObjectPath(bucket, key)) + if isErrNotDir(err) { + return nil, fs.ErrNotExist + } + if err != nil { + return nil, err + } + isObj, err := p.isLiveObject(fi, bucket, key) + if err != nil { + return nil, err + } + if !isObj { + return nil, fs.ErrNotExist + } + return fi, nil +} + +// objVersionAttrPath returns the bucket and object that the attributes of +// the version versionId of key are stored at: key itself for its current +// version, an entry of the versioning directory for other versions. An +// empty versionId selects the current version, and NoSuchKey is returned if +// key has none. +func (p *Posix) objVersionAttrPath(bucket, key, versionId string) (string, string, error) { + _, err := p.statLiveObject(bucket, key) + if isErrNameTooLong(err) { + return "", "", s3err.GetKeyTooLongErr(int64(len(key)), 1024) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", "", fmt.Errorf("stat object: %w", err) + } + isLive := err == nil + + if versionId == "" { + if !isLive { + return "", "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + return bucket, key, nil + } + + if isLive { + vId, err := p.meta.RetrieveAttribute(nil, bucket, key, versionIdKey) + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } else if err != nil { + return "", "", fmt.Errorf("get obj versionId: %w", err) + } + if string(vId) == versionId { + return bucket, key, nil + } + } + + return filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), versionId), nil +} + // clearDirObjectAttrs removes the directory object attributes, including // legacy metadata attributes, from the directory at bucket/key. The etag is // kept: it marks the directory as an object, so a failure before the new @@ -1962,7 +2025,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create // set object tagging if tags != nil { - err := p.PutObjectTagging(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", tags) + err := p.storeObjectTags(bucket, filepath.Join(objdir, uploadID), tags) if err != nil { // cleanup object if returning error os.RemoveAll(filepath.Join(tmppath, uploadID)) @@ -1993,7 +2056,10 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create // set object legal hold if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", true) + err := p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectLegalHoldKey, []byte{1}) + } if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) @@ -2020,7 +2086,10 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("parse object lock retention: %w", err) } - err = p.PutObjectRetention(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", retParsed) + err = p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectRetentionKey, retParsed) + } if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) @@ -2313,7 +2382,9 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C }, "", nil } // Directory is gone: the concurrent call already completed and cleaned up. - if _, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil { + // A directory at the object path is the object of the key with a + // trailing slash, not the completed upload. + if fi, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil && !fi.IsDir() { etag := multipartClaimToken if p.dataIntegrityEtag { etagBytes, etagErr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) @@ -2383,13 +2454,17 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C defer os.Rename(uploadIDInProgress, uploadIDDir) defer p.meta.RenameObject(bucket, newMetaObj, oldMetaObj) - // Fast-fail precondition check before the parts are assembled. This is - // only advisory: the authoritative check is repeated while holding the - // object publish lock just before the final link. + // Fast-fail precondition and directory checks before the parts are + // assembled. These are only advisory: the authoritative checks are + // repeated while holding the object publish lock just before the final + // link. err = p.checkPutPreconditions(bucket, object, input.IfMatch, input.IfNoneMatch) if err != nil { return res, "", err } + if d, err := os.Stat(p.ObjectPath(bucket, object)); err == nil && d.IsDir() { + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, activeUploadName)) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -2717,9 +2792,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C vEnabled := p.isBucketVersioningEnabled(vStatus) d, err := os.Stat(objname) + if err == nil && d.IsDir() { + // the directory is the object of the key with a trailing slash, or + // the parent of other objects: its attributes are not this object's + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } // if the versioning is enabled first create the file object version - if p.versioningEnabled() && vEnabled && err == nil && !d.IsDir() { + if p.versioningEnabled() && vEnabled && err == nil { _, err := p.createObjVersion(bucket, object, d.Size(), acct, false) if err != nil { return res, "", fmt.Errorf("create object version: %w", err) @@ -4189,6 +4269,13 @@ func (p *Posix) checkPutPreconditions(bucket, object string, ifMatch, ifNoneMatc return s3err.GetAPIError(s3err.ErrNotImplemented) } + // the etag at the object path may be the one of the key with or + // without the trailing slash + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + return backend.EvaluateObjectPutPreconditions("", ifMatch, ifNoneMatch, false) + } + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) if err == nil || errors.Is(err, fs.ErrNotExist) || errors.Is(err, meta.ErrNoSuchKey) { return backend.EvaluateObjectPutPreconditions(string(etagBytes), ifMatch, ifNoneMatch, err == nil) @@ -4420,14 +4507,6 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err) } - // Set object tagging - if tags != nil { - err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) - if err != nil { - return s3response.PutObjectOutput{}, err - } - } - dirETag := emptyMD5 if p.dataIntegrityEtag { dirETag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksumAlgorithm)), expectedSum) @@ -4483,6 +4562,14 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje _ = os.Chtimes(name, now, now) } + // Set object tagging once the etag makes the directory an object + if tags != nil { + err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + err = p.putObjectLockSettings(ctx, po) if err != nil { return s3response.PutObjectOutput{}, err @@ -4896,15 +4983,15 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if isDir { - isObj, err := p.isLiveDirObject(fi, bucket, object) - if err != nil { - return nil, err - } - if !isObj { - // AWS returns success if the object does not exist - return &s3.DeleteObjectOutput{}, nil - } + // the entry at the object path may be the object of the key + // with or without the trailing slash + isObj, err := p.isLiveObject(fi, bucket, object) + if err != nil { + return nil, err + } + if !isObj { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil } err = evalPreconditions(fi, bucket, object) @@ -4962,28 +5049,19 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } else { versionPath := p.genObjVersionPath(bucket, object) - if isDir { - // the attributes at a directory object path may belong to a - // file or to a directory that isn't an object - fi, err := os.Stat(objpath) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - // AWS returns success if the object does not exist - return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil - } - if isErrNameTooLong(err) { - return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) - } - if err != nil { - return nil, fmt.Errorf("stat object: %w", err) - } - isObj, err := p.isLiveDirObject(fi, bucket, object) - if err != nil { - return nil, err - } - if !isObj { - // AWS returns success if the object does not exist - return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil - } + // the attributes at the object path may belong to the key with + // or without the trailing slash, or to a directory that isn't an + // object + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) @@ -4997,16 +5075,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, fmt.Errorf("get obj versionId: %w", err) } if errors.Is(err, meta.ErrNoSuchKey) { - // With sidecar, ErrNoSuchKey means "attribute absent" regardless of - // whether the data file exists. If the file is absent the object - // does not exist at all → AWS returns success for DeleteObject. - // Also handle ENOTDIR: when a key such as "foo/bar" is requested - // but "foo" is a regular file (not a directory), the path cannot - // contain any object. - _, statErr := os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) { - return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil - } vId = []byte(nullVersionId) } @@ -6836,36 +6904,14 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId return nil, err } - if versionId == "" { - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if isErrNameTooLong(err) { - return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) - } - if err != nil { - return nil, fmt.Errorf("stat object: %w", err) - } + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - if versionId != "" { - if !p.versioningEnabled() { - //TODO: Maybe we need to return our custom error here? - return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) - } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) - } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } err = p.ensureNotDeleteMarker(bucket, object, versionId) @@ -6923,36 +6969,14 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return err } - if versionId == "" { - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if isErrNameTooLong(err) { - return s3err.GetAPIError(s3err.ErrKeyTooLong) - } - if err != nil { - return fmt.Errorf("stat object: %w", err) - } + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - if versionId != "" { - if !p.versioningEnabled() { - //TODO: Maybe we need to return our custom error here? - return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) - } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) - } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } err = p.ensureNotDeleteMarker(bucket, object, versionId) @@ -6977,12 +7001,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return nil } - b, err := json.Marshal(tags) - if err != nil { - return fmt.Errorf("marshal tags: %w", err) - } - - err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) + err = p.storeObjectTags(bucket, object, tags) if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) @@ -6996,6 +7015,16 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return nil } +// storeObjectTags stores tags as the tagging attribute of bucket/object +func (p *Posix) storeObjectTags(bucket, object string, tags map[string]string) error { + b, err := json.Marshal(tags) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + return p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) +} + func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object, versionId string) error { if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) @@ -7317,7 +7346,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err } @@ -7342,17 +7371,9 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } } @@ -7385,7 +7406,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err } @@ -7403,17 +7424,9 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } } @@ -7451,7 +7464,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err } @@ -7469,17 +7482,9 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } } @@ -7506,7 +7511,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err } @@ -7524,17 +7529,9 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } } diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 02b9e1a1..c9961db4 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -990,6 +990,12 @@ func TestPosix(ts *TestState) { ts.Run(DeleteObject_name_too_long) ts.Run(CopyObject_overwrite_same_dir_object) ts.Run(CopyObject_overwrite_same_file_object) + ts.Run(CompleteMultipartUpload_overwrite_dir_obj) + if ts.conf.versioningEnabled { + ts.Run(CompleteMultipartUpload_overwrite_dir_obj_delete_marker) + } + ts.Run(ObjectTagging_trailing_slash_counterpart) + ts.Run(ObjectLock_trailing_slash_counterpart) ts.Run(DeleteObject_directory_not_empty) if !ts.conf.windowsTests { ts.Run(PutObject_race_with_delete) @@ -1094,6 +1100,9 @@ func TestScoutfs(ts *TestState) { ts.Run(DeleteObject_name_too_long) ts.Run(CopyObject_overwrite_same_dir_object) ts.Run(CopyObject_overwrite_same_file_object) + ts.Run(CompleteMultipartUpload_overwrite_dir_obj) + ts.Run(ObjectTagging_trailing_slash_counterpart) + ts.Run(ObjectLock_trailing_slash_counterpart) ts.Run(DeleteObject_directory_not_empty) } @@ -1977,6 +1986,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_DeleteObjectTagging_invalid_versionId) ts.Run(Versioning_DeleteObjectTagging_non_existing_object_version) ts.Run(Versioning_PutGetDeleteObjectTagging_success) + ts.Run(Versioning_ObjectTagging_trailing_slash_counterpart) // GetObjectAttributes action ts.Run(Versioning_GetObjectAttributes_invalid_versionId) ts.Run(Versioning_GetObjectAttributes_object_version) @@ -1987,6 +1997,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_DeleteObject_dir_object_latest_version) ts.Run(Versioning_DeleteObject_non_existing_object) ts.Run(Versioning_DeleteObject_implicit_dir) + ts.Run(Versioning_DeleteObject_trailing_slash_counterpart) if !ts.conf.windowsTests { ts.Run(Versioning_DeleteObject_delete_a_delete_marker) ts.Run(Versioning_DeleteObject_dir_object_with_children) @@ -2047,6 +2058,8 @@ func TestVersioning(ts *TestState) { if !ts.conf.windowsTests { ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention) } + ts.Run(Versioning_WORM_trailing_slash_counterpart) + ts.Run(Versioning_WORM_null_version_locked_with_legal_hold) // Concurrent requests // Versioninig_concurrent_upload_object ts.Run(Versioning_AccessControl_GetObjectVersion) @@ -2989,6 +3002,10 @@ func GetIntTests() IntTests { "DeleteObject_name_too_long": DeleteObject_name_too_long, "CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object, "CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object, + "CompleteMultipartUpload_overwrite_dir_obj": CompleteMultipartUpload_overwrite_dir_obj, + "CompleteMultipartUpload_overwrite_dir_obj_delete_marker": CompleteMultipartUpload_overwrite_dir_obj_delete_marker, + "ObjectTagging_trailing_slash_counterpart": ObjectTagging_trailing_slash_counterpart, + "ObjectLock_trailing_slash_counterpart": ObjectLock_trailing_slash_counterpart, "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, "DeleteObject_directory_object": DeleteObject_directory_object, "DeleteObject_success": DeleteObject_success, @@ -3499,6 +3516,7 @@ func GetIntTests() IntTests { "Versioning_DeleteObjectTagging_invalid_versionId": Versioning_DeleteObjectTagging_invalid_versionId, "Versioning_DeleteObjectTagging_non_existing_object_version": Versioning_DeleteObjectTagging_non_existing_object_version, "Versioning_PutGetDeleteObjectTagging_success": Versioning_PutGetDeleteObjectTagging_success, + "Versioning_ObjectTagging_trailing_slash_counterpart": Versioning_ObjectTagging_trailing_slash_counterpart, "Versioning_GetObjectAttributes_invalid_versionId": Versioning_GetObjectAttributes_invalid_versionId, "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, @@ -3507,6 +3525,7 @@ func GetIntTests() IntTests { "Versioning_DeleteObject_dir_object_latest_version": Versioning_DeleteObject_dir_object_latest_version, "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, "Versioning_DeleteObject_implicit_dir": Versioning_DeleteObject_implicit_dir, + "Versioning_DeleteObject_trailing_slash_counterpart": Versioning_DeleteObject_trailing_slash_counterpart, "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, "Versioning_DeleteObject_dir_object_with_children": Versioning_DeleteObject_dir_object_with_children, "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, @@ -3557,6 +3576,8 @@ func GetIntTests() IntTests { "Versioning_WORM_CopyObject_overwrite_locked_object": Versioning_WORM_CopyObject_overwrite_locked_object, "Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object": Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object, "Versioning_WORM_remove_delete_marker_under_bucket_default_retention": Versioning_WORM_remove_delete_marker_under_bucket_default_retention, + "Versioning_WORM_trailing_slash_counterpart": Versioning_WORM_trailing_slash_counterpart, + "Versioning_WORM_null_version_locked_with_legal_hold": Versioning_WORM_null_version_locked_with_legal_hold, "Versioning_AccessControl_GetObjectVersion": Versioning_AccessControl_GetObjectVersion, "Versioning_AccessControl_HeadObjectVersion": Versioning_AccessControl_HeadObjectVersion, "Versioning_AccessControl_object_tagging_policy": Versioning_AccessControl_object_tagging_policy, diff --git a/tests/integration/posix.go b/tests/integration/posix.go index b9401a77..71454f59 100644 --- a/tests/integration/posix.go +++ b/tests/integration/posix.go @@ -17,6 +17,7 @@ package integration import ( "context" "fmt" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -222,6 +223,255 @@ func CopyObject_overwrite_same_file_object(s *S3Conf) error { }) } +func CompleteMultipartUpload_overwrite_dir_obj(s *S3Conf) error { + testName := "CompleteMultipartUpload_overwrite_dir_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "foo/", "foo" + _, err := putObjects(s3client, []string{dir}, bucket) + if err != nil { + return err + } + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + + // the directory object isn't taken for the object of a completed upload + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: getPtr("non-existing-upload-id"), + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + + // the failed upload can still be completed or aborted + return checkAndAbortUpload(s3client, bucket, obj, *mp.UploadId) + }) +} + +func CompleteMultipartUpload_overwrite_dir_obj_delete_marker(s *S3Conf) error { + testName := "CompleteMultipartUpload_overwrite_dir_obj_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "foo/", "foo" + versions, err := createObjVersions(s3client, bucket, dir, 1) + if err != nil { + return err + } + versions[0].IsLatest = getPtr(false) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + delMarkers := []types.DeleteMarkerEntry{ + {Key: &dir, VersionId: out.VersionId, IsLatest: getPtr(true)}, + } + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + + // the delete marker of the directory object is left in place + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(delMarkers, res.DeleteMarkers) { + return fmt.Errorf("expected the delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return checkAndAbortUpload(s3client, bucket, obj, *mp.UploadId) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func ObjectTagging_trailing_slash_counterpart(s *S3Conf) error { + testName := "ObjectTagging_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + tagSet := []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}} + + // a key and the same key with a trailing slash have one path: the + // object of one of them isn't an object of the other + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key=value"), + }, s3client) + if err != nil { + return err + } + + err = checkObjectTaggingErr(s3client, bucket, other, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if !areTagsSame(res.TagSet, tagSet) { + return fmt.Errorf("%v: expected the tag set to be %v, instead got %v", + obj, tagSet, res.TagSet) + } + } + + // neither key names the parent directory of an object + _, err := putObjects(s3client, []string{"my-parent/obj"}, bucket) + if err != nil { + return err + } + for _, key := range []string{"my-parent/", "my-parent"} { + err := checkObjectTaggingErr(s3client, bucket, key, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", key, err) + } + } + + return nil + }) +} + +func ObjectLock_trailing_slash_counterpart(s *S3Conf) error { + testName := "ObjectLock_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rDate := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + lockedObjs := []objToDelete{} + + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &rDate, + }, s3client) + if err != nil { + return err + } + lockedObjs = append(lockedObjs, objToDelete{key: obj, removeOnlyLeglHold: true}) + + err = checkObjectLockErr(s3client, bucket, other, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + // the lock of the object doesn't protect the other key + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + err = checkObjectLock(s3client, bucket, obj, "", rDate) + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + // PutObject_race_with_delete tests the race between PutObject and DeleteObject // in the same subdirectory. // One goroutine sequentially puts "race-dir/0.txt" … "race-dir/N-1.txt". diff --git a/tests/integration/utils.go b/tests/integration/utils.go index e12976bb..7f5e2cbb 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -3989,3 +3989,170 @@ func checkDeleteObjectsErrsInOrder(got []types.Error, want []keyDenial) error { } return nil } + +// checkObjectTaggingErr checks that getting, putting and deleting the +// tagging of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectTaggingErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("other-key"), Value: getPtr("other-value")}}, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("delete object tagging: %w", err) + } + + return nil +} + +// checkObjectLockErr checks that getting and putting the legal hold and the +// retention of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectLockErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object retention: %w", err) + } + + rDate := time.Now().Add(time.Hour * 2) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object retention: %w", err) + } + + return nil +} + +// checkObjectLock checks that the version versionId of key is under legal +// hold and has a governance retention until rDate. An empty versionId +// selects the current version. +func checkObjectLock(client *s3.Client, bucket, key, versionId string, rDate time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + lHold, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if lHold.LegalHold == nil || lHold.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %q, instead got %v", + types.ObjectLockLegalHoldStatusOn, lHold.LegalHold) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ret, err := client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if ret.Retention == nil || ret.Retention.Mode != types.ObjectLockRetentionModeGovernance || + ret.Retention.RetainUntilDate == nil || ret.Retention.RetainUntilDate.Unix() != rDate.Unix() { + return fmt.Errorf("expected a %q retention until %v, instead got %+v", + types.ObjectLockRetentionModeGovernance, rDate.Format(time.RFC3339), ret.Retention) + } + + return nil +} + +// checkAndAbortUpload checks that the upload uploadId of key is the only +// multipart upload in the bucket and aborts it +func checkAndAbortUpload(client *s3.Client, bucket, key, uploadId string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + if len(res.Uploads) != 1 || getString(res.Uploads[0].Key) != key || + getString(res.Uploads[0].UploadId) != uploadId { + return fmt.Errorf("expected the upload %v of %v to be listed, instead got %+v", + uploadId, key, res.Uploads) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + }) + cancel() + return err +} diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 5af1e0ed..277846f8 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -2016,6 +2016,83 @@ func Versioning_DeleteObject_dir_object_with_children(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +func Versioning_DeleteObject_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_DeleteObject_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + expected := []types.ObjectVersion{} + // deleting the key with or without a trailing slash doesn't delete + // the object or any of its versions + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + versions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + expected = append(expected, versions...) + + for _, versionId := range []*string{versions[0].VersionId, versions[1].VersionId} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + VersionId: versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + if getString(out.VersionId) != *versionId { + return fmt.Errorf("%v: expected the versionId to be %v, instead got %v", + other, *versionId, getString(out.VersionId)) + } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("%v: expected the response DeleteMarker to be false", other) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if getString(res.VersionId) != getString(versions[0].VersionId) { + return fmt.Errorf("%v: expected the versionId to be %v, instead got %v", + obj, getString(versions[0].VersionId), getString(res.VersionId)) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(expected, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + expected, res.Versions) + } + + return nil + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_Delete_null_versionId_object(s *S3Conf) error { testName := "Versioning_Delete_null_versionId_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4196,6 +4273,139 @@ func Versioning_WORM_remove_delete_marker_under_bucket_default_retention(s *S3Co }, withLock()) } +func Versioning_WORM_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_WORM_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rDate := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + lockedObjs := []objToDelete{} + + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &rDate, + }, s3client) + if err != nil { + return err + } + versionId := getString(res.res.VersionId) + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: versionId, + removeOnlyLeglHold: true, + }) + + // the version belongs to the object, not to the other key + err = checkObjectLockErr(s3client, bucket, other, versionId, s3err.GetAPIError(s3err.ErrNoSuchVersion)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + VersionId: &versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + err = checkObjectLock(s3client, bucket, obj, versionId, rDate) + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + +func Versioning_WORM_null_version_locked_with_legal_hold(s *S3Conf) error { + testName := "Versioning_WORM_null_version_locked_with_legal_hold" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []string{"my-obj", "my-dir/"} + // the objects are put before versioning is enabled: their + // current versions are the null versions + _, err := putObjects(s3client, objs, bucket) + if err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{ + Bucket: &bucket, + ObjectLockConfiguration: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + }, + }) + cancel() + if err != nil { + return err + } + + lockedObjs := []objToDelete{} + err = forEachKey(objs, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: nullVersionId, + removeOnlyLeglHold: true, + }) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + if err != nil { + return err + } + if res.LegalHold == nil || res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %q, instead got %v", + types.ObjectLockLegalHoldStatusOn, res.LegalHold) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)) + }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }) +} + func Versioning_AccessControl_GetObjectVersion(s *S3Conf) error { testName := "Versioning_AccessControl_GetObjectVersion" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5032,3 +5242,46 @@ func Versioning_PutGetDeleteObjectTagging_success(s *S3Conf) error { }) }, withVersioning(types.BucketVersioningStatusEnabled)) } + +func Versioning_ObjectTagging_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_ObjectTagging_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + tagSet := []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}} + + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key=value"), + }, s3client) + if err != nil { + return err + } + versionId := getString(res.res.VersionId) + + // the version belongs to the object, not to the other key + err = checkObjectTaggingErr(s3client, bucket, other, versionId, s3err.GetAPIError(s3err.ErrNoSuchVersion)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if !areTagsSame(out.TagSet, tagSet) { + return fmt.Errorf("%v: expected the tag set to be %v, instead got %v", + obj, tagSet, out.TagSet) + } + } + + return nil + }, withVersioning(types.BucketVersioningStatusEnabled)) +} From d204d2e2383ac511d6641e47603a2c6ed5996290 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sun, 20 Sep 2026 15:35:08 +0400 Subject: [PATCH 4/6] fix: stop posix copying data from a delete marker `CopyObject` and `UploadPartCopy` never looked at the source's `delete-marker` attribute. A delete marker leaves the data file in place at the object path and only flags it, so a copy whose source resolved to a marker opened that file and succeeded, handing back the data of the version the marker had deleted. AWS rejects such a copy: `NoSuchKey` when the marker is the current version of the key, and `InvalidRequest` when the marker is named by version id, the latter regardless of whether it is the latest version. Versions the marker hides stay copyable by version id. Both copy paths now run the resolved source through `checkCopySourceDeleteMarker` once the entry has been validated, returning `NoSuchKey` for an unqualified source and the new `ErrCopySourceDeleteMarker` for one carrying a version id. --- backend/posix/posix.go | 24 +++++ s3err/s3err.go | 6 ++ tests/integration/group-tests.go | 4 + tests/integration/utils.go | 22 +++++ tests/integration/versioning.go | 151 +++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a93ad686..b3c7eaaf 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1618,6 +1618,24 @@ func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { return true, nil } +// checkCopySourceDeleteMarker rejects a copy whose source resolves to a +// delete marker: the key has no current version when the marker is the +// latest, and a marker named by version id holds no data to copy. +func (p *Posix) checkCopySourceDeleteMarker(bucket, object, versionId string) error { + isDel, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return err + } + if !isDel { + return nil + } + if versionId != "" { + return s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker) + } + + return s3err.GetAPIError(s3err.ErrNoSuchKey) +} + // Converts the file to object version. Finds all the object versions, // delete markers from the versioning directory and returns func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { @@ -4034,6 +4052,9 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) if strings.HasSuffix(srcObject, "/") != fi.IsDir() { return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyPartResult{}, err + } // a directory object holds no data srcSize := fi.Size() if fi.IsDir() { @@ -6237,6 +6258,9 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput if !strings.HasSuffix(srcObject, "/") && fi.IsDir() { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyObjectOutput{}, err + } // a directory object holds no data srcSize := fi.Size() var srcBody io.Reader = f diff --git a/s3err/s3err.go b/s3err/s3err.go index 5a02d440..6262d77e 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -131,6 +131,7 @@ const ( ErrMissingDateHeader ErrGetUploadsWithKey ErrVersionsWithKey + ErrCopySourceDeleteMarker ErrInvalidRequest ErrAuthNotSetup ErrNotImplemented @@ -453,6 +454,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "There is no such thing as the ?versions sub-resource for a key", HTTPStatusCode: http.StatusBadRequest, }, + ErrCopySourceDeleteMarker: { + Code: "InvalidRequest", + Description: "The source of a copy request may not specifically refer to a delete marker by version id.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidRequest: { Code: "InvalidRequest", Description: "Invalid Request.", diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index c9961db4..cbed13f6 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -1957,6 +1957,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_CopyObject_success) ts.Run(Versioning_CopyObject_non_existing_version_id) ts.Run(Versioning_CopyObject_from_an_object_version) + ts.Run(Versioning_CopyObject_from_a_delete_marker) if !ts.conf.windowsTests { ts.Run(Versioning_CopyObject_special_chars) } @@ -2026,6 +2027,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId) ts.Run(Versioning_UploadPartCopy_non_existing_versionId) ts.Run(Versioning_UploadPartCopy_from_an_object_version) + ts.Run(Versioning_UploadPartCopy_from_a_delete_marker) // Object lock configuration ts.Run(Versioning_object_lock_not_enabled_on_bucket_creation) ts.Run(Versioning_Enable_object_lock) @@ -3494,6 +3496,7 @@ func GetIntTests() IntTests { "Versioning_CopyObject_success": Versioning_CopyObject_success, "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, + "Versioning_CopyObject_from_a_delete_marker": Versioning_CopyObject_from_a_delete_marker, "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, "Versioning_HeadObject_non_existing_object_version": Versioning_HeadObject_non_existing_object_version, @@ -3550,6 +3553,7 @@ func GetIntTests() IntTests { "Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId": Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId, "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, + "Versioning_UploadPartCopy_from_a_delete_marker": Versioning_UploadPartCopy_from_a_delete_marker, "Versioning_object_lock_not_enabled_on_bucket_creation": Versioning_object_lock_not_enabled_on_bucket_creation, "Versioning_Enable_object_lock": Versioning_Enable_object_lock, "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 7f5e2cbb..f849a966 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -2375,6 +2375,28 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts return versions, nil } +// createDeleteMarker deletes object without a version id, making the +// resulting delete marker the current version, and returns its version id. +func createDeleteMarker(client *s3.Client, bucket, object string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err != nil { + return "", err + } + if out.DeleteMarker == nil || !*out.DeleteMarker { + return "", fmt.Errorf("expected a delete marker to be created for %v", object) + } + if getString(out.VersionId) == "" { + return "", fmt.Errorf("expected non empty delete marker versionId for %v", object) + } + + return *out.VersionId, nil +} + // objDataLen returns the data length to upload for key: a directory // object can't hold data func objDataLen(key string, lgth int64) int64 { diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 277846f8..45f2491a 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -670,6 +670,78 @@ func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +// A copy source that resolves to a delete marker is rejected: the key has no +// current version when the marker is the latest, and naming the marker by +// version id is an invalid request. Versions the marker hides stay copyable. +func Versioning_CopyObject_from_a_delete_marker(s *S3Conf) error { + testName := "Versioning_CopyObject_from_a_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" + if err := setup(s, dstBucket); err != nil { + return err + } + + err := forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + + delMarker, err := createDeleteMarker(s3client, bucket, srcObj) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, delMarker)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, getString(srcObjVersions[0].VersionId))), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != getString(srcObjVersions[0].VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersions[0].VersionId), getString(out.CopySourceVersionId)) + } + + return nil + }) + if err != nil { + return err + } + + return teardown(s, dstBucket) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_CopyObject_special_chars(s *S3Conf) error { testName := "Versioning_CopyObject_special_chars" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3009,6 +3081,85 @@ func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +// A copy source that resolves to a delete marker is rejected: the key has no +// current version when the marker is the latest, and naming the marker by +// version id is an invalid request. Versions the marker hides stay copyable. +func Versioning_UploadPartCopy_from_a_delete_marker(s *S3Conf) error { + testName := "Versioning_UploadPartCopy_from_a_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" + if err := setup(s, dstBucket); err != nil { + return err + } + + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + + delMarker, err := createDeleteMarker(s3client, bucket, srcObj) + if err != nil { + return err + } + + mp, err := createMp(s3client, dstBucket, dstObj) + if err != nil { + return err + } + + partNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, delMarker)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, getString(srcObjVersions[0].VersionId))), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != getString(srcObjVersions[0].VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersions[0].VersionId), getString(out.CopySourceVersionId)) + } + + return teardown(s, dstBucket) + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_Enable_object_lock(s *S3Conf) error { testName := "Versioning_Enable_object_lock" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { From ba02c766abf09da68b014366bdc5686211f6101c Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sun, 20 Sep 2026 16:07:40 +0400 Subject: [PATCH 5/6] fix: stop posix rewriting the current version on a copy to itself `CopyObject` treated a destination path equal to the source path as an in-place metadata rewrite, regardless of bucket versioning. On a versioned bucket a self copy therefore edited the current version and returned its existing version id instead of creating a new one. This also defeated object lock: `CheckObjectAccess` skips the retention and legal hold checks for overwrites on version-enabled buckets because an overwrite is expected to create a new version, so a `COMPLIANCE` retained or legal held version had its metadata replaced underneath it. A self copy is now rewritten in place only when the bucket is unversioned, and otherwise goes through the regular copy path, which creates a new version and leaves the one it replaces untouched. The rejection of a self copy that replaces nothing moved out of the in-place branch and is now applied only when the copy source carries no version id. Naming a version explicitly makes the request a regular copy, which AWS accepts even with the `COPY` metadata directive, while versitygw answered `InvalidRequest`. Source tagging for a `COPY` tagging directive is now read before `PutObject` writes the destination rather than after. A self copy replaces the source object's attributes, so the later read returned nothing and the tags were dropped from the new version. --- backend/posix/posix.go | 53 +++-- backend/posix/posix_io_helpers.go | 19 ++ backend/posix/posix_io_helpers_test.go | 79 +++++++ tests/integration/group-tests.go | 6 + tests/integration/versioning.go | 275 +++++++++++++++++++++++++ 5 files changed, 416 insertions(+), 16 deletions(-) create mode 100644 backend/posix/posix_io_helpers_test.go diff --git a/backend/posix/posix.go b/backend/posix/posix.go index b3c7eaaf..165a67f9 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -6310,11 +6310,18 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput var chType types.ChecksumType dstObjdPath := joinPathWithTrailer(p.BucketPath(dstBucket), dstObject) - if dstObjdPath == objPath { - if input.MetadataDirective == types.MetadataDirectiveCopy { - return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) - } + // A copy of an object onto itself is rejected unless it replaces the + // object metadata. Naming a source version makes it a regular copy. + selfCopy := dstObjdPath == objPath + if selfCopy && srcVersionId == "" && + input.MetadataDirective == types.MetadataDirectiveCopy { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + // In a versioned bucket a self copy creates a new version like any other + // write, so only unversioned buckets are rewritten in place. + versioned := p.versioningEnabled() && vStatus != "" + if selfCopy && !versioned { // Delete the object metadata err = p.meta.DeleteAttribute(dstBucket, dstObject, metadataHdr) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6452,6 +6459,14 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput checksums.Algorithm = input.ChecksumAlgorithm } + // A self copy publishes the new version over the source path, which + // on Windows can't be renamed over while the source is still open. + // PutObject reads the body before publishing, so the handle is + // released as soon as the data has been staged. + if selfCopy { + srcBody = &closeOnEOFReader{r: srcBody, c: f} + } + putObjectInput := s3response.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, @@ -6488,23 +6503,29 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput putObjectInput.Tagging = input.Tagging } - res, err := p.PutObject(withCtxNoSlot(ctx), putObjectInput) - if err != nil { - return s3response.CopyObjectOutput{}, err - } - - // copy the source object tagging after the destination object - // creation, if tagging directive is "COPY" + // read the source tagging before the destination is written, as a + // self copy replaces the source object's metadata + var srcTagging []byte + var hasSrcTagging bool if input.TaggingDirective == types.TaggingDirectiveCopy { tagging, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, tagHdr) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return s3response.CopyObjectOutput{}, fmt.Errorf("get source object tagging: %w", err) } - if err == nil { - err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, tagging) - if err != nil { - return s3response.CopyObjectOutput{}, fmt.Errorf("set destination object tagging: %w", err) - } + srcTagging, hasSrcTagging = tagging, err == nil + } + + res, err := p.PutObject(withCtxNoSlot(ctx), putObjectInput) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + // the source tagging is stored after the destination object creation, + // if tagging directive is "COPY" + if hasSrcTagging { + err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, srcTagging) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("set destination object tagging: %w", err) } } diff --git a/backend/posix/posix_io_helpers.go b/backend/posix/posix_io_helpers.go index 62eed318..76d8fd70 100644 --- a/backend/posix/posix_io_helpers.go +++ b/backend/posix/posix_io_helpers.go @@ -16,6 +16,7 @@ package posix import ( "bufio" + "errors" "io" "log" "sync" @@ -45,6 +46,24 @@ func (b *bufferedReadCloser) Close() error { return b.c.Close() } +// closeOnEOFReader closes c once r is drained, for readers whose source has +// to be released before the caller is done with the reader. +type closeOnEOFReader struct { + r io.Reader + c io.Closer + closed bool +} + +func (e *closeOnEOFReader) Read(p []byte) (int, error) { + n, err := e.r.Read(p) + if errors.Is(err, io.EOF) && !e.closed { + e.closed = true + e.c.Close() + } + + return n, err +} + var odirectUnsupportedWarnByOp sync.Map func warnODirectUnsupportedOnce(op string, err error) { diff --git a/backend/posix/posix_io_helpers_test.go b/backend/posix/posix_io_helpers_test.go new file mode 100644 index 00000000..5beb962f --- /dev/null +++ b/backend/posix/posix_io_helpers_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "bytes" + "io" + "strings" + "testing" +) + +type countingCloser struct { + count int +} + +func (c *countingCloser) Close() error { + c.count++ + return nil +} + +func TestCloseOnEOFReader(t *testing.T) { + c := &countingCloser{} + r := &closeOnEOFReader{r: strings.NewReader("hello"), c: c} + + buf := make([]byte, 2) + n, err := r.Read(buf) + if err != nil { + t.Fatalf("read: %v", err) + } + if n != 2 { + t.Fatalf("expected 2 bytes, got %v", n) + } + if c.count != 0 { + t.Fatalf("expected the source to stay open before EOF, closed %v times", c.count) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read all: %v", err) + } + if !bytes.Equal(data, []byte("llo")) { + t.Fatalf("expected the remaining data to be llo, got %s", data) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once at EOF, closed %v times", c.count) + } + + // reads past EOF don't close the source again + if _, err := r.Read(buf); err != io.EOF { + t.Fatalf("expected io.EOF, got %v", err) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once, closed %v times", c.count) + } +} + +func TestCloseOnEOFReaderEmptySource(t *testing.T) { + c := &countingCloser{} + r := &closeOnEOFReader{r: strings.NewReader(""), c: c} + + if _, err := io.ReadAll(r); err != nil { + t.Fatalf("read all: %v", err) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once at EOF, closed %v times", c.count) + } +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index cbed13f6..6f5d8c79 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -1958,6 +1958,8 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_CopyObject_non_existing_version_id) ts.Run(Versioning_CopyObject_from_an_object_version) ts.Run(Versioning_CopyObject_from_a_delete_marker) + ts.Run(Versioning_CopyObject_to_itself) + ts.Run(Versioning_CopyObject_to_itself_from_the_current_version) if !ts.conf.windowsTests { ts.Run(Versioning_CopyObject_special_chars) } @@ -2056,6 +2058,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_WORM_delete_marker_locked_object_compliance_retention) ts.Run(Versioning_WORM_PutObject_overwrite_locked_object) ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object) + ts.Run(Versioning_WORM_CopyObject_to_itself_locked_object) ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object) if !ts.conf.windowsTests { ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention) @@ -3497,6 +3500,8 @@ func GetIntTests() IntTests { "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, "Versioning_CopyObject_from_a_delete_marker": Versioning_CopyObject_from_a_delete_marker, + "Versioning_CopyObject_to_itself": Versioning_CopyObject_to_itself, + "Versioning_CopyObject_to_itself_from_the_current_version": Versioning_CopyObject_to_itself_from_the_current_version, "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, "Versioning_HeadObject_non_existing_object_version": Versioning_HeadObject_non_existing_object_version, @@ -3578,6 +3583,7 @@ func GetIntTests() IntTests { "Versioning_WORM_delete_marker_locked_object_compliance_retention": Versioning_WORM_delete_marker_locked_object_compliance_retention, "Versioning_WORM_PutObject_overwrite_locked_object": Versioning_WORM_PutObject_overwrite_locked_object, "Versioning_WORM_CopyObject_overwrite_locked_object": Versioning_WORM_CopyObject_overwrite_locked_object, + "Versioning_WORM_CopyObject_to_itself_locked_object": Versioning_WORM_CopyObject_to_itself_locked_object, "Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object": Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object, "Versioning_WORM_remove_delete_marker_under_bucket_default_retention": Versioning_WORM_remove_delete_marker_under_bucket_default_retention, "Versioning_WORM_trailing_slash_counterpart": Versioning_WORM_trailing_slash_counterpart, diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 45f2491a..8e71ecd5 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -742,6 +742,182 @@ func Versioning_CopyObject_from_a_delete_marker(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +// A copy of an object onto itself in a versioned bucket is an ordinary +// write: it creates a new version and leaves the one it replaces untouched. +// Without a metadata directive there is nothing to replace, so it's rejected. +func Versioning_CopyObject_to_itself(s *S3Conf) error { + testName := "Versioning_CopyObject_to_itself" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // directory objects always carry the directory content-type + srcContentType, dstContentType := "text/plain", "application/json" + if strings.HasSuffix(obj, "/") { + srcContentType, dstContentType = directoryContentType, directoryContentType + } + + srcMeta := map[string]string{"key": "value"} + r, err := putObjectWithData(objDataLen(obj, 1234), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: getPtr("text/plain"), + Metadata: srcMeta, + }, s3client) + if err != nil { + return err + } + + srcVersionId := getString(r.res.VersionId) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopyDest)); err != nil { + return err + } + + dstMeta := map[string]string{"new-key": "new-value"} + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + MetadataDirective: types.MetadataDirectiveReplace, + ContentType: getPtr("application/json"), + Metadata: dstMeta, + }) + cancel() + if err != nil { + return err + } + + dstVersionId := getString(out.VersionId) + if dstVersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if dstVersionId == srcVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", dstVersionId) + } + + // the replaced version keeps its own metadata + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &srcVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.ContentType) != srcContentType { + return fmt.Errorf("expected the source version content-type to be %v, instead got %v", + srcContentType, getString(res.ContentType)) + } + if !areMapsSame(res.Metadata, srcMeta) { + return fmt.Errorf("expected the source version metadata to be %v, instead got %v", + srcMeta, res.Metadata) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != dstVersionId { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + dstVersionId, getString(res.VersionId)) + } + if getString(res.ContentType) != dstContentType { + return fmt.Errorf("expected the new version content-type to be %v, instead got %v", + dstContentType, getString(res.ContentType)) + } + if !areMapsSame(res.Metadata, dstMeta) { + return fmt.Errorf("expected the new version metadata to be %v, instead got %v", + dstMeta, res.Metadata) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + vRes, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if len(vRes.Versions) != 2 { + return fmt.Errorf("expected 2 object versions, instead got %v", len(vRes.Versions)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +// Naming the current version in the copy source makes a copy onto the same +// key a regular copy, so it is accepted even without a metadata directive. +func Versioning_CopyObject_to_itself_from_the_current_version(s *S3Conf) error { + testName := "Versioning_CopyObject_to_itself_from_the_current_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + + srcVersionId := getString(versions[0].VersionId) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, obj, srcVersionId)), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != srcVersionId { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + srcVersionId, getString(out.CopySourceVersionId)) + } + if getString(out.VersionId) == srcVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &srcVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != srcVersionId { + return fmt.Errorf("expected the source version to remain, instead got %v", + getString(res.VersionId)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_CopyObject_special_chars(s *S3Conf) error { testName := "Versioning_CopyObject_special_chars" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4250,6 +4426,105 @@ func Versioning_WORM_CopyObject_overwrite_locked_object(s *S3Conf) error { }, withLock()) } +// A copy of a locked object onto itself creates a new version, leaving the +// locked one and its legal hold in place. +func Versioning_WORM_CopyObject_to_itself_locked_object(s *S3Conf) error { + testName := "Versioning_WORM_CopyObject_to_itself_locked_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + + v := versions[0] + v.IsLatest = getPtr(false) + lockedVersionId := getString(v.VersionId) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + MetadataDirective: types.MetadataDirectiveReplace, + ContentType: getPtr("application/json"), + }) + cancel() + if err != nil { + return err + } + + if getString(copyResult.VersionId) == lockedVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", + getString(copyResult.VersionId)) + } + + version := types.ObjectVersion{ + ETag: copyResult.CopyObjectResult.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: v.Size, + VersionId: copyResult.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: copyResult.CopyObjectResult.ChecksumType, + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions([]types.ObjectVersion{version, v}, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", + []types.ObjectVersion{version, v}, out.Versions) + } + + // the legal hold stays on the version it was set on + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lhRes, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &lockedVersionId, + }) + cancel() + if err != nil { + return err + } + + if lhRes.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %v, instead got %v", + types.ObjectLockLegalHoldStatusOn, lhRes.LegalHold.Status) + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: lockedVersionId, + removeOnlyLeglHold: true, + }, + }) + }) + }, withLock()) +} + func Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object(s *S3Conf) error { testName := "Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { From e1d72907e464ba8cec0af0d3be3cf1fafbd7d34b Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 21 Sep 2026 20:34:47 +0400 Subject: [PATCH 6/6] fix: handle the null version id correctly in posix versioning Fixes #2165 `GetObject` marked an object carrying no `versionIdKey` attribute as the null version and then overwrote that with the empty attribute value, and `HeadObject` never marked it at all, so neither reported a version id for an object put into a versioning-suspended bucket. Both now resolve it through a shared `liveObjVersionId`, which reports `null` only when the bucket has versioning configured so that a bucket that was never versioned keeps reporting no version id at all. `HeadObject` also resolved an explicit `?versionId=null` into the versioning directory twice, once for the missing attribute and once for the mismatch that followed from it, and answered `404` for a version that `GetObject` served. It now treats the missing attribute as the null version the way `GetObject` does. `latestObjVersion` took the last version directory entry as the newest, but `os.ReadDir` sorts by name and `null` sorts after every `ulid`. Deleting the current version of an object whose history also held a null version therefore restored `null` rather than the version created immediately before the deleted one. The `ulid` entries keep their name order, which is creation order, and the null version is placed by its modification time, the same rule `fileToObjVersions` already applies. --- backend/posix/posix.go | 99 +++++++--- tests/integration/group-tests.go | 10 + tests/integration/versioning.go | 329 +++++++++++++++++++++++++++++++ 3 files changed, 414 insertions(+), 24 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 165a67f9..f1b689fd 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1226,6 +1226,30 @@ func (p *Posix) isBucketVersioningSuspended(s types.BucketVersioningStatus) bool return s == types.BucketVersioningStatusSuspended } +// liveObjVersionId returns the version id to report for the current version +// of bucket/object. An object with no versionId attribute is the null +// version, but only a bucket with versioning configured has versions at all: +// in a bucket that was never versioned the object has no version id. +func (p *Posix) liveObjVersionId(ctx context.Context, bucket, object string) (string, error) { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if err == nil { + return string(vId), nil + } + if !errors.Is(err, meta.ErrNoSuchKey) { + return "", fmt.Errorf("get obj versionId: %w", err) + } + + status, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return "", fmt.Errorf("get bucket versioning status: %w", err) + } + if status == "" { + return "", nil + } + + return nullVersionId, nil +} + // Generates the object version path in the versioning directory func (p *Posix) genObjVersionPath(bucket, key string) string { return filepath.Join(p.versioningDir, bucket, genObjVersionKey(key)) @@ -5155,9 +5179,9 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }, nil } - srcObjVersion, err := latestObjVersion(ents).Info() + srcObjVersion, err := latestObjVersion(ents) if err != nil { - return nil, fmt.Errorf("get file info: %w", err) + return nil, fmt.Errorf("get latest obj version: %w", err) } srcVersionId := srcObjVersion.Name() sf, err := os.Open(filepath.Join(versionPath, srcVersionId)) @@ -5326,11 +5350,45 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } -// latestObjVersion returns the entry of the version, among the version -// directory entries, that becomes the latest one when the latest version -// of the object is deleted -func latestObjVersion(ents []fs.DirEntry) fs.DirEntry { - return ents[len(ents)-1] +// latestObjVersion returns the version, among the version directory entries, +// that becomes the latest one when the latest version of the object is +// deleted. The entries are named after the version id and os.ReadDir sorts +// them by name, which puts the ulid version ids in creation order. The null +// version id doesn't sort with them, so it's placed by its modification time, +// which is the time the version was created. +func latestObjVersion(ents []fs.DirEntry) (fs.FileInfo, error) { + var latest, nullEnt fs.DirEntry + for _, ent := range ents { + if ent.Name() == nullVersionId { + nullEnt = ent + continue + } + latest = ent + } + + switch { + case latest == nil && nullEnt == nil: + return nil, fs.ErrNotExist + case latest == nil: + return nullEnt.Info() + case nullEnt == nil: + return latest.Info() + } + + latestInfo, err := latest.Info() + if err != nil { + return nil, err + } + nullInfo, err := nullEnt.Info() + if err != nil { + return nil, err + } + + if nullInfo.ModTime().After(latestInfo.ModTime()) { + return nullInfo, nil + } + + return latestInfo, nil } // deleteDirObjectLatestVersion removes the latest version of the directory @@ -5371,12 +5429,11 @@ func (p *Posix) deleteDirObjectLatestVersion(bucket, key string) error { return nil } - srcVersion := latestObjVersion(ents) - srcVersionId := srcVersion.Name() - srcInfo, err := srcVersion.Info() + srcInfo, err := latestObjVersion(ents) if err != nil { - return fmt.Errorf("get file info: %w", err) + return fmt.Errorf("get latest obj version: %w", err) } + srcVersionId := srcInfo.Name() // replace the attributes in place, so that the directory keeps its etag for _, attr := range dirObjectAttrs { @@ -5623,14 +5680,10 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge // If versioning is configured get the object versionId if p.versioningEnabled() && versionId == "" { - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, meta.ErrNoSuchKey) { - versionId = nullVersionId - } else if err != nil { + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { return nil, err } - - versionId = string(vId) } if fid.IsDir() { @@ -5879,8 +5932,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, fmt.Errorf("get obj versionId: %w", err) } if errors.Is(err, meta.ErrNoSuchKey) { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + // an object without a versionId attribute is the null version + vId = []byte(nullVersionId) } if string(vId) != versionId { @@ -5944,12 +5997,10 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } if p.versioningEnabled() && versionId == "" { - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get object versionId: %v", err) + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { + return nil, err } - - versionId = string(vId) } objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fi) diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 6f5d8c79..f7311936 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -1972,6 +1972,8 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_HeadObject_success) ts.Run(Versioning_HeadObject_dir_object_versions) ts.Run(Versioning_HeadObject_without_versionId) + ts.Run(Versioning_HeadObject_null_version_without_versionId) + ts.Run(Versioning_HeadObject_null_versionId_obj) ts.Run(Versioning_HeadObject_delete_marker) // GetObject action ts.Run(Versioning_GetObject_invalid_versionId) @@ -1980,6 +1982,8 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_GetObject_delete_marker_without_versionId) ts.Run(Versioning_GetObject_delete_marker) ts.Run(Versioning_GetObject_null_versionId_obj) + ts.Run(Versioning_GetObject_null_version_without_versionId) + ts.Run(Versioning_unversioned_bucket_omits_versionId) // object tagging actions ts.Run(Versioning_PutObjectTagging_invalid_versionId) ts.Run(Versioning_PutObjectTagging_non_existing_object_version) @@ -1998,6 +2002,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_DeleteObject_invalid_versionId) ts.Run(Versioning_DeleteObject_delete_object_version) ts.Run(Versioning_DeleteObject_dir_object_latest_version) + ts.Run(Versioning_DeleteObject_latest_version_with_null_version) ts.Run(Versioning_DeleteObject_non_existing_object) ts.Run(Versioning_DeleteObject_implicit_dir) ts.Run(Versioning_DeleteObject_trailing_slash_counterpart) @@ -3509,12 +3514,16 @@ func GetIntTests() IntTests { "Versioning_HeadObject_success": Versioning_HeadObject_success, "Versioning_HeadObject_dir_object_versions": Versioning_HeadObject_dir_object_versions, "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, + "Versioning_HeadObject_null_versionId_obj": Versioning_HeadObject_null_versionId_obj, + "Versioning_HeadObject_null_version_without_versionId": Versioning_HeadObject_null_version_without_versionId, "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, "Versioning_GetObject_non_existing_object_version": Versioning_GetObject_non_existing_object_version, "Versioning_GetObject_success": Versioning_GetObject_success, "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, + "Versioning_unversioned_bucket_omits_versionId": Versioning_unversioned_bucket_omits_versionId, + "Versioning_GetObject_null_version_without_versionId": Versioning_GetObject_null_version_without_versionId, "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, "Versioning_PutObjectTagging_invalid_versionId": Versioning_PutObjectTagging_invalid_versionId, "Versioning_PutObjectTagging_non_existing_object_version": Versioning_PutObjectTagging_non_existing_object_version, @@ -3530,6 +3539,7 @@ func GetIntTests() IntTests { "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, "Versioning_DeleteObject_invalid_versionId": Versioning_DeleteObject_invalid_versionId, "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_latest_version_with_null_version": Versioning_DeleteObject_latest_version_with_null_version, "Versioning_DeleteObject_dir_object_latest_version": Versioning_DeleteObject_dir_object_latest_version, "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, "Versioning_DeleteObject_implicit_dir": Versioning_DeleteObject_implicit_dir, diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 8e71ecd5..3e32d2a7 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -1269,6 +1269,105 @@ func Versioning_HeadObject_without_versionId(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +// Versioning_HeadObject_null_version_without_versionId heads an object put +// into a versioning-suspended bucket, without naming a version id. +func Versioning_HeadObject_null_versionId_obj(s *S3Conf) error { + testName := "Versioning_HeadObject_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + keys, dataLen := []string{"my-obj", "my-dir/"}, int64(321) + // the objects are put before versioning is enabled + etags := make(map[string]string, len(keys)) + err := forEachKey(keys, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, dataLen), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + etags[obj] = getString(out.res.ETag) + return nil + }) + if err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + return forEachKey(keys, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &nullVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != objDataLen(obj, dataLen) { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + objDataLen(obj, dataLen), res.ContentLength) + } + if getString(res.ETag) != etags[obj] { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + etags[obj], getString(res.ETag)) + } + + return nil + }) + }) +} + +func Versioning_HeadObject_null_version_without_versionId(s *S3Conf) error { + testName := "Versioning_HeadObject_null_version_without_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dataLen := objDataLen(obj, 765) + out, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != dataLen { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + dataLen, res.ContentLength) + } + if getString(res.ETag) != getString(out.res.ETag) { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + getString(out.res.ETag), getString(res.ETag)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusSuspended)) +} + func Versioning_HeadObject_delete_marker(s *S3Conf) error { testName := "Versioning_HeadObject_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -1611,6 +1710,120 @@ func Versioning_GetObject_null_versionId_obj(s *S3Conf) error { }) } +// Versioning_GetObject_null_version_without_versionId reads an object put +// into a versioning-suspended bucket, without naming a version id. The object +// is the null version and the response has to report it as null rather than +// leave the version id out. +func Versioning_GetObject_null_version_without_versionId(s *S3Conf) error { + testName := "Versioning_GetObject_null_version_without_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dataLen := objDataLen(obj, 543) + out, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + // a put into a versioning-suspended bucket creates the null + // version and reports no version id + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != dataLen { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + dataLen, res.ContentLength) + } + if getString(res.ETag) != getString(out.res.ETag) { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + getString(out.res.ETag), getString(res.ETag)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusSuspended)) +} + +// Versioning_unversioned_bucket_omits_versionId reads an object in a bucket +// that never had versioning configured. Such a bucket has no versions at all, +// so neither GetObject nor HeadObject reports a version id, not even the null +// one, even though the gateway itself runs with versioning enabled. +func Versioning_unversioned_bucket_omits_versionId(s *S3Conf) error { + testName := "Versioning_unversioned_bucket_omits_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + vRes, err := s3client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + // guard the premise of the test + if vRes.Status != "" { + return fmt.Errorf("expected the bucket versioning to be unconfigured, instead got %v", + vRes.Status) + } + + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 432), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + gRes, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if gRes.VersionId != nil { + return fmt.Errorf("expected GetObject to omit the versionId, instead got %v", + *gRes.VersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + hRes, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if hRes.VersionId != nil { + return fmt.Errorf("expected HeadObject to omit the versionId, instead got %v", + *hRes.VersionId) + } + + return nil + }) + }) +} + func Versioning_GetObjectAttributes_invalid_versionId(s *S3Conf) error { testName := "Versioning_GetObjectAttributes_invalid_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -1961,6 +2174,122 @@ func Versioning_DeleteObject_dir_object_latest_version(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } +// Versioning_DeleteObject_latest_version_with_null_version deletes the current +// version of an object whose history also holds a null version, created while +// versioning was suspended. The version that becomes current has to be the one +// created right before the deleted version, not the older null version. +func Versioning_DeleteObject_latest_version_with_null_version(s *S3Conf) error { + testName := "Versioning_DeleteObject_latest_version_with_null_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // the versions are told apart by a metadata entry, as a + // directory object carries no data + put := func(marker string) (*s3.PutObjectOutput, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + return s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: map[string]string{"marker": marker}, + }) + } + + if _, err := put("v1"); err != nil { + return err + } + if _, err := put("v2"); err != nil { + return err + } + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + // the null version sits in the middle of the version history + if _, err := put("null"); err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + third, err := put("v3") + if err != nil { + return err + } + latest, err := put("v4") + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: latest.VersionId, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) != getString(latest.VersionId) { + return fmt.Errorf("expected the deleted versionId to be %v, instead got %v", + getString(latest.VersionId), getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + expectedMeta := map[string]string{"marker": "v3"} + if !areMapsSame(res.Metadata, expectedMeta) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + expectedMeta, res.Metadata) + } + if getString(res.VersionId) != getString(third.VersionId) { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + getString(third.VersionId), getString(res.VersionId)) + } + + // the null version has to be left untouched by the delete + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + versions, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + nullFound := false + for _, v := range versions.Versions { + if getString(v.VersionId) == nullVersionId { + nullFound = true + if v.IsLatest != nil && *v.IsLatest { + return fmt.Errorf("expected the null version not to be the latest") + } + } + } + if !nullFound { + return fmt.Errorf("expected the null version to be kept in %v", + versions.Versions) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { testName := "Versioning_DeleteObject_non_existing_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {