diff --git a/auth/object_lock.go b/auth/object_lock.go index 77083cb2..ba502e50 100644 --- a/auth/object_lock.go +++ b/auth/object_lock.go @@ -495,7 +495,9 @@ func (s objectLockState) checkObject(ctx context.Context, be backend.Backend, ia checkRetention := true retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId) - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + // an object or version that doesn't exist has nothing to protect + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) || + errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) { return nil } // the object is a delete marker, if a `MethodNotAllowed` error is returned @@ -538,7 +540,8 @@ func (s objectLockState) checkObject(ctx context.Context, be backend.Backend, ia status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId) if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) || + errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) { return nil } if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { diff --git a/backend/posix/objlock.go b/backend/posix/objlock.go index 13098e37..8d507d12 100644 --- a/backend/posix/objlock.go +++ b/backend/posix/objlock.go @@ -49,6 +49,11 @@ import ( // file takes its place, which is unsafe to detect reliably on NFS due to // attribute caching. // +// The process-local slot is picked from the bucket hash and the shard, so all +// requests for one lock file take the same slot (fcntl locks don't exclude +// within a process), while the same key in different buckets usually takes +// different slots and doesn't wait on an unrelated lock. +// // The lock is held only for the commit phase (condition re-check, metadata // stores, final link/rename) — request bodies are staged to a temp file // before the lock is taken. The OS releases advisory locks automatically when @@ -131,15 +136,22 @@ func objLockShard(object string) uint8 { return sum[0] } +// objLockSlot returns the process-local slot index for the lock file of the +// shard in the bucket with bucketHash. +func objLockSlot(bucketHash [sha256.Size]byte, shard uint8) uint8 { + return bucketHash[0] ^ shard +} + // lockObjectPublish acquires the publish lock for bucket/object. It returns a // release function that must be called (typically deferred) once the new // object state is visible. All code paths that create or replace an object at // its final key must hold this lock across condition evaluation and // publication. func (p *Posix) lockObjectPublish(ctx context.Context, bucket, object string) (func(), error) { + bucketHash := sha256.Sum256([]byte(bucket)) shard := objLockShard(object) - slot := p.objLockSlots[shard] + slot := p.objLockSlots[objLockSlot(bucketHash, shard)] select { case <-ctx.Done(): return nil, ctx.Err() @@ -154,7 +166,7 @@ func (p *Posix) lockObjectPublish(ctx context.Context, bucket, object string) (f return releaseLocal, nil } - f, err := p.openObjLockFile(bucket, shard) + f, err := p.openObjLockFile(bucketHash, shard) if err != nil { releaseLocal() return nil, err @@ -188,9 +200,8 @@ func newObjLockSlots() [objLockShards]chan struct{} { } // openObjLockFile opens (creating as needed) the lock file for the shard in -// the given bucket. -func (p *Posix) openObjLockFile(bucket string, shard uint8) (*os.File, error) { - bucketHash := sha256.Sum256([]byte(bucket)) +// the bucket with bucketHash. +func (p *Posix) openObjLockFile(bucketHash [sha256.Size]byte, shard uint8) (*os.File, error) { lockDir := filepath.Join(p.rootdir, objLockDir, fmt.Sprintf("%x", bucketHash)) name := filepath.Join(lockDir, fmt.Sprintf("%02x", shard)) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 803e3977..f1b689fd 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -631,21 +631,20 @@ func (p *Posix) doesBucketExist(bucket string) error { return nil } -func (p *Posix) doesBucketAndObjectExist(bucket, object string) error { +// doesBucketAndObjectExist checks that bucket exists and, when versionId is +// empty, that object has a current version. A specific version is looked up +// once versionId is validated. +func (p *Posix) doesBucketAndObjectExist(bucket, object, versionId string) error { err := p.doesBucketExist(bucket) if err != nil { return err } - - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil { - return fmt.Errorf("stat object: %w", err) + if versionId != "" { + return nil } - return nil + _, _, err = p.objVersionAttrPath(bucket, object, "") + return err } func (p *Posix) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { @@ -1227,6 +1226,30 @@ func (p *Posix) isBucketVersioningSuspended(s types.BucketVersioningStatus) bool return s == types.BucketVersioningStatusSuspended } +// liveObjVersionId returns the version id to report for the current version +// of bucket/object. An object with no versionId attribute is the null +// version, but only a bucket with versioning configured has versions at all: +// in a bucket that was never versioned the object has no version id. +func (p *Posix) liveObjVersionId(ctx context.Context, bucket, object string) (string, error) { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if err == nil { + return string(vId), nil + } + if !errors.Is(err, meta.ErrNoSuchKey) { + return "", fmt.Errorf("get obj versionId: %w", err) + } + + status, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return "", fmt.Errorf("get bucket versioning status: %w", err) + } + if status == "" { + return "", nil + } + + return nullVersionId, nil +} + // Generates the object version path in the versioning directory func (p *Posix) genObjVersionPath(bucket, key string) string { return filepath.Join(p.versioningDir, bucket, genObjVersionKey(key)) @@ -1264,6 +1287,145 @@ func isRemovableAttr(attr string) bool { } } +// dirObjectAttrs are the attributes that make up the state of a directory +// object. The directory is kept across versions and is also the parent of +// other objects, so it may carry attributes that don't belong to the object: +// only these are copied into, restored from and cleared between versions. +// An attribute stored on objects must also be listed here. +var dirObjectAttrs = []string{ + etagkey, + checksumsKey, + contentTypeHdr, + contentEncHdr, + contentLangHdr, + contentDispHdr, + cacheCtrlHdr, + expiresHdr, + websiteRedirectHdr, + metadataHdr, + tagHdr, + objectLegalHoldKey, + objectRetentionKey, + versionIdKey, + deleteMarkerKey, +} + +// isDirObject reports whether the directory at bucket/key is a directory +// object: only directories created with a put carry an etag +func (p *Posix) isDirObject(bucket, key string) (bool, error) { + _, err := p.meta.RetrieveAttribute(nil, bucket, key, etagkey) + if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get dir etag: %w", err) + } + return true, nil +} + +// isLiveObject reports whether fi, the entry at the path of key, is the +// current version of key. A key and the same key with a trailing slash +// share one path and one set of attributes: a file there is the object of +// the key without the slash, a directory object the object of the key with +// it. +func (p *Posix) isLiveObject(fi os.FileInfo, bucket, key string) (bool, error) { + if fi.IsDir() != strings.HasSuffix(key, "/") { + return false, nil + } + if !fi.IsDir() { + return true, nil + } + return p.isDirObject(bucket, key) +} + +// statLiveObject returns the file info of the current version of key. An +// error matching fs.ErrNotExist is returned when key has no current +// version, including when the entry at its path is another key's object. +func (p *Posix) statLiveObject(bucket, key string) (os.FileInfo, error) { + fi, err := os.Stat(p.ObjectPath(bucket, key)) + if isErrNotDir(err) { + return nil, fs.ErrNotExist + } + if err != nil { + return nil, err + } + isObj, err := p.isLiveObject(fi, bucket, key) + if err != nil { + return nil, err + } + if !isObj { + return nil, fs.ErrNotExist + } + return fi, nil +} + +// objVersionAttrPath returns the bucket and object that the attributes of +// the version versionId of key are stored at: key itself for its current +// version, an entry of the versioning directory for other versions. An +// empty versionId selects the current version, and NoSuchKey is returned if +// key has none. +func (p *Posix) objVersionAttrPath(bucket, key, versionId string) (string, string, error) { + _, err := p.statLiveObject(bucket, key) + if isErrNameTooLong(err) { + return "", "", s3err.GetKeyTooLongErr(int64(len(key)), 1024) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", "", fmt.Errorf("stat object: %w", err) + } + isLive := err == nil + + if versionId == "" { + if !isLive { + return "", "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + return bucket, key, nil + } + + if isLive { + vId, err := p.meta.RetrieveAttribute(nil, bucket, key, versionIdKey) + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } else if err != nil { + return "", "", fmt.Errorf("get obj versionId: %w", err) + } + if string(vId) == versionId { + return bucket, key, nil + } + } + + return filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), versionId), nil +} + +// clearDirObjectAttrs removes the directory object attributes, including +// legacy metadata attributes, from the directory at bucket/key. The etag is +// kept: it marks the directory as an object, so a failure before the new +// attributes are stored leaves the object and its versions visible. +func (p *Posix) clearDirObjectAttrs(bucket, key string) error { + attrs, err := p.meta.ListAttributes(bucket, key) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("list object attributes: %w", err) + } + for _, attr := range attrs { + if isValidMeta(attr) { + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + } + + for _, attr := range dirObjectAttrs { + if attr == etagkey { + continue + } + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + return nil +} + // Creates a new copy(version) of an object in the versioning directory func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account, removeAttributes bool) (versionPath string, err error) { sf, err := os.Open(p.ObjectPath(bucket, key)) @@ -1283,9 +1445,19 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun versionId = nullVersionId } - attrs, err := p.meta.ListAttributes(bucket, key) - if err != nil { - return versionPath, fmt.Errorf("load object attributes: %w", err) + // a directory object version is an empty file carrying the + // directory object attributes + isDir := strings.HasSuffix(key, "/") + attrs := dirObjectAttrs + if isDir { + size = 0 + // store legacy metadata attributes as the metadata attribute + p.loadObjectMetadata(sf, bucket, key) + } else { + attrs, err = p.meta.ListAttributes(bucket, key) + if err != nil { + return versionPath, fmt.Errorf("load object attributes: %w", err) + } } versionBucketPath := filepath.Join(p.versioningDir, bucket) @@ -1303,10 +1475,12 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun originalMTime = srcInfo.ModTime() } - // Prioritize copy_file_range for internal file-to-file version copies. - _, err = io.Copy(f.File(), sf) - if err != nil { - return versionPath, err + if !isDir { + // Prioritize copy_file_range for internal file-to-file version copies. + _, err = io.Copy(f.File(), sf) + if err != nil { + return versionPath, err + } } // Restore original mtime after copy @@ -1325,6 +1499,9 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun // Copy the object attributes(metadata) for _, attr := range attrs { data, err := p.meta.RetrieveAttribute(sf, bucket, key, attr) + if isDir && errors.Is(err, meta.ErrNoSuchKey) { + continue + } if err != nil { return versionPath, fmt.Errorf("list %v attribute: %w", attr, err) } @@ -1465,6 +1642,24 @@ func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { return true, nil } +// checkCopySourceDeleteMarker rejects a copy whose source resolves to a +// delete marker: the key has no current version when the marker is the +// latest, and a marker named by version id holds no data to copy. +func (p *Posix) checkCopySourceDeleteMarker(bucket, object, versionId string) error { + isDel, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return err + } + if !isDel { + return nil + } + if versionId != "" { + return s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker) + } + + return s3err.GetAPIError(s3err.ErrNoSuchKey) +} + // Converts the file to object version. Finds all the object versions, // delete markers from the versioning directory and returns func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { @@ -1479,53 +1674,21 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { Truncated: true, }, nil } + // a directory is listed under the key with the trailing slash + key := path if d.IsDir() { - // directory object only happens if directory empty - // check to see if this is a directory object by checking etag - etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) - if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { - return nil, backend.ErrSkipObj - } - if err != nil { - return nil, fmt.Errorf("get etag: %w", err) - } - etag := string(etagBytes) - - fi, err := d.Info() - if errors.Is(err, fs.ErrNotExist) { - return nil, backend.ErrSkipObj - } - if err != nil { - return nil, fmt.Errorf("get fileinfo: %w", err) - } - - key := path + "/" - // Directory objects don't contain data - size := int64(0) - versionId := "null" - - objects = append(objects, s3response.ObjectVersion{ - ETag: &etag, - Key: &key, - LastModified: backend.GetTimePtr(fi.ModTime()), - IsLatest: getBoolPtr(true), - Size: &size, - VersionId: &versionId, - StorageClass: types.ObjectVersionStorageClassStandard, - }) - - return &backend.ObjVersionFuncResult{ - ObjectVersions: objects, - DelMarkers: delMarkers, - Truncated: availableObjCount == 1, - }, nil + key = path + "/" } - // file object, get object info and fill out object data + // get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { return nil, backend.ErrSkipObj } + if d.IsDir() && errors.Is(err, meta.ErrNoSuchKey) { + // a directory is listed only if it's a directory object + return nil, backend.ErrSkipObj + } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get etag: %w", err) } @@ -1552,6 +1715,10 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } size := fi.Size() + if d.IsDir() { + // directory objects don't contain data + size = 0 + } isDel, err := p.isObjDeleteMarker(bucket, path) if err != nil { @@ -1563,7 +1730,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { IsLatest: getBoolPtr(true), VersionId: &versionId, LastModified: backend.GetTimePtr(fi.ModTime()), - Key: &path, + Key: &key, }) } else { // Retrieve checksum @@ -1574,7 +1741,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { objects = append(objects, s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(fi.ModTime()), Size: &size, VersionId: &versionId, @@ -1604,7 +1771,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } // List all the versions of the object in the versioning directory - versionPath := p.genObjVersionPath(bucket, path) + versionPath := p.genObjVersionPath(bucket, key) dirEnts, err := os.ReadDir(versionPath) if errors.Is(err, fs.ErrNotExist) { return &backend.ObjVersionFuncResult{ @@ -1642,7 +1809,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { nullObjDelMarker = &types.DeleteMarkerEntry{ VersionId: backend.GetPtrFromString("null"), LastModified: backend.GetTimePtr(nf.ModTime()), - Key: &path, + Key: &key, IsLatest: getBoolPtr(false), } } else { @@ -1665,7 +1832,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { nullVersionIdObj = &s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(nf.ModTime()), Size: &size, VersionId: backend.GetPtrFromString("null"), @@ -1775,7 +1942,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { delMarkers = append(delMarkers, types.DeleteMarkerEntry{ VersionId: &versionId, LastModified: backend.GetTimePtr(f.ModTime()), - Key: &path, + Key: &key, IsLatest: getBoolPtr(false), }) } else { @@ -1786,7 +1953,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } objects = append(objects, s3response.ObjectVersion{ ETag: &etag, - Key: &path, + Key: &key, LastModified: backend.GetTimePtr(f.ModTime()), Size: &size, VersionId: &versionId, @@ -1900,7 +2067,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create // set object tagging if tags != nil { - err := p.PutObjectTagging(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", tags) + err := p.storeObjectTags(bucket, filepath.Join(objdir, uploadID), tags) if err != nil { // cleanup object if returning error os.RemoveAll(filepath.Join(tmppath, uploadID)) @@ -1931,7 +2098,10 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create // set object legal hold if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", true) + err := p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectLegalHoldKey, []byte{1}) + } if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) @@ -1958,7 +2128,10 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("parse object lock retention: %w", err) } - err = p.PutObjectRetention(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", retParsed) + err = p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectRetentionKey, retParsed) + } if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) @@ -2251,7 +2424,9 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C }, "", nil } // Directory is gone: the concurrent call already completed and cleaned up. - if _, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil { + // A directory at the object path is the object of the key with a + // trailing slash, not the completed upload. + if fi, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil && !fi.IsDir() { etag := multipartClaimToken if p.dataIntegrityEtag { etagBytes, etagErr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) @@ -2321,13 +2496,17 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C defer os.Rename(uploadIDInProgress, uploadIDDir) defer p.meta.RenameObject(bucket, newMetaObj, oldMetaObj) - // Fast-fail precondition check before the parts are assembled. This is - // only advisory: the authoritative check is repeated while holding the - // object publish lock just before the final link. + // Fast-fail precondition and directory checks before the parts are + // assembled. These are only advisory: the authoritative checks are + // repeated while holding the object publish lock just before the final + // link. err = p.checkPutPreconditions(bucket, object, input.IfMatch, input.IfNoneMatch) if err != nil { return res, "", err } + if d, err := os.Stat(p.ObjectPath(bucket, object)); err == nil && d.IsDir() { + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, activeUploadName)) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -2655,9 +2834,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C vEnabled := p.isBucketVersioningEnabled(vStatus) d, err := os.Stat(objname) + if err == nil && d.IsDir() { + // the directory is the object of the key with a trailing slash, or + // the parent of other objects: its attributes are not this object's + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } // if the versioning is enabled first create the file object version - if p.versioningEnabled() && vEnabled && err == nil && !d.IsDir() { + if p.versioningEnabled() && vEnabled && err == nil { _, err := p.createObjVersion(bucket, object, d.Size(), acct, false) if err != nil { return res, "", fmt.Errorf("create object version: %w", err) @@ -3889,8 +4073,26 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) if err != nil { return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) } + if strings.HasSuffix(srcObject, "/") != fi.IsDir() { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyPartResult{}, err + } + // a directory object holds no data + srcSize := fi.Size() + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyPartResult{}, err + } + if !isObj { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + } - startOffset, length, err := backend.ParseCopySourceRange(fi.Size(), *upi.CopySourceRange) + startOffset, length, err := backend.ParseCopySourceRange(srcSize, *upi.CopySourceRange) if err != nil { return s3response.CopyPartResult{}, err } @@ -4112,6 +4314,13 @@ func (p *Posix) checkPutPreconditions(bucket, object string, ifMatch, ifNoneMatc return s3err.GetAPIError(s3err.ErrNotImplemented) } + // the etag at the object path may be the one of the key with or + // without the trailing slash + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + return backend.EvaluateObjectPutPreconditions("", ifMatch, ifNoneMatch, false) + } + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) if err == nil || errors.Is(err, fs.ErrNotExist) || errors.Is(err, meta.ErrNoSuchKey) { return backend.EvaluateObjectPutPreconditions(string(etagBytes), ifMatch, ifNoneMatch, err == nil) @@ -4128,11 +4337,22 @@ func (p *Posix) snapshotObjVersion(bucket, key string, vStatus types.BucketVersi return nil } + isDir := strings.HasSuffix(key, "/") d, err := os.Stat(p.ObjectPath(bucket, key)) - if err != nil || d.IsDir() { + if err != nil || d.IsDir() != isDir { // nothing to snapshot return nil } + if isDir { + isObj, err := p.isDirObject(bucket, key) + if err != nil { + return err + } + if !isObj { + // nothing to snapshot + return nil + } + } var isVersionIdMissing bool if p.isBucketVersioningSuspended(vStatus) { @@ -4268,6 +4488,37 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, err } + expectedSum := getEmptyChecksumValue(checksumAlgorithm) + if checksumValue != "" && expectedSum != checksumValue { + return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + + // reject object lock settings the bucket doesn't support before + // the directory object is changed + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn || po.ObjectLockMode != "" { + err = p.isBucketObjectLockEnabled(*po.Bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err + } + versioned := p.versioningEnabled() && vStatus != "" + + // In a versioned bucket the directory is kept across versions: its + // current version is copied to the versioning directory and its + // object attributes are then replaced with the new version's. + err = p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + if err != nil { + return s3response.PutObjectOutput{}, err + } + err = p.mkdirAll(name, uid, gid, doChown) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -4279,21 +4530,28 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, err } + var versionID string + if versioned { + err = p.clearDirObjectAttrs(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } else { + versionID = ulid.Make().String() + } + } + err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err) } - // Set object tagging - if tags != nil { - err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) - if err != nil { - return s3response.PutObjectOutput{}, err - } - } - - expectedSum := getEmptyChecksumValue(checksumAlgorithm) - dirETag := emptyMD5 if p.dataIntegrityEtag { dirETag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksumAlgorithm)), expectedSum) @@ -4321,10 +4579,6 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - if checksumValue != "" && expectedSum != checksumValue { - return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) - } - // set empty checksum checksum := s3response.Checksum{ Type: types.ChecksumTypeFullObject, @@ -4338,9 +4592,37 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) } - // for directory object no version is created + if versionID != "" { + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + if versioned { + // The directory mtime is the version's last modified time. + // Setting it needs the directory to be owned by the gateway, + // otherwise it's left as is. + now := time.Now() + _ = os.Chtimes(name, now, now) + } + + // Set object tagging once the etag makes the directory an object + if tags != nil { + err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err + } + return s3response.PutObjectOutput{ ETag: dirETag, + VersionID: versionID, Size: &contentLength, ChecksumType: checksum.Type, ChecksumCRC32: checksum.CRC32, @@ -4614,34 +4896,9 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - // Set object legal hold - if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", true) - if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { - err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) - } - return s3response.PutObjectOutput{}, err - } - } - - // Set object retention - if po.ObjectLockMode != "" { - retention := types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), - RetainUntilDate: po.ObjectLockRetainUntilDate, - } - retParsed, err := json.Marshal(retention) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) - } - err = p.PutObjectRetention(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", retParsed) - if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { - err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) - } - return s3response.PutObjectOutput{}, err - } + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err } return s3response.PutObjectOutput{ @@ -4662,6 +4919,42 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje }, nil } +// putObjectLockSettings sets the legal hold and retention requested with +// the put on the object that was just published +func (p *Posix) putObjectLockSettings(ctx context.Context, po s3response.PutObjectInput) error { + // Set object legal hold + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + err := p.PutObjectLegalHold(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", true) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + // Set object retention + if po.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), + RetainUntilDate: po.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + return fmt.Errorf("parse object lock retention: %w", err) + } + err = p.PutObjectRetention(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + return nil +} + func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { release, err := p.acquireActionSlot(ctx) if err != nil { @@ -4721,8 +5014,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }) } - // Directory objects can't have versions - if !isDir && p.versioningEnabled() && vStatus != "" { + if p.versioningEnabled() && vStatus != "" { if getString(input.VersionId) == "" { // if the versionId is not specified, make the current version a delete marker fi, err := os.Stat(objpath) @@ -4736,6 +5028,16 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + // the entry at the object path may be the object of the key + // with or without the trailing slash + isObj, err := p.isLiveObject(fi, bucket, object) + if err != nil { + return nil, err + } + if !isObj { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } err = evalPreconditions(fi, bucket, object) if err != nil { @@ -4792,6 +5094,21 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } else { versionPath := p.genObjVersionPath(bucket, object) + // the attributes at the object path may belong to the key with + // or without the trailing slash, or to a directory that isn't an + // object + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist @@ -4803,16 +5120,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, fmt.Errorf("get obj versionId: %w", err) } if errors.Is(err, meta.ErrNoSuchKey) { - // With sidecar, ErrNoSuchKey means "attribute absent" regardless of - // whether the data file exists. If the file is absent the object - // does not exist at all → AWS returns success for DeleteObject. - // Also handle ENOTDIR: when a key such as "foo/bar" is requested - // but "foo" is a regular file (not a directory), the path cannot - // contain any object. - _, statErr := os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) { - return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil - } vId = []byte(nullVersionId) } @@ -4829,6 +5136,16 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, err } + if isDir { + err = p.deleteDirObjectLatestVersion(bucket, object) + if err != nil { + return nil, err + } + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } err = os.Remove(objpath) if err != nil && !errors.Is(err, fs.ErrNotExist) && !isErrNotDir(err) { return nil, fmt.Errorf("remove obj version: %w", err) @@ -4862,9 +5179,9 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }, nil } - srcObjVersion, err := ents[len(ents)-1].Info() + srcObjVersion, err := latestObjVersion(ents) if err != nil { - return nil, fmt.Errorf("get file info: %w", err) + return nil, fmt.Errorf("get latest obj version: %w", err) } srcVersionId := srcObjVersion.Name() sf, err := os.Open(filepath.Join(versionPath, srcVersionId)) @@ -5033,6 +5350,126 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } +// latestObjVersion returns the version, among the version directory entries, +// that becomes the latest one when the latest version of the object is +// deleted. The entries are named after the version id and os.ReadDir sorts +// them by name, which puts the ulid version ids in creation order. The null +// version id doesn't sort with them, so it's placed by its modification time, +// which is the time the version was created. +func latestObjVersion(ents []fs.DirEntry) (fs.FileInfo, error) { + var latest, nullEnt fs.DirEntry + for _, ent := range ents { + if ent.Name() == nullVersionId { + nullEnt = ent + continue + } + latest = ent + } + + switch { + case latest == nil && nullEnt == nil: + return nil, fs.ErrNotExist + case latest == nil: + return nullEnt.Info() + case nullEnt == nil: + return latest.Info() + } + + latestInfo, err := latest.Info() + if err != nil { + return nil, err + } + nullInfo, err := nullEnt.Info() + if err != nil { + return nil, err + } + + if nullInfo.ModTime().After(latestInfo.ModTime()) { + return nullInfo, nil + } + + return latestInfo, nil +} + +// deleteDirObjectLatestVersion removes the latest version of the directory +// object at bucket/key. The newest remaining version is restored onto the +// directory. With no versions left the directory stops being an object and +// is removed, unless other objects are under it. +func (p *Posix) deleteDirObjectLatestVersion(bucket, key string) error { + objpath := p.ObjectPath(bucket, key) + versionPath := p.genObjVersionPath(bucket, key) + + ents, err := os.ReadDir(versionPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + err := os.Remove(objpath) + if isErrDirNotEmpty(err) { + // the directory stays as the parent of the objects under it + err = p.clearDirObjectAttrs(bucket, key) + if err != nil { + return err + } + err = p.meta.DeleteAttribute(bucket, key, etagkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove etag attribute: %w", err) + } + return nil + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove dir object: %w", err) + } + err = p.meta.DeleteAttributes(bucket, key) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("delete object attributes: %w", err) + } + p.removeParents(bucket, key) + return nil + } + + srcInfo, err := latestObjVersion(ents) + if err != nil { + return fmt.Errorf("get latest obj version: %w", err) + } + srcVersionId := srcInfo.Name() + + // replace the attributes in place, so that the directory keeps its etag + for _, attr := range dirObjectAttrs { + data, err := p.meta.RetrieveAttribute(nil, versionPath, srcVersionId, attr) + if errors.Is(err, meta.ErrNoSuchKey) { + err = p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + continue + } + if err != nil { + return fmt.Errorf("load %v attribute: %w", attr, err) + } + + err = p.meta.StoreAttribute(nil, bucket, key, attr, data) + if err != nil { + return fmt.Errorf("store %v attribute: %w", attr, err) + } + } + + err = os.Remove(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return fmt.Errorf("remove obj version: %w", err) + } + + _ = p.meta.DeleteAttributes(versionPath, srcVersionId) + p.removeParents(filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), srcVersionId)) + + // The restored version keeps its last modified time. Setting it needs + // the directory to be owned by the gateway, otherwise it's left as is. + _ = os.Chtimes(objpath, time.Now(), srcInfo.ModTime()) + + return nil +} + func (p *Posix) removeParents(bucket, object string) { // this will remove all parent directories that were not // specifically uploaded with a put object. we detect @@ -5195,12 +5632,12 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge // in '/') have an etag attribute. Directories created incidentally on the // filesystem or as parent directories during object upload should not be // accessible via get-object. - _, derr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) - if errors.Is(derr, meta.ErrNoSuchKey) || errors.Is(derr, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err } - if derr != nil { - return nil, fmt.Errorf("get dir etag: %w", derr) + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } } @@ -5241,6 +5678,14 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge return nil, err } + // If versioning is configured get the object versionId + if p.versioningEnabled() && versionId == "" { + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { + return nil, err + } + } + if fid.IsDir() { _, _, _, err := backend.ParseObjectRange(0, *input.Range) if err != nil { @@ -5299,18 +5744,6 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge }, nil } - // If versioning is configured get the object versionId - if p.versioningEnabled() && versionId == "" { - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, meta.ErrNoSuchKey) { - versionId = nullVersionId - } else if err != nil { - return nil, err - } - - versionId = string(vId) - } - // openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent // DeleteObject can call os.Remove on this file while the GET response body // is still being streamed. On POSIX, os.Open is sufficient. @@ -5499,8 +5932,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, fmt.Errorf("get obj versionId: %w", err) } if errors.Is(err, meta.ErrNoSuchKey) { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + // an object without a versionId attribute is the null version + vId = []byte(nullVersionId) } if string(vId) != versionId { @@ -5535,12 +5968,12 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. // in '/') have an etag attribute. Directories created incidentally on the // filesystem or as parent directories during object upload should not be // accessible via head-object. - _, derr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) - if errors.Is(derr, meta.ErrNoSuchKey) || errors.Is(derr, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err } - if derr != nil { - return nil, fmt.Errorf("get dir etag: %w", derr) + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } } @@ -5564,12 +5997,10 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } if p.versioningEnabled() && versionId == "" { - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get object versionId: %v", err) + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { + return nil, err } - - versionId = string(vId) } objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fi) @@ -5878,7 +6309,24 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput if !strings.HasSuffix(srcObject, "/") && fi.IsDir() { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if fi.Size() > p.copyObjectThreshold { + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyObjectOutput{}, err + } + // a directory object holds no data + srcSize := fi.Size() + var srcBody io.Reader = f + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if !isObj { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + srcBody = strings.NewReader("") + } + if srcSize > p.copyObjectThreshold { return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(p.copyObjectThreshold) } @@ -5913,11 +6361,18 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput var chType types.ChecksumType dstObjdPath := joinPathWithTrailer(p.BucketPath(dstBucket), dstObject) - if dstObjdPath == objPath { - if input.MetadataDirective == types.MetadataDirectiveCopy { - return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) - } + // A copy of an object onto itself is rejected unless it replaces the + // object metadata. Naming a source version makes it a regular copy. + selfCopy := dstObjdPath == objPath + if selfCopy && srcVersionId == "" && + input.MetadataDirective == types.MetadataDirectiveCopy { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + // In a versioned bucket a self copy creates a new version like any other + // write, so only unversioned buckets are rewritten in place. + versioned := p.versioningEnabled() && vStatus != "" + if selfCopy && !versioned { // Delete the object metadata err = p.meta.DeleteAttribute(dstBucket, dstObject, metadataHdr) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6042,7 +6497,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput } } } else { - contentLength := fi.Size() + contentLength := srcSize checksums, err := p.retrieveChecksums(f, srcBucket, srcObject) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6055,10 +6510,18 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput checksums.Algorithm = input.ChecksumAlgorithm } + // A self copy publishes the new version over the source path, which + // on Windows can't be renamed over while the source is still open. + // PutObject reads the body before publishing, so the handle is + // released as soon as the data has been staged. + if selfCopy { + srcBody = &closeOnEOFReader{r: srcBody, c: f} + } + putObjectInput := s3response.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, - Body: f, + Body: srcBody, ContentLength: &contentLength, ChecksumAlgorithm: checksums.Algorithm, ContentType: input.ContentType, @@ -6091,23 +6554,29 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput putObjectInput.Tagging = input.Tagging } - res, err := p.PutObject(withCtxNoSlot(ctx), putObjectInput) - if err != nil { - return s3response.CopyObjectOutput{}, err - } - - // copy the source object tagging after the destination object - // creation, if tagging directive is "COPY" + // read the source tagging before the destination is written, as a + // self copy replaces the source object's metadata + var srcTagging []byte + var hasSrcTagging bool if input.TaggingDirective == types.TaggingDirectiveCopy { tagging, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, tagHdr) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return s3response.CopyObjectOutput{}, fmt.Errorf("get source object tagging: %w", err) } - if err == nil { - err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, tagging) - if err != nil { - return s3response.CopyObjectOutput{}, fmt.Errorf("set destination object tagging: %w", err) - } + srcTagging, hasSrcTagging = tagging, err == nil + } + + res, err := p.PutObject(withCtxNoSlot(ctx), putObjectInput) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + // the source tagging is stored after the destination object creation, + // if tagging directive is "COPY" + if hasSrcTagging { + err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, srcTagging) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("set destination object tagging: %w", err) } } @@ -6237,7 +6706,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } } if d.IsDir() { - // directory object only happens if directory empty + // a directory is listed only if it's a directory object // check to see if this is a directory object by checking etag etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { @@ -6248,6 +6717,12 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } etag := string(etagBytes) + // If the directory object is a delete marker, skip + isDel, _ := p.isObjDeleteMarker(bucket, path) + if isDel { + return s3response.Object{}, backend.ErrSkipObj + } + fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { return s3response.Object{}, backend.ErrSkipObj @@ -6525,36 +7000,14 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId return nil, err } - if versionId == "" { - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if isErrNameTooLong(err) { - return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) - } - if err != nil { - return nil, fmt.Errorf("stat object: %w", err) - } + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - if versionId != "" { - if !p.versioningEnabled() { - //TODO: Maybe we need to return our custom error here? - return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) - } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) - } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } err = p.ensureNotDeleteMarker(bucket, object, versionId) @@ -6612,36 +7065,14 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return err } - if versionId == "" { - _, err = os.Stat(p.ObjectPath(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if isErrNameTooLong(err) { - return s3err.GetAPIError(s3err.ErrKeyTooLong) - } - if err != nil { - return fmt.Errorf("stat object: %w", err) - } + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - if versionId != "" { - if !p.versioningEnabled() { - //TODO: Maybe we need to return our custom error here? - return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) - } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) - } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } err = p.ensureNotDeleteMarker(bucket, object, versionId) @@ -6666,12 +7097,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return nil } - b, err := json.Marshal(tags) - if err != nil { - return fmt.Errorf("marshal tags: %w", err) - } - - err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) + err = p.storeObjectTags(bucket, object, tags) if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) @@ -6685,6 +7111,16 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return nil } +// storeObjectTags stores tags as the tagging attribute of bucket/object +func (p *Posix) storeObjectTags(bucket, object string, tags map[string]string) error { + b, err := json.Marshal(tags) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + return p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) +} + func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object, versionId string) error { if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) @@ -7006,7 +7442,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err } @@ -7031,17 +7467,9 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } } @@ -7074,7 +7502,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err } @@ -7092,17 +7520,9 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } } @@ -7140,7 +7560,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err } @@ -7158,17 +7578,9 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err } } @@ -7195,7 +7607,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } - err = p.doesBucketAndObjectExist(bucket, object) + err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err } @@ -7213,17 +7625,9 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI //TODO: Maybe we need to return our custom error here? return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } - vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return nil, fmt.Errorf("get obj versionId: %w", err) - } - - if string(vId) != versionId { - bucket = filepath.Join(p.versioningDir, bucket) - object = filepath.Join(genObjVersionKey(object), versionId) + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err } } diff --git a/backend/posix/posix_conditional_put_test.go b/backend/posix/posix_conditional_put_test.go index 9e392fae..cce428c1 100644 --- a/backend/posix/posix_conditional_put_test.go +++ b/backend/posix/posix_conditional_put_test.go @@ -17,6 +17,7 @@ package posix import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "io" @@ -85,9 +86,9 @@ func TestObjectPublishLockHonorsContextWhileWaiting(t *testing.T) { bucket := "testbucket" createTestBucket(t, p, bucket) - shard := objLockShard("cancel-wait") - <-p.objLockSlots[shard] - defer func() { p.objLockSlots[shard] <- struct{}{} }() + slot := objLockSlot(sha256.Sum256([]byte(bucket)), objLockShard("cancel-wait")) + <-p.objLockSlots[slot] + defer func() { p.objLockSlots[slot] <- struct{}{} }() if _, err := os.Stat(p.ObjectPath(bucket, objLockDir)); !errors.Is(err, fs.ErrNotExist) { t.Fatalf("bucket contains publish lock directory: %v", err) } @@ -134,6 +135,25 @@ func TestObjectPublishLockHonorsCancellationAfterSlotAcquired(t *testing.T) { } } +func TestObjectPublishLockDoesNotBlockOtherBuckets(t *testing.T) { + p := newTestPosix(t, metaModes(t)["xattr"]) + + unlock, err := p.lockObjectPublish(context.Background(), "bucket-a", "my-obj") + if err != nil { + t.Fatalf("lock object publish: %v", err) + } + defer unlock() + + // the same key in another bucket has its own lock file + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + unlockOther, err := p.lockObjectPublish(ctx, "bucket-b", "my-obj") + if err != nil { + t.Fatalf("lock object publish in another bucket: %v", err) + } + unlockOther() +} + func TestObjectPublishLockModes(t *testing.T) { tests := []struct { name string diff --git a/backend/posix/posix_io_helpers.go b/backend/posix/posix_io_helpers.go index 62eed318..76d8fd70 100644 --- a/backend/posix/posix_io_helpers.go +++ b/backend/posix/posix_io_helpers.go @@ -16,6 +16,7 @@ package posix import ( "bufio" + "errors" "io" "log" "sync" @@ -45,6 +46,24 @@ func (b *bufferedReadCloser) Close() error { return b.c.Close() } +// closeOnEOFReader closes c once r is drained, for readers whose source has +// to be released before the caller is done with the reader. +type closeOnEOFReader struct { + r io.Reader + c io.Closer + closed bool +} + +func (e *closeOnEOFReader) Read(p []byte) (int, error) { + n, err := e.r.Read(p) + if errors.Is(err, io.EOF) && !e.closed { + e.closed = true + e.c.Close() + } + + return n, err +} + var odirectUnsupportedWarnByOp sync.Map func warnODirectUnsupportedOnce(op string, err error) { diff --git a/backend/posix/posix_io_helpers_test.go b/backend/posix/posix_io_helpers_test.go new file mode 100644 index 00000000..5beb962f --- /dev/null +++ b/backend/posix/posix_io_helpers_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "bytes" + "io" + "strings" + "testing" +) + +type countingCloser struct { + count int +} + +func (c *countingCloser) Close() error { + c.count++ + return nil +} + +func TestCloseOnEOFReader(t *testing.T) { + c := &countingCloser{} + r := &closeOnEOFReader{r: strings.NewReader("hello"), c: c} + + buf := make([]byte, 2) + n, err := r.Read(buf) + if err != nil { + t.Fatalf("read: %v", err) + } + if n != 2 { + t.Fatalf("expected 2 bytes, got %v", n) + } + if c.count != 0 { + t.Fatalf("expected the source to stay open before EOF, closed %v times", c.count) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read all: %v", err) + } + if !bytes.Equal(data, []byte("llo")) { + t.Fatalf("expected the remaining data to be llo, got %s", data) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once at EOF, closed %v times", c.count) + } + + // reads past EOF don't close the source again + if _, err := r.Read(buf); err != io.EOF { + t.Fatalf("expected io.EOF, got %v", err) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once, closed %v times", c.count) + } +} + +func TestCloseOnEOFReaderEmptySource(t *testing.T) { + c := &countingCloser{} + r := &closeOnEOFReader{r: strings.NewReader(""), c: c} + + if _, err := io.ReadAll(r); err != nil { + t.Fatalf("read all: %v", err) + } + if c.count != 1 { + t.Fatalf("expected the source to be closed once at EOF, closed %v times", c.count) + } +} diff --git a/backend/walk.go b/backend/walk.go index 767004fc..d0b9bfda 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -177,11 +177,18 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM return fs.SkipDir } + // A directory is compared to the markers as the key of its + // directory object, which has a trailing slash. + key := path + if d.IsDir() { + key += "/" + } + if !pastMarker { - if path == keyMarker { + if key == keyMarker { pastMarker = true } - if path < keyMarker { + if key < keyMarker { return nil } } @@ -195,28 +202,32 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM // building to match. So only skip if path isn't a prefix of prefix // and prefix isn't a prefix of path. if prefix != "" && - !strings.HasPrefix(path+"/", prefix) && - !strings.HasPrefix(prefix, path+"/") { + !strings.HasPrefix(key, prefix) && + !strings.HasPrefix(prefix, key) { return fs.SkipDir } // Don't recurse into subdirectories when listing with delimiter. + // The walk also enters the directories that sort before the key + // marker, so the common prefix is the part of the key up to the + // first delimiter after the prefix. if delimiter == "/" && - prefix != path+"/" && - strings.HasPrefix(path+"/", prefix) { - cpmap.Add(path + "/") + prefix != key && + strings.HasPrefix(key, prefix) { + before, _, _ := strings.Cut(strings.TrimPrefix(key, prefix), delimiter) + cpmap.Add(prefix + before + delimiter) return fs.SkipDir } // Skip ancestor directories of the specified prefix; only process // the directory that exactly matches the prefix. - // At this point we know strings.HasPrefix(prefix, path+"/") holds + // At this point we know strings.HasPrefix(prefix, key) holds // (i.e. path is an ancestor of the prefix directory). Skip it // unless it is the exact prefix directory. // Note: WalkVersions always walks from "." (unlike Walk, which // narrows the root) because versioning marker semantics require // visiting all entries in order, so this guard is needed instead. - if prefix != "" && strings.HasPrefix(prefix, path+"/") && path+"/" != prefix { + if prefix != "" && strings.HasPrefix(prefix, key) && key != prefix { return nil } @@ -231,7 +242,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM delMarkers = append(delMarkers, res.DelMarkers...) if res.Truncated { truncated = true - nextMarker = path + nextMarker = key nextVersionIdMarker = res.NextVersionIdMarker return fs.SkipAll } diff --git a/s3err/s3err.go b/s3err/s3err.go index 5a02d440..6262d77e 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -131,6 +131,7 @@ const ( ErrMissingDateHeader ErrGetUploadsWithKey ErrVersionsWithKey + ErrCopySourceDeleteMarker ErrInvalidRequest ErrAuthNotSetup ErrNotImplemented @@ -453,6 +454,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "There is no such thing as the ?versions sub-resource for a key", HTTPStatusCode: http.StatusBadRequest, }, + ErrCopySourceDeleteMarker: { + Code: "InvalidRequest", + Description: "The source of a copy request may not specifically refer to a delete marker by version id.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidRequest: { Code: "InvalidRequest", Description: "Invalid Request.", diff --git a/tests/integration/ListObjectVersions.go b/tests/integration/ListObjectVersions.go index 27a3932e..eaf9b51c 100644 --- a/tests/integration/ListObjectVersions.go +++ b/tests/integration/ListObjectVersions.go @@ -26,7 +26,7 @@ import ( func ListObjectVersions_VD_success(s *S3Conf) error { testName := "ListObjectVersions_VD_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - versions := []types.ObjectVersion{} + versions, dirVersions := []types.ObjectVersion{}, []types.ObjectVersion{} for i := range 5 { dLgth := int64(i * 100) key := fmt.Sprintf("my-obj-%v", i) @@ -46,8 +46,29 @@ func ListObjectVersions_VD_success(s *S3Conf) error { VersionId: getPtr("null"), StorageClass: types.ObjectVersionStorageClassStandard, }) + + dir := fmt.Sprintf("my-dir-%v/", i) + out, err = putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &dir, + }, s3client) + if err != nil { + return err + } + + dirVersions = append(dirVersions, types.ObjectVersion{ + ETag: out.res.ETag, + IsLatest: getBoolPtr(true), + Key: &dir, + Size: getPtr(int64(0)), + VersionId: getPtr("null"), + StorageClass: types.ObjectVersionStorageClassStandard, + }) } + // the directory objects are listed before the regular objects + versions = append(dirVersions, versions...) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, @@ -97,11 +118,68 @@ func ListObjectVersions_negative_max_keys(s *S3Conf) error { func ListObjectVersions_list_single_object_versions(s *S3Conf) error { testName := "ListObjectVersions_list_single_object_versions" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - object := "my-obj" - versions, err := createObjVersions(s3client, bucket, object, 5) + return forEachKey([]string{"my-obj", "my-dir/"}, func(object string) error { + versions, err := createObjVersions(s3client, bucket, object, 5) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &object, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, out.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, out.Versions) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { + testName := "ListObjectVersions_list_multiple_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2, obj3 := "foo", "bar", "baz" + // "qux-quux" sorts before the directory object "qux/", which + // sorts before the object under it + dir, beforeDir, inDir := "qux/", "qux-quux", "qux/quux" + + obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) if err != nil { return err } + obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + if err != nil { + return err + } + obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + if err != nil { + return err + } + // "qux/" is put over the existing parent directory of "qux/quux" + inDirVersions, err := createObjVersions(s3client, bucket, inDir, 2) + if err != nil { + return err + } + dirVersions, err := createObjVersions(s3client, bucket, dir, 3) + if err != nil { + return err + } + beforeDirVersions, err := createObjVersions(s3client, bucket, beforeDir, 2) + if err != nil { + return err + } + + versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + versions = append(append(append(versions, beforeDirVersions...), dirVersions...), inDirVersions...) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ @@ -121,38 +199,100 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error { }, withVersioning(types.BucketVersioningStatusEnabled)) } -func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { - testName := "ListObjectVersions_list_multiple_object_versions" +func ListObjectVersions_dir_object_with_children(s *S3Conf) error { + testName := "ListObjectVersions_dir_object_with_children" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj1, obj2, obj3 := "foo", "bar", "baz" + obj, dir, child, nestedObj := "my-obj", "my-dir/", "my-dir/child", "my-dir/sub/obj" - obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) + objVersions, err := createObjVersions(s3client, bucket, obj, 2) if err != nil { return err } - obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + dirVersions, err := createObjVersions(s3client, bucket, dir, 3) if err != nil { return err } - obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + childVersions, err := createObjVersions(s3client, bucket, child, 2) + if err != nil { + return err + } + nestedVersions, err := createObjVersions(s3client, bucket, nestedObj, 1) if err != nil { return err } - versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) - + // the objects under "my-dir/" stay listed after it's deleted ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, + Key: &dir, }) cancel() if err != nil { return err } - if !compareVersions(versions, out.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, out.Versions) + dirVersions[0].IsLatest = getBoolPtr(false) + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getBoolPtr(true), + Key: &dir, + VersionId: res.VersionId, + }, + } + + for i, test := range []struct { + prefix *string + delimiter *string + versions []types.ObjectVersion + delMarkers []types.DeleteMarkerEntry + prefixes []string + }{ + // "my-dir/" is rolled up into a common prefix + { + delimiter: getPtr("/"), + versions: objVersions, + prefixes: []string{dir}, + }, + // "my-dir/" itself and "my-dir/child" have no delimiter + // after the prefix + { + prefix: &dir, + delimiter: getPtr("/"), + versions: append(dirVersions, childVersions...), + delMarkers: delMarkers, + prefixes: []string{"my-dir/sub/"}, + }, + // "my-dir/sub" isn't a directory object + { + prefix: &dir, + versions: append(append(dirVersions, childVersions...), nestedVersions...), + delMarkers: delMarkers, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: test.prefix, + Delimiter: test.delimiter, + }) + cancel() + if err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + + if !compareVersions(test.versions, out.Versions) { + return fmt.Errorf("test case %d failed: expected the versions to be %v, instead got %v", + i, sprintVersions(test.versions), sprintVersions(out.Versions)) + } + if !compareDelMarkers(test.delMarkers, out.DeleteMarkers) { + return fmt.Errorf("test case %d failed: expected the delete markers to be %v, instead got %v", + i, test.delMarkers, out.DeleteMarkers) + } + if !comparePrefixes(test.prefixes, out.CommonPrefixes) { + return fmt.Errorf("test case %d failed: expected the common prefixes to be %v, instead got %v", + i, test.prefixes, sprintPrefixes(out.CommonPrefixes)) + } } return nil @@ -162,7 +302,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { testName := "ListObjectVersions_multiple_object_versions_truncated" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj1, obj2, obj3 := "foo", "bar", "baz" + obj1, obj2, obj3, dir, child := "foo", "bar", "baz", "dir/", "dir/child" obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) if err != nil { @@ -176,87 +316,85 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { if err != nil { return err } + dirVersions, err := createObjVersions(s3client, bucket, dir, 4) + if err != nil { + return err + } + childVersions, err := createObjVersions(s3client, bucket, child, 3) + if err != nil { + return err + } - versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + versions := append(append(obj2Versions, obj3Versions...), dirVersions...) + versions = append(append(versions, childVersions...), obj1Versions...) maxKeys := int32(5) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - MaxKeys: &maxKeys, - }) - cancel() - if err != nil { - return err - } + // the pages end on noncurrent versions of "baz", "dir/" and "dir/child" + var keyMarker, versionIdMarker *string + for page := 0; page*int(maxKeys) < len(versions); page++ { + start := page * int(maxKeys) + end := min(start+int(maxKeys), len(versions)) + truncated := end < len(versions) - if out.Name == nil { - return fmt.Errorf("expected the bucket name to be %v, instead got nil", - bucket) - } - if *out.Name != bucket { - return fmt.Errorf("expected the bucket name to be %v, instead got %v", - bucket, *out.Name) - } - if out.IsTruncated == nil || !*out.IsTruncated { - return fmt.Errorf("expected the output to be truncated") - } - if out.MaxKeys == nil { - return fmt.Errorf("expected the max-keys to be %v, instead got nil", - maxKeys) - } - if *out.MaxKeys != maxKeys { - return fmt.Errorf("expected the max-keys to be %v, instead got %v", - maxKeys, *out.MaxKeys) - } - if getString(out.NextKeyMarker) != getString(versions[maxKeys-1].Key) { - return fmt.Errorf("expected the NextKeyMarker to be %v, instead got %v", - getString(versions[maxKeys-1].Key), getString(out.NextKeyMarker)) - } - if getString(out.NextVersionIdMarker) != getString(versions[maxKeys-1].VersionId) { - return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", - getString(versions[maxKeys-1].VersionId), getString(out.NextVersionIdMarker)) - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }) + cancel() + if err != nil { + return fmt.Errorf("page %v: %w", page, err) + } - if !compareVersions(versions[:maxKeys], out.Versions) { - return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", - sprintVersions(versions[:maxKeys]), sprintVersions(out.Versions)) - } + if out.Name == nil { + return fmt.Errorf("page %v: expected the bucket name to be %v, instead got nil", + page, bucket) + } + if *out.Name != bucket { + return fmt.Errorf("page %v: expected the bucket name to be %v, instead got %v", + page, bucket, *out.Name) + } + isTruncated := out.IsTruncated != nil && *out.IsTruncated + if isTruncated != truncated { + return fmt.Errorf("page %v: expected the output truncation to be %v, instead got %v", + page, truncated, isTruncated) + } + if out.MaxKeys == nil { + return fmt.Errorf("page %v: expected the max-keys to be %v, instead got nil", + page, maxKeys) + } + if *out.MaxKeys != maxKeys { + return fmt.Errorf("page %v: expected the max-keys to be %v, instead got %v", + page, maxKeys, *out.MaxKeys) + } + if getString(out.KeyMarker) != getString(keyMarker) { + return fmt.Errorf("page %v: expected the KeyMarker to be %v, instead got %v", + page, getString(keyMarker), getString(out.KeyMarker)) + } + if getString(out.VersionIdMarker) != getString(versionIdMarker) { + return fmt.Errorf("page %v: expected the VersionIdMarker to be %v, instead got %v", + page, getString(versionIdMarker), getString(out.VersionIdMarker)) + } + if truncated { + last := versions[end-1] + if getString(out.NextKeyMarker) != getString(last.Key) { + return fmt.Errorf("page %v: expected the NextKeyMarker to be %v, instead got %v", + page, getString(last.Key), getString(out.NextKeyMarker)) + } + if getString(out.NextVersionIdMarker) != getString(last.VersionId) { + return fmt.Errorf("page %v: expected the NextVersionIdMarker to be %v, instead got %v", + page, getString(last.VersionId), getString(out.NextVersionIdMarker)) + } + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - KeyMarker: out.NextKeyMarker, - VersionIdMarker: out.NextVersionIdMarker, - }) - cancel() - if err != nil { - return err - } + if !compareVersions(versions[start:end], out.Versions) { + return fmt.Errorf("page %v: expected the resulting object versions to be %v, instead got %v", + page, sprintVersions(versions[start:end]), sprintVersions(out.Versions)) + } - if out.Name == nil { - return fmt.Errorf("expected the bucket name to be %v, instead got nil", - bucket) - } - if *out.Name != bucket { - return fmt.Errorf("expected the bucket name to be %v, instead got %v", - bucket, *out.Name) - } - if out.IsTruncated != nil && *out.IsTruncated { - return fmt.Errorf("expected the output not to be truncated") - } - if getString(out.KeyMarker) != getString(versions[maxKeys-1].Key) { - return fmt.Errorf("expected the KeyMarker to be %v, instead got %v", - getString(versions[maxKeys-1].Key), getString(out.KeyMarker)) - } - if getString(out.VersionIdMarker) != getString(versions[maxKeys-1].VersionId) { - return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", - getString(versions[maxKeys-1].VersionId), getString(out.VersionIdMarker)) - } - - if !compareVersions(versions[maxKeys:], out.Versions) { - return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", - sprintVersions(versions[:maxKeys]), sprintVersions(out.Versions)) + keyMarker, versionIdMarker = out.NextKeyMarker, out.NextVersionIdMarker } return nil @@ -266,170 +404,180 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { func ListObjectVersions_with_delete_markers(s *S3Conf) error { testName := "ListObjectVersions_with_delete_markers" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - versions[0].IsLatest = getBoolPtr(false) + versions[0].IsLatest = getBoolPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + delMarkers := []types.DeleteMarkerEntry{} + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + Key: &obj, + VersionId: out.VersionId, + IsLatest: getBoolPtr(true), + }) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(res.DeleteMarkers, delMarkers) { + return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + return nil }) - cancel() - if err != nil { - return err - } - - delMarkers := []types.DeleteMarkerEntry{} - delMarkers = append(delMarkers, types.DeleteMarkerEntry{ - Key: &obj, - VersionId: out.VersionId, - IsLatest: getBoolPtr(true), - }) - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, res.Versions) - } - if !compareDelMarkers(res.DeleteMarkers, delMarkers) { - return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", - delMarkers, res.DeleteMarkers) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error { testName := "ListObjectVersions_containing_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) - if err != nil { - return err - } + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } - objLgth := int64(543) - out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + objLgth := objDataLen(obj, 543) + out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - versions[0].IsLatest = getBoolPtr(false) + versions[0].IsLatest = getBoolPtr(false) - versions = append([]types.ObjectVersion{ - { - ETag: out.res.ETag, - IsLatest: getBoolPtr(false), - Key: &obj, - Size: &objLgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - }, - }, versions...) + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &objLgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + }, + }, versions...) - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } - newVersions, err := createObjVersions(s3client, bucket, obj, 4) - if err != nil { - return err - } + newVersions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } - versions = append(newVersions, versions...) + versions = append(newVersions, versions...) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed object versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed object versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { testName := "ListObjectVersions_single_null_versionId_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, objLgth := "my-obj", int64(890) - out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + // the objects are put before versioning is enabled and are + // listed in this order + objs, objLgth := []string{"my-dir/", "my-obj"}, int64(890) + versions := []types.ObjectVersion{} + for _, obj := range objs { + size := objDataLen(obj, objLgth) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - versions := []types.ObjectVersion{ - { + versions = append(versions, types.ObjectVersion{ ETag: out.res.ETag, Key: &obj, StorageClass: types.ObjectVersionStorageClassStandard, IsLatest: getBoolPtr(false), - Size: &objLgth, + Size: &size, VersionId: &nullVersionId, - }, + }) } - delMarkers := []types.DeleteMarkerEntry{ - { + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + delMarkers := []types.DeleteMarkerEntry{} + for _, obj := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ IsLatest: getBoolPtr(true), Key: &obj, VersionId: res.VersionId, - }, + }) } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, }) @@ -454,7 +602,7 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { func ListObjectVersions_checksum(s *S3Conf) error { testName := "ListObjectVersions_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - versions := []types.ObjectVersion{} + versions, dirVersions := []types.ObjectVersion{}, []types.ObjectVersion{} for i, algo := range types.ChecksumAlgorithmCrc32.Values() { vers, err := createObjVersions(s3client, bucket, fmt.Sprintf("obj-%v", i), 1, withChecksumAlgo(algo)) if err != nil { @@ -462,8 +610,18 @@ func ListObjectVersions_checksum(s *S3Conf) error { } versions = append(versions, vers...) + + vers, err = createObjVersions(s3client, bucket, fmt.Sprintf("dir-%v/", i), 1, withChecksumAlgo(algo)) + if err != nil { + return err + } + + dirVersions = append(dirVersions, vers...) } + // the directory objects are listed before the regular objects + versions = append(dirVersions, versions...) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 92ea4ebe..01542058 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -994,6 +994,12 @@ func TestPosix(ts *TestState) { ts.Run(DeleteObject_name_too_long) ts.Run(CopyObject_overwrite_same_dir_object) ts.Run(CopyObject_overwrite_same_file_object) + ts.Run(CompleteMultipartUpload_overwrite_dir_obj) + if ts.conf.versioningEnabled { + ts.Run(CompleteMultipartUpload_overwrite_dir_obj_delete_marker) + } + ts.Run(ObjectTagging_trailing_slash_counterpart) + ts.Run(ObjectLock_trailing_slash_counterpart) ts.Run(DeleteObject_directory_not_empty) if !ts.conf.windowsTests { ts.Run(PutObject_race_with_delete) @@ -1098,6 +1104,9 @@ func TestScoutfs(ts *TestState) { ts.Run(DeleteObject_name_too_long) ts.Run(CopyObject_overwrite_same_dir_object) ts.Run(CopyObject_overwrite_same_file_object) + ts.Run(CompleteMultipartUpload_overwrite_dir_obj) + ts.Run(ObjectTagging_trailing_slash_counterpart) + ts.Run(ObjectLock_trailing_slash_counterpart) ts.Run(DeleteObject_directory_not_empty) } @@ -1945,12 +1954,16 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_PutObject_null_versionId_obj) ts.Run(Versioning_PutObject_overwrite_null_versionId_obj) ts.Run(Versioning_PutObject_success) + ts.Run(Versioning_PutObject_dir_object_new_version_resets_attributes) // CopyObject action ts.Run(Versioning_CopyObject_invalid_versionId) ts.Run(Versioning_CopyObject_encoded_versionid_separator_invalid_versionId) ts.Run(Versioning_CopyObject_success) ts.Run(Versioning_CopyObject_non_existing_version_id) ts.Run(Versioning_CopyObject_from_an_object_version) + ts.Run(Versioning_CopyObject_from_a_delete_marker) + ts.Run(Versioning_CopyObject_to_itself) + ts.Run(Versioning_CopyObject_to_itself_from_the_current_version) if !ts.conf.windowsTests { ts.Run(Versioning_CopyObject_special_chars) } @@ -1961,7 +1974,10 @@ func TestVersioning(ts *TestState) { } ts.Run(Versioning_HeadObject_invalid_parent) ts.Run(Versioning_HeadObject_success) + ts.Run(Versioning_HeadObject_dir_object_versions) ts.Run(Versioning_HeadObject_without_versionId) + ts.Run(Versioning_HeadObject_null_version_without_versionId) + ts.Run(Versioning_HeadObject_null_versionId_obj) ts.Run(Versioning_HeadObject_delete_marker) // GetObject action ts.Run(Versioning_GetObject_invalid_versionId) @@ -1970,6 +1986,8 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_GetObject_delete_marker_without_versionId) ts.Run(Versioning_GetObject_delete_marker) ts.Run(Versioning_GetObject_null_versionId_obj) + ts.Run(Versioning_GetObject_null_version_without_versionId) + ts.Run(Versioning_unversioned_bucket_omits_versionId) // object tagging actions ts.Run(Versioning_PutObjectTagging_invalid_versionId) ts.Run(Versioning_PutObjectTagging_non_existing_object_version) @@ -1979,6 +1997,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_DeleteObjectTagging_invalid_versionId) ts.Run(Versioning_DeleteObjectTagging_non_existing_object_version) ts.Run(Versioning_PutGetDeleteObjectTagging_success) + ts.Run(Versioning_ObjectTagging_trailing_slash_counterpart) // GetObjectAttributes action ts.Run(Versioning_GetObjectAttributes_invalid_versionId) ts.Run(Versioning_GetObjectAttributes_object_version) @@ -1986,9 +2005,14 @@ func TestVersioning(ts *TestState) { // DeleteObject actions ts.Run(Versioning_DeleteObject_invalid_versionId) ts.Run(Versioning_DeleteObject_delete_object_version) + ts.Run(Versioning_DeleteObject_dir_object_latest_version) + ts.Run(Versioning_DeleteObject_latest_version_with_null_version) ts.Run(Versioning_DeleteObject_non_existing_object) + ts.Run(Versioning_DeleteObject_implicit_dir) + ts.Run(Versioning_DeleteObject_trailing_slash_counterpart) if !ts.conf.windowsTests { ts.Run(Versioning_DeleteObject_delete_a_delete_marker) + ts.Run(Versioning_DeleteObject_dir_object_with_children) } ts.Run(Versioning_Delete_null_versionId_object) ts.Run(Versioning_DeleteObject_nested_dir_object) @@ -2001,6 +2025,7 @@ func TestVersioning(ts *TestState) { ts.Run(ListObjectVersions_negative_max_keys) ts.Run(ListObjectVersions_list_single_object_versions) ts.Run(ListObjectVersions_list_multiple_object_versions) + ts.Run(ListObjectVersions_dir_object_with_children) ts.Run(ListObjectVersions_multiple_object_versions_truncated) ts.Run(ListObjectVersions_with_delete_markers) ts.Run(ListObjectVersions_containing_null_versionId_obj) @@ -2013,6 +2038,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId) ts.Run(Versioning_UploadPartCopy_non_existing_versionId) ts.Run(Versioning_UploadPartCopy_from_an_object_version) + ts.Run(Versioning_UploadPartCopy_from_a_delete_marker) // Object lock configuration ts.Run(Versioning_object_lock_not_enabled_on_bucket_creation) ts.Run(Versioning_Enable_object_lock) @@ -2033,6 +2059,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_Put_GetObjectLegalHold_success) // WORM protection ts.Run(Versioning_WORM_obj_version_locked_with_legal_hold) + ts.Run(Versioning_WORM_dir_object_lock_headers) ts.Run(Versioning_WORM_obj_version_locked_with_governance_retention) ts.Run(Versioning_WORM_obj_version_locked_with_compliance_retention) ts.Run(Versioning_WORM_delete_marker_locked_object_legal_hold) @@ -2040,10 +2067,13 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_WORM_delete_marker_locked_object_compliance_retention) ts.Run(Versioning_WORM_PutObject_overwrite_locked_object) ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object) + ts.Run(Versioning_WORM_CopyObject_to_itself_locked_object) ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object) if !ts.conf.windowsTests { ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention) } + ts.Run(Versioning_WORM_trailing_slash_counterpart) + ts.Run(Versioning_WORM_null_version_locked_with_legal_hold) // Concurrent requests // Versioninig_concurrent_upload_object ts.Run(Versioning_AccessControl_GetObjectVersion) @@ -2990,6 +3020,10 @@ func GetIntTests() IntTests { "DeleteObject_name_too_long": DeleteObject_name_too_long, "CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object, "CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object, + "CompleteMultipartUpload_overwrite_dir_obj": CompleteMultipartUpload_overwrite_dir_obj, + "CompleteMultipartUpload_overwrite_dir_obj_delete_marker": CompleteMultipartUpload_overwrite_dir_obj_delete_marker, + "ObjectTagging_trailing_slash_counterpart": ObjectTagging_trailing_slash_counterpart, + "ObjectLock_trailing_slash_counterpart": ObjectLock_trailing_slash_counterpart, "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, "DeleteObject_directory_object": DeleteObject_directory_object, "DeleteObject_success": DeleteObject_success, @@ -3473,23 +3507,32 @@ func GetIntTests() IntTests { "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, "Versioning_PutObject_success": Versioning_PutObject_success, + "Versioning_PutObject_dir_object_new_version_resets_attributes": Versioning_PutObject_dir_object_new_version_resets_attributes, "Versioning_CopyObject_invalid_versionId": Versioning_CopyObject_invalid_versionId, "Versioning_CopyObject_encoded_versionid_separator_invalid_versionId": Versioning_CopyObject_encoded_versionid_separator_invalid_versionId, "Versioning_CopyObject_success": Versioning_CopyObject_success, "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, + "Versioning_CopyObject_from_a_delete_marker": Versioning_CopyObject_from_a_delete_marker, + "Versioning_CopyObject_to_itself": Versioning_CopyObject_to_itself, + "Versioning_CopyObject_to_itself_from_the_current_version": Versioning_CopyObject_to_itself_from_the_current_version, "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, "Versioning_HeadObject_non_existing_object_version": Versioning_HeadObject_non_existing_object_version, "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, "Versioning_HeadObject_success": Versioning_HeadObject_success, + "Versioning_HeadObject_dir_object_versions": Versioning_HeadObject_dir_object_versions, "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, + "Versioning_HeadObject_null_versionId_obj": Versioning_HeadObject_null_versionId_obj, + "Versioning_HeadObject_null_version_without_versionId": Versioning_HeadObject_null_version_without_versionId, "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, "Versioning_GetObject_non_existing_object_version": Versioning_GetObject_non_existing_object_version, "Versioning_GetObject_success": Versioning_GetObject_success, "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, + "Versioning_unversioned_bucket_omits_versionId": Versioning_unversioned_bucket_omits_versionId, + "Versioning_GetObject_null_version_without_versionId": Versioning_GetObject_null_version_without_versionId, "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, "Versioning_PutObjectTagging_invalid_versionId": Versioning_PutObjectTagging_invalid_versionId, "Versioning_PutObjectTagging_non_existing_object_version": Versioning_PutObjectTagging_non_existing_object_version, @@ -3499,13 +3542,19 @@ func GetIntTests() IntTests { "Versioning_DeleteObjectTagging_invalid_versionId": Versioning_DeleteObjectTagging_invalid_versionId, "Versioning_DeleteObjectTagging_non_existing_object_version": Versioning_DeleteObjectTagging_non_existing_object_version, "Versioning_PutGetDeleteObjectTagging_success": Versioning_PutGetDeleteObjectTagging_success, + "Versioning_ObjectTagging_trailing_slash_counterpart": Versioning_ObjectTagging_trailing_slash_counterpart, "Versioning_GetObjectAttributes_invalid_versionId": Versioning_GetObjectAttributes_invalid_versionId, "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, "Versioning_DeleteObject_invalid_versionId": Versioning_DeleteObject_invalid_versionId, "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_latest_version_with_null_version": Versioning_DeleteObject_latest_version_with_null_version, + "Versioning_DeleteObject_dir_object_latest_version": Versioning_DeleteObject_dir_object_latest_version, "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, + "Versioning_DeleteObject_implicit_dir": Versioning_DeleteObject_implicit_dir, + "Versioning_DeleteObject_trailing_slash_counterpart": Versioning_DeleteObject_trailing_slash_counterpart, "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_DeleteObject_dir_object_with_children": Versioning_DeleteObject_dir_object_with_children, "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, "Versioning_DeleteObject_nested_dir_object": Versioning_DeleteObject_nested_dir_object, "Versioning_DeleteObject_non_existing_objects": Versioning_DeleteObject_non_existing_objects, @@ -3516,6 +3565,7 @@ func GetIntTests() IntTests { "ListObjectVersions_negative_max_keys": ListObjectVersions_negative_max_keys, "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, + "ListObjectVersions_dir_object_with_children": ListObjectVersions_dir_object_with_children, "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, @@ -3527,6 +3577,7 @@ func GetIntTests() IntTests { "Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId": Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId, "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, + "Versioning_UploadPartCopy_from_a_delete_marker": Versioning_UploadPartCopy_from_a_delete_marker, "Versioning_object_lock_not_enabled_on_bucket_creation": Versioning_object_lock_not_enabled_on_bucket_creation, "Versioning_Enable_object_lock": Versioning_Enable_object_lock, "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, @@ -3543,6 +3594,7 @@ func GetIntTests() IntTests { "Versioning_PutGetObjectLegalHold_delete_marker": Versioning_PutGetObjectLegalHold_delete_marker, "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, + "Versioning_WORM_dir_object_lock_headers": Versioning_WORM_dir_object_lock_headers, "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, "Versioning_WORM_delete_marker_locked_object_legal_hold": Versioning_WORM_delete_marker_locked_object_legal_hold, @@ -3550,8 +3602,11 @@ func GetIntTests() IntTests { "Versioning_WORM_delete_marker_locked_object_compliance_retention": Versioning_WORM_delete_marker_locked_object_compliance_retention, "Versioning_WORM_PutObject_overwrite_locked_object": Versioning_WORM_PutObject_overwrite_locked_object, "Versioning_WORM_CopyObject_overwrite_locked_object": Versioning_WORM_CopyObject_overwrite_locked_object, + "Versioning_WORM_CopyObject_to_itself_locked_object": Versioning_WORM_CopyObject_to_itself_locked_object, "Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object": Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object, "Versioning_WORM_remove_delete_marker_under_bucket_default_retention": Versioning_WORM_remove_delete_marker_under_bucket_default_retention, + "Versioning_WORM_trailing_slash_counterpart": Versioning_WORM_trailing_slash_counterpart, + "Versioning_WORM_null_version_locked_with_legal_hold": Versioning_WORM_null_version_locked_with_legal_hold, "Versioning_AccessControl_GetObjectVersion": Versioning_AccessControl_GetObjectVersion, "Versioning_AccessControl_HeadObjectVersion": Versioning_AccessControl_HeadObjectVersion, "Versioning_AccessControl_object_tagging_policy": Versioning_AccessControl_object_tagging_policy, diff --git a/tests/integration/posix.go b/tests/integration/posix.go index b9401a77..71454f59 100644 --- a/tests/integration/posix.go +++ b/tests/integration/posix.go @@ -17,6 +17,7 @@ package integration import ( "context" "fmt" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -222,6 +223,255 @@ func CopyObject_overwrite_same_file_object(s *S3Conf) error { }) } +func CompleteMultipartUpload_overwrite_dir_obj(s *S3Conf) error { + testName := "CompleteMultipartUpload_overwrite_dir_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "foo/", "foo" + _, err := putObjects(s3client, []string{dir}, bucket) + if err != nil { + return err + } + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + + // the directory object isn't taken for the object of a completed upload + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: getPtr("non-existing-upload-id"), + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + + // the failed upload can still be completed or aborted + return checkAndAbortUpload(s3client, bucket, obj, *mp.UploadId) + }) +} + +func CompleteMultipartUpload_overwrite_dir_obj_delete_marker(s *S3Conf) error { + testName := "CompleteMultipartUpload_overwrite_dir_obj_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "foo/", "foo" + versions, err := createObjVersions(s3client, bucket, dir, 1) + if err != nil { + return err + } + versions[0].IsLatest = getPtr(false) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + delMarkers := []types.DeleteMarkerEntry{ + {Key: &dir, VersionId: out.VersionId, IsLatest: getPtr(true)}, + } + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + {ETag: parts[0].ETag, PartNumber: parts[0].PartNumber}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + + // the delete marker of the directory object is left in place + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(delMarkers, res.DeleteMarkers) { + return fmt.Errorf("expected the delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return checkAndAbortUpload(s3client, bucket, obj, *mp.UploadId) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func ObjectTagging_trailing_slash_counterpart(s *S3Conf) error { + testName := "ObjectTagging_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + tagSet := []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}} + + // a key and the same key with a trailing slash have one path: the + // object of one of them isn't an object of the other + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key=value"), + }, s3client) + if err != nil { + return err + } + + err = checkObjectTaggingErr(s3client, bucket, other, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if !areTagsSame(res.TagSet, tagSet) { + return fmt.Errorf("%v: expected the tag set to be %v, instead got %v", + obj, tagSet, res.TagSet) + } + } + + // neither key names the parent directory of an object + _, err := putObjects(s3client, []string{"my-parent/obj"}, bucket) + if err != nil { + return err + } + for _, key := range []string{"my-parent/", "my-parent"} { + err := checkObjectTaggingErr(s3client, bucket, key, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", key, err) + } + } + + return nil + }) +} + +func ObjectLock_trailing_slash_counterpart(s *S3Conf) error { + testName := "ObjectLock_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rDate := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + lockedObjs := []objToDelete{} + + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &rDate, + }, s3client) + if err != nil { + return err + } + lockedObjs = append(lockedObjs, objToDelete{key: obj, removeOnlyLeglHold: true}) + + err = checkObjectLockErr(s3client, bucket, other, "", s3err.GetAPIError(s3err.ErrNoSuchKey)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + // the lock of the object doesn't protect the other key + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + err = checkObjectLock(s3client, bucket, obj, "", rDate) + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + // PutObject_race_with_delete tests the race between PutObject and DeleteObject // in the same subdirectory. // One goroutine sequentially puts "race-dir/0.txt" … "race-dir/N-1.txt". diff --git a/tests/integration/utils.go b/tests/integration/utils.go index a04b4d02..f849a966 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -1234,22 +1234,24 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, var csum [32]byte var data []byte - if input.Body == nil && lgth != 0 { - data = make([]byte, lgth) - rand.Read(data) + if input.Body == nil { + if lgth != 0 { + data = make([]byte, lgth) + rand.Read(data) - csum = sha256.Sum256(data) - if cfg.checksumAlgorithm != "" { - hasher, err := NewHasher(cfg.checksumAlgorithm) - if err != nil { - return nil, err + if cfg.checksumAlgorithm != "" { + hasher, err := NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, err + } + + hasher.Write(data) + sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) } - - hasher.Write(data) - sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) + input.Body = bytes.NewReader(data) } - input.Body = bytes.NewReader(data) + csum = sha256.Sum256(data) } ctx, cancel := context.WithTimeout(context.Background(), longTimeout) @@ -2298,10 +2300,10 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts versions := []types.ObjectVersion{} for i := range count { rNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) - dataLength := rNumber.Int64() if err != nil { return nil, err } + dataLength := objDataLen(object, rNumber.Int64()) r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, @@ -2373,6 +2375,49 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts return versions, nil } +// createDeleteMarker deletes object without a version id, making the +// resulting delete marker the current version, and returns its version id. +func createDeleteMarker(client *s3.Client, bucket, object string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err != nil { + return "", err + } + if out.DeleteMarker == nil || !*out.DeleteMarker { + return "", fmt.Errorf("expected a delete marker to be created for %v", object) + } + if getString(out.VersionId) == "" { + return "", fmt.Errorf("expected non empty delete marker versionId for %v", object) + } + + return *out.VersionId, nil +} + +// objDataLen returns the data length to upload for key: a directory +// object can't hold data +func objDataLen(key string, lgth int64) int64 { + if strings.HasSuffix(key, "/") { + return 0 + } + return lgth +} + +// forEachKey runs fn for each of the keys and prefixes a returned error +// with the key it failed for. Tests use it to run the same scenario for +// a regular object and a directory object. +func forEachKey(keys []string, fn func(key string) error) error { + for _, key := range keys { + if err := fn(key); err != nil { + return fmt.Errorf("%v: %w", key, err) + } + } + return nil +} + // ReverseSlice reverses a slice of any type func reverseSlice[T any](s []T) []T { for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { @@ -3966,3 +4011,170 @@ func checkDeleteObjectsErrsInOrder(got []types.Error, want []keyDenial) error { } return nil } + +// checkObjectTaggingErr checks that getting, putting and deleting the +// tagging of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectTaggingErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("other-key"), Value: getPtr("other-value")}}, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("delete object tagging: %w", err) + } + + return nil +} + +// checkObjectLockErr checks that getting and putting the legal hold and the +// retention of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectLockErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object retention: %w", err) + } + + rDate := time.Now().Add(time.Hour * 2) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object retention: %w", err) + } + + return nil +} + +// checkObjectLock checks that the version versionId of key is under legal +// hold and has a governance retention until rDate. An empty versionId +// selects the current version. +func checkObjectLock(client *s3.Client, bucket, key, versionId string, rDate time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + lHold, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if lHold.LegalHold == nil || lHold.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %q, instead got %v", + types.ObjectLockLegalHoldStatusOn, lHold.LegalHold) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ret, err := client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if ret.Retention == nil || ret.Retention.Mode != types.ObjectLockRetentionModeGovernance || + ret.Retention.RetainUntilDate == nil || ret.Retention.RetainUntilDate.Unix() != rDate.Unix() { + return fmt.Errorf("expected a %q retention until %v, instead got %+v", + types.ObjectLockRetentionModeGovernance, rDate.Format(time.RFC3339), ret.Retention) + } + + return nil +} + +// checkAndAbortUpload checks that the upload uploadId of key is the only +// multipart upload in the bucket and aborts it +func checkAndAbortUpload(client *s3.Client, bucket, key, uploadId string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + if len(res.Uploads) != 1 || getString(res.Uploads[0].Key) != key || + getString(res.Uploads[0].UploadId) != uploadId { + return fmt.Errorf("expected the upload %v of %v to be listed, instead got %+v", + uploadId, key, res.Uploads) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + }) + cancel() + return err +} diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 88bc015a..3e32d2a7 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -33,54 +33,88 @@ import ( func Versioning_DeleteBucket_not_empty(s *S3Conf) error { testName := "Versioning_DeleteBucket_not_empty" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty)); err != nil { + return err + } + + // delete the key versions, so that the next key + // is the only one in the bucket + for _, version := range versions { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err != nil { + return err + } + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutObject_suspended_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_suspended_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - out, err := putObjectWithData(1222, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, 1222), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - return nil + return nil + }) }, withVersioning(types.BucketVersioningStatusSuspended)) } func Versioning_PutObject_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, lgth := "my-obj", int64(1234) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + nullVersions := map[string]types.ObjectVersion{} + err := forEachKey(keys, func(obj string) error { + lgth := objDataLen(obj, 1234) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + nullVersions[obj] = types.ObjectVersion{ + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + } + return nil + }) if err != nil { return err } @@ -91,46 +125,45 @@ func Versioning_PutObject_null_versionId_obj(s *S3Conf) error { return err } - versions, err := createObjVersions(s3client, bucket, obj, 4) - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } - versions = append(versions, types.ObjectVersion{ - ETag: out.res.ETag, - IsLatest: getBoolPtr(false), - Key: &obj, - Size: &lgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, + versions = append(versions, nullVersions[obj]) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }) } func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { testName := "Versioning_PutObject_overwrite_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := putObjectWithData(int64(1233), &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + err := forEachKey(keys, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 1233), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + return err + }) if err != nil { return err } @@ -141,7 +174,16 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { return err } - versions, err := createObjVersions(s3client, bucket, obj, 4) + objVersions := map[string][]types.ObjectVersion{} + err = forEachKey(keys, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } + + objVersions[obj] = versions + return nil + }) if err != nil { return err } @@ -152,67 +194,221 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { return err } - lgth := int64(3200) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + lgth := objDataLen(obj, 3200) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - if out.res.VersionId != nil { - return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", - getString(out.res.VersionId)) - } + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } - versions[0].IsLatest = getBoolPtr(false) + versions := objVersions[obj] + versions[0].IsLatest = getBoolPtr(false) - versions = append([]types.ObjectVersion{ - { - ETag: out.res.ETag, - IsLatest: getBoolPtr(true), - Key: &obj, - Size: &lgth, - VersionId: &nullVersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: out.res.ChecksumType, - }, - }, versions...) + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(true), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.res.ChecksumType, + }, + }, versions...) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the listed versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil }) } func Versioning_PutObject_success(s *S3Conf) error { testName := "Versioning_PutObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected the versionId to be returned") + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_PutObject_dir_object_new_version_resets_attributes(s *S3Conf) error { + testName := "Versioning_PutObject_dir_object_new_version_resets_attributes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + tags := []types.Tag{ + {Key: getPtr("key1"), Value: getPtr("val1")}, + {Key: getPtr("key2"), Value: getPtr("val2")}, + } + meta := map[string]string{ + "foo": "bar", + "baz": "quxx", + } + redirect := "/some/redirect" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: getPtr("my-obj"), + first, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key1=val1&key2=val2"), + Metadata: meta, + WebsiteRedirectLocation: &redirect, }) cancel() if err != nil { return err } - if res.VersionId == nil || *res.VersionId == "" { - return fmt.Errorf("expected the versionId to be returned") + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + second, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(first.VersionId) == "" || getString(second.VersionId) == "" { + return fmt.Errorf("expected non empty versionIds, instead got %q and %q", + getString(first.VersionId), getString(second.VersionId)) + } + if *first.VersionId == *second.VersionId { + return fmt.Errorf("expected a new versionId, instead got %v twice", *first.VersionId) + } + + // the current version holds none of the first version attributes + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + cur, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(cur.VersionId) != *second.VersionId { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + *second.VersionId, getString(cur.VersionId)) + } + if len(cur.Metadata) != 0 { + return fmt.Errorf("expected empty metadata, instead got %v", cur.Metadata) + } + if cur.WebsiteRedirectLocation != nil { + return fmt.Errorf("expected nil website-redirect-location, instead got %v", + *cur.WebsiteRedirectLocation) + } + if cur.TagCount != nil { + return fmt.Errorf("expected nil tag count, instead got %v", *cur.TagCount) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + curTags, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if len(curTags.TagSet) != 0 { + return fmt.Errorf("expected empty tag set, instead got %v", curTags.TagSet) + } + + // the first version keeps its attributes + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + old, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: first.VersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(old.VersionId) != *first.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *first.VersionId, getString(old.VersionId)) + } + if old.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if *old.ContentLength != 0 { + return fmt.Errorf("expected zero content-length, instead got %v", + *old.ContentLength) + } + if getString(old.ContentType) != directoryContentType { + return fmt.Errorf("expected the content-type to be %v, instead got %v", + directoryContentType, getString(old.ContentType)) + } + if !areMapsSame(meta, old.Metadata) { + return fmt.Errorf("expected the metadata to be %v, instead got %v", + meta, old.Metadata) + } + if getString(old.WebsiteRedirectLocation) != redirect { + return fmt.Errorf("expected the website-redirect-location to be %v, instead got %v", + redirect, getString(old.WebsiteRedirectLocation)) + } + if old.TagCount == nil { + return fmt.Errorf("expected non nil TagCount") + } + if *old.TagCount != int32(len(tags)) { + return fmt.Errorf("expected the tag count to be %v, instead got %v", + len(tags), *old.TagCount) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + oldTags, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: first.VersionId, + }) + cancel() + if err != nil { + return err + } + + if !areTagsSame(tags, oldTags.TagSet) { + return fmt.Errorf("expected the tag set to be %v, instead got %v", + tags, oldTags.TagSet) } return nil @@ -272,101 +468,117 @@ func Versioning_CopyObject_encoded_versionid_separator_invalid_versionId(s *S3Co func Versioning_CopyObject_success(s *S3Conf) error { testName := "Versioning_CopyObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstObj := "dst-obj" - srcBucket, srcObj := getBucketName(), "src-obj" - + srcBucket := getBucketName() if err := setup(s, srcBucket); err != nil { return err } - dstObjVersions, err := createObjVersions(s3client, bucket, dstObj, 1) - if err != nil { - return err + srcObjLens := map[string]int64{ + "src-obj": 2345, + "src-dir/": 0, + } + for srcObj, srcObjLen := range srcObjLens { + _, err := putObjectWithData(srcObjLen, &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } } - srcObjLen := int64(2345) - _, err = putObjectWithData(srcObjLen, &s3.PutObjectInput{ - Bucket: &srcBucket, - Key: &srcObj, - }, s3client) - if err != nil { - return err + // destination object -> source object + srcObjs := map[string]string{ + "dst-obj": "src-obj", + "dst-dir/": "src-dir/", + "dst-file": "src-dir/", } + err := forEachKey([]string{"dst-obj", "dst-dir/", "dst-file"}, func(dstObj string) error { + srcObj := srcObjs[dstObj] + dstObjVersions, err := createObjVersions(s3client, bucket, dstObj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v", srcBucket, srcObj)), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", srcBucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId in the result") + } + + srcObjLen := srcObjLens[srcObj] + dstObjVersions[0].IsLatest = getBoolPtr(false) + versions := append([]types.ObjectVersion{ + { + ETag: out.CopyObjectResult.ETag, + IsLatest: getBoolPtr(true), + Key: &dstObj, + Size: &srcObjLen, + VersionId: out.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.CopyObjectResult.ChecksumType, + }, + }, dstObjVersions...) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &dstObj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", + versions, res.Versions) + } + + return nil }) - cancel() if err != nil { return err } - if err := teardown(s, srcBucket); err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId in the result") - } - - dstObjVersions[0].IsLatest = getBoolPtr(false) - versions := append([]types.ObjectVersion{ - { - ETag: out.CopyObjectResult.ETag, - IsLatest: getBoolPtr(true), - Key: &dstObj, - Size: &srcObjLen, - VersionId: out.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: out.CopyObjectResult.ChecksumType, - }, - }, dstObjVersions...) - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the resulting versions to be %v, instead got %v", - versions, res.Versions) - } - - return nil + return teardown(s, srcBucket) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { testName := "Versioning_CopyObject_non_existing_version_id" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstBucket, dstObj := getBucketName(), "my-obj" - srcObj := "my-obj" - + dstBucket := getBucketName() if err := setup(s, dstBucket); err != nil { return err } - _, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &dstBucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVRZ", - bucket, srcObj)), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVRZ", + bucket, obj)), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + if err != nil { return err } @@ -381,135 +593,406 @@ func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { testName := "Versioning_CopyObject_from_an_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcBucket, srcObj, dstObj := getBucketName(), "my-obj", "my-dst-obj" + srcBucket := getBucketName() if err := setup(s, srcBucket, withVersioning(types.BucketVersioningStatusEnabled)); err != nil { return err } - srcObjVersions, err := createObjVersions(s3client, srcBucket, srcObj, 1) - if err != nil { - return err + // copy the noncurrent and then the current source object version + dstObjs := map[string]string{ + "my-obj": "my-dst-obj", + "my-dir/": "my-dst-dir/", } - srcObjVersion := srcObjVersions[0] + err := forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstObj := dstObjs[srcObj] + srcObjVersions, err := createObjVersions(s3client, srcBucket, srcObj, 2) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", srcBucket, srcObj, *srcObjVersion.VersionId)), + for _, srcObjVersion := range reverseSlice(srcObjVersions) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", srcBucket, srcObj, *srcObjVersion.VersionId)), + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if out.CopySourceVersionId == nil { + return fmt.Errorf("expected non nil CopySourceVersionId") + } + if *out.CopySourceVersionId != *srcObjVersion.VersionId { + return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", + *srcObjVersion.VersionId, *out.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *res.ContentLength != *srcObjVersion.Size { + return fmt.Errorf("expected the copied object size to be %v, instead got %v", + *srcObjVersion.Size, *res.ContentLength) + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", + *out.VersionId, *res.VersionId) + } + } + + return nil }) - cancel() if err != nil { return err } - if err := teardown(s, srcBucket); err != nil { + return teardown(s, srcBucket) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +// A copy source that resolves to a delete marker is rejected: the key has no +// current version when the marker is the latest, and naming the marker by +// version id is an invalid request. Versions the marker hides stay copyable. +func Versioning_CopyObject_from_a_delete_marker(s *S3Conf) error { + testName := "Versioning_CopyObject_from_a_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" + if err := setup(s, dstBucket); err != nil { return err } - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - if out.CopySourceVersionId == nil { - return fmt.Errorf("expected non nil CopySourceVersionId") - } - if *out.CopySourceVersionId != *srcObjVersion.VersionId { - return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", - *srcObjVersion.VersionId, *out.CopySourceVersionId) - } + err := forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &dstObj, - VersionId: out.VersionId, + delMarker, err := createDeleteMarker(s3client, bucket, srcObj) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, delMarker)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, getString(srcObjVersions[0].VersionId))), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != getString(srcObjVersions[0].VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersions[0].VersionId), getString(out.CopySourceVersionId)) + } + + return nil }) - cancel() if err != nil { return err } - if res.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *res.ContentLength != *srcObjVersion.Size { - return fmt.Errorf("expected the copied object size to be %v, instead got %v", - *srcObjVersion.Size, *res.ContentLength) - } - if *res.VersionId != *out.VersionId { - return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", - *out.VersionId, *res.VersionId) - } + return teardown(s, dstBucket) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} - return nil +// A copy of an object onto itself in a versioned bucket is an ordinary +// write: it creates a new version and leaves the one it replaces untouched. +// Without a metadata directive there is nothing to replace, so it's rejected. +func Versioning_CopyObject_to_itself(s *S3Conf) error { + testName := "Versioning_CopyObject_to_itself" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // directory objects always carry the directory content-type + srcContentType, dstContentType := "text/plain", "application/json" + if strings.HasSuffix(obj, "/") { + srcContentType, dstContentType = directoryContentType, directoryContentType + } + + srcMeta := map[string]string{"key": "value"} + r, err := putObjectWithData(objDataLen(obj, 1234), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: getPtr("text/plain"), + Metadata: srcMeta, + }, s3client) + if err != nil { + return err + } + + srcVersionId := getString(r.res.VersionId) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopyDest)); err != nil { + return err + } + + dstMeta := map[string]string{"new-key": "new-value"} + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + MetadataDirective: types.MetadataDirectiveReplace, + ContentType: getPtr("application/json"), + Metadata: dstMeta, + }) + cancel() + if err != nil { + return err + } + + dstVersionId := getString(out.VersionId) + if dstVersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if dstVersionId == srcVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", dstVersionId) + } + + // the replaced version keeps its own metadata + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &srcVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.ContentType) != srcContentType { + return fmt.Errorf("expected the source version content-type to be %v, instead got %v", + srcContentType, getString(res.ContentType)) + } + if !areMapsSame(res.Metadata, srcMeta) { + return fmt.Errorf("expected the source version metadata to be %v, instead got %v", + srcMeta, res.Metadata) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != dstVersionId { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + dstVersionId, getString(res.VersionId)) + } + if getString(res.ContentType) != dstContentType { + return fmt.Errorf("expected the new version content-type to be %v, instead got %v", + dstContentType, getString(res.ContentType)) + } + if !areMapsSame(res.Metadata, dstMeta) { + return fmt.Errorf("expected the new version metadata to be %v, instead got %v", + dstMeta, res.Metadata) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + vRes, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if len(vRes.Versions) != 2 { + return fmt.Errorf("expected 2 object versions, instead got %v", len(vRes.Versions)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +// Naming the current version in the copy source makes a copy onto the same +// key a regular copy, so it is accepted even without a metadata directive. +func Versioning_CopyObject_to_itself_from_the_current_version(s *S3Conf) error { + testName := "Versioning_CopyObject_to_itself_from_the_current_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + + srcVersionId := getString(versions[0].VersionId) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, obj, srcVersionId)), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != srcVersionId { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + srcVersionId, getString(out.CopySourceVersionId)) + } + if getString(out.VersionId) == srcVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &srcVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != srcVersionId { + return fmt.Errorf("expected the source version to remain, instead got %v", + getString(res.VersionId)) + } + + return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_special_chars(s *S3Conf) error { testName := "Versioning_CopyObject_special_chars" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcObj, dstBucket, dstObj := "foo?bar", getBucketName(), "bar&foo" + dstBucket := getBucketName() err := setup(s, dstBucket) if err != nil { return err } - srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err + // source object -> destination object + dstObjs := map[string]string{ + "foo?bar": "bar&foo", + "baz?dir/": "dir&baz/", } + err = forEachKey([]string{"foo?bar", "baz?dir/"}, func(srcObj string) error { + dstObj := dstObjs[srcObj] + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } - srcObjVersionId := *srcObjVersions[0].VersionId - copySource := fmt.Sprintf("%v/%v?versionId=%v", - bucket, - url.PathEscape(srcObj), - url.QueryEscape(srcObjVersionId), - ) + srcObjVersionId := *srcObjVersions[0].VersionId + copySource := fmt.Sprintf("%v/%v?versionId=%v", + bucket, + url.PathEscape(srcObj), + url.QueryEscape(srcObjVersionId), + ) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(copySource), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(copySource), + }) + cancel() + if err != nil { + return err + } + + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if res.CopySourceVersionId == nil { + return fmt.Errorf("expected non nil CopySourceVersionId") + } + if *res.CopySourceVersionId != srcObjVersionId { + return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", + srcObjVersionId, *res.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + VersionId: res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.VersionId != *res.VersionId { + return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", + *res.VersionId, *out.VersionId) + } + + return nil }) - cancel() if err != nil { return err } - if res.VersionId == nil || *res.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - if res.CopySourceVersionId == nil { - return fmt.Errorf("expected non nil CopySourceVersionId") - } - if *res.CopySourceVersionId != srcObjVersionId { - return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", - srcObjVersionId, *res.CopySourceVersionId) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &dstObj, - VersionId: res.VersionId, - }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.VersionId != *res.VersionId { - return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", - *res.VersionId, *out.VersionId) - } - err = teardown(s, dstBucket) if err != nil { return err @@ -548,27 +1031,28 @@ func Versioning_HeadObject_invalid_versionId(s *S3Conf) error { func Versioning_HeadObject_non_existing_object_version(s *S3Conf) error { testName := "Versioning_HeadObject_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - return nil }) } @@ -585,58 +1069,169 @@ func Versioning_HeadObject_invalid_parent(s *S3Conf) error { return err } - obj = "not-a-dir/bad-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + return forEachKey([]string{"not-a-dir/bad-obj", "not-a-dir/bad-dir/"}, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - return nil }) } func Versioning_HeadObject_success(s *S3Conf) error { testName := "Versioning_HeadObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - r, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", + dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *r.res.VersionId, *out.VersionId) + } + if strings.HasSuffix(obj, "/") && getString(out.ContentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(out.ContentType)) + } + + return nil }) - cancel() - if err != nil { - return err + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_HeadObject_dir_object_versions(s *S3Conf) error { + testName := "Versioning_HeadObject_dir_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + + type dirVersion struct { + versionId *string + metadata map[string]string } - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") + // each version gets its own metadata key + versions := []dirVersion{} + for i := range 3 { + metadata := map[string]string{ + fmt.Sprintf("key%v", i): fmt.Sprintf("value%v", i), + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: metadata, + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + versions = append(versions, dirVersion{ + versionId: res.VersionId, + metadata: metadata, + }) } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") + + checkVersion := func(v dirVersion, contentLength *int64, contentType, versionId *string, metadata map[string]string) error { + if contentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if *contentLength != 0 { + return fmt.Errorf("expected the object content-length to be 0, instead got %v", + *contentLength) + } + if getString(contentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(contentType)) + } + if getString(versionId) != *v.versionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *v.versionId, getString(versionId)) + } + if !areMapsSame(metadata, v.metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + v.metadata, metadata) + } + return nil } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) + + for _, v := range versions { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + head, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: v.versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("head version %v: %w", *v.versionId, err) + } + err = checkVersion(v, head.ContentLength, head.ContentType, head.VersionId, head.Metadata) + if err != nil { + return fmt.Errorf("head version %v: %w", *v.versionId, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + get, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: v.versionId, + }) + if err != nil { + cancel() + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } + bdy, err := io.ReadAll(get.Body) + get.Body.Close() + cancel() + if err != nil { + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } + if len(bdy) != 0 { + return fmt.Errorf("get version %v: expected empty body, instead got %v bytes", + *v.versionId, len(bdy)) + } + err = checkVersion(v, get.ContentLength, get.ContentType, get.VersionId, get.Metadata) + if err != nil { + return fmt.Errorf("get version %v: %w", *v.versionId, err) + } } return nil @@ -646,82 +1241,183 @@ func Versioning_HeadObject_success(s *S3Conf) error { func Versioning_HeadObject_without_versionId(s *S3Conf) error { testName := "Versioning_HeadObject_without_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - lastVersion := versions[0] + lastVersion := versions[0] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != *lastVersion.VersionId { + return fmt.Errorf("expected versionId to be %v, instead got %v", + *lastVersion.VersionId, getString(res.VersionId)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +// Versioning_HeadObject_null_version_without_versionId heads an object put +// into a versioning-suspended bucket, without naming a version id. +func Versioning_HeadObject_null_versionId_obj(s *S3Conf) error { + testName := "Versioning_HeadObject_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + keys, dataLen := []string{"my-obj", "my-dir/"}, int64(321) + // the objects are put before versioning is enabled + etags := make(map[string]string, len(keys)) + err := forEachKey(keys, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, dataLen), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + etags[obj] = getString(out.res.ETag) + return nil }) - cancel() if err != nil { return err } - if getString(res.VersionId) != *lastVersion.VersionId { - return fmt.Errorf("expected versionId to be %v, instead got %v", - *lastVersion.VersionId, getString(res.VersionId)) + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err } - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) + return forEachKey(keys, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &nullVersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != objDataLen(obj, dataLen) { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + objDataLen(obj, dataLen), res.ContentLength) + } + if getString(res.ETag) != etags[obj] { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + etags[obj], getString(res.ETag)) + } + + return nil + }) + }) +} + +func Versioning_HeadObject_null_version_without_versionId(s *S3Conf) error { + testName := "Versioning_HeadObject_null_version_without_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dataLen := objDataLen(obj, 765) + out, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != dataLen { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + dataLen, res.ContentLength) + } + if getString(res.ETag) != getString(out.res.ETag) { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + getString(out.res.ETag), getString(res.ETag)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusSuspended)) } func Versioning_HeadObject_delete_marker(s *S3Conf) error { testName := "Versioning_HeadObject_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, "MethodNotAllowed"); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkSdkApiErr(err, "MethodNotAllowed"); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err := checkSdkApiErr(err, "NotFound"); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -752,131 +1448,124 @@ func Versioning_GetObject_invalid_versionId(s *S3Conf) error { func Versioning_GetObject_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObject_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_success(s *S3Conf) error { testName := "Versioning_GetObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - r, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - // Get the object by versionId - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: r.res.VersionId, + // getObject checks that the object read with versionId is + // the uploaded version + getObject := func(versionId *string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + defer cancel() + if err != nil { + return err + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if out.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", + dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *r.res.VersionId, *out.VersionId) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + out.Body.Close() + + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("incorrect output content") + } + + return nil + } + + // Get the object by versionId + if err := getObject(r.res.VersionId); err != nil { + return err + } + + // Get the object without versionId + if err := getObject(nil); err != nil { + return err + } + + // Get the noncurrent object version by versionId + _, err = putObjectWithData(objDataLen(obj, 1000), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + return getObject(r.res.VersionId) }) - defer cancel() - if err != nil { - return err - } - - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) - } - - bdy, err := io.ReadAll(out.Body) - if err != nil { - return err - } - out.Body.Close() - - outCsum := sha256.Sum256(bdy) - if outCsum != r.csum { - return fmt.Errorf("incorrect output content") - } - - // Get the object without versionId - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - defer cancel() - if err != nil { - return err - } - - if out.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if out.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if *out.ContentLength != dLen { - return fmt.Errorf("expected the object content-length to be %v, instead got %v", - dLen, *out.ContentLength) - } - if *out.VersionId != *r.res.VersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - *r.res.VersionId, *out.VersionId) - } - - bdy, err = io.ReadAll(out.Body) - if err != nil { - return err - } - out.Body.Close() - - outCsum = sha256.Sum256(bdy) - if outCsum != r.csum { - return fmt.Errorf("incorrect output content") - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_delete_marker_without_versionId(s *S3Conf) error { testName := "Versioning_GetObject_delete_marker_without_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := putObjectWithData(1234, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys := []string{"my-obj", "my-dir/"} + // the objects are put before versioning is enabled + err := forEachKey(keys, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 1234), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + return err + }) if err != nil { return err } @@ -886,80 +1575,92 @@ func Versioning_GetObject_delete_marker_without_versionId(s *S3Conf) error { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + return forEachKey(keys, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } - return nil + return nil + }) }) } func Versioning_GetObject_delete_marker(s *S3Conf) error { testName := "Versioning_GetObject_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dLen := int64(2000) - obj := "my-obj" - _, err := putObjectWithData(dLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dLen := objDataLen(obj, 2000) + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if out.VersionId == nil || *out.VersionId == "" { - return fmt.Errorf("expected non empty versionId") - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_null_versionId_obj(s *S3Conf) error { testName := "Versioning_GetObject_null_versionId_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, lgth := "my-obj", int64(234) - out, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + keys, dataLen := []string{"my-obj", "my-dir/"}, int64(234) + // the objects are put before versioning is enabled + etags := make(map[string]string, len(keys)) + err := forEachKey(keys, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, dataLen), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + etags[obj] = getString(out.res.ETag) + return nil + }) if err != nil { return err } @@ -969,40 +1670,157 @@ func Versioning_GetObject_null_versionId_obj(s *S3Conf) error { return err } + return forEachKey(keys, func(obj string) error { + lgth, etag := objDataLen(obj, dataLen), etags[obj] + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &nullVersionId, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil { + return fmt.Errorf("expected non nil ContentLength") + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil VersionId") + } + if res.ETag == nil { + return fmt.Errorf("expected non nil ETag") + } + if *res.ContentLength != lgth { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + lgth, *res.ContentLength) + } + if *res.VersionId != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, insted got %v", + nullVersionId, *res.VersionId) + } + if *res.ETag != etag { + return fmt.Errorf("expecte the ETag to be %v, instead got %v", + etag, *res.ETag) + } + + return nil + }) + }) +} + +// Versioning_GetObject_null_version_without_versionId reads an object put +// into a versioning-suspended bucket, without naming a version id. The object +// is the null version and the response has to report it as null rather than +// leave the version id out. +func Versioning_GetObject_null_version_without_versionId(s *S3Conf) error { + testName := "Versioning_GetObject_null_version_without_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + dataLen := objDataLen(obj, 543) + out, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + // a put into a versioning-suspended bucket creates the null + // version and reports no version id + if out.res.VersionId != nil { + return fmt.Errorf("expected PutObject response to omit versionId, instead got %v", + getString(out.res.VersionId)) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } + if res.ContentLength == nil || *res.ContentLength != dataLen { + return fmt.Errorf("expected the Content-Length to be %v, instead got %v", + dataLen, res.ContentLength) + } + if getString(res.ETag) != getString(out.res.ETag) { + return fmt.Errorf("expected the ETag to be %v, instead got %v", + getString(out.res.ETag), getString(res.ETag)) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusSuspended)) +} + +// Versioning_unversioned_bucket_omits_versionId reads an object in a bucket +// that never had versioning configured. Such a bucket has no versions at all, +// so neither GetObject nor HeadObject reports a version id, not even the null +// one, even though the gateway itself runs with versioning enabled. +func Versioning_unversioned_bucket_omits_versionId(s *S3Conf) error { + testName := "Versioning_unversioned_bucket_omits_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: &nullVersionId, + vRes, err := s3client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, }) cancel() if err != nil { return err } - - if res.ContentLength == nil { - return fmt.Errorf("expected non nil ContentLength") - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil VersionId") - } - if res.ETag == nil { - return fmt.Errorf("expected non nil ETag") - } - if *res.ContentLength != lgth { - return fmt.Errorf("expected the Content-Length to be %v, instead got %v", - lgth, *res.ContentLength) - } - if *res.VersionId != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, insted got %v", - nullVersionId, *res.VersionId) - } - if *res.ETag != *out.res.ETag { - return fmt.Errorf("expecte the ETag to be %v, instead got %v", - *out.res.ETag, *res.ETag) + // guard the premise of the test + if vRes.Status != "" { + return fmt.Errorf("expected the bucket versioning to be unconfigured, instead got %v", + vRes.Status) } - return nil + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 432), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + gRes, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if gRes.VersionId != nil { + return fmt.Errorf("expected GetObject to omit the versionId, instead got %v", + *gRes.VersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + hRes, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if hRes.VersionId != nil { + return fmt.Errorf("expected HeadObject to omit the versionId, instead got %v", + *hRes.VersionId) + } + + return nil + }) }) } @@ -1032,95 +1850,97 @@ func Versioning_GetObjectAttributes_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectAttributes_object_version(s *S3Conf) error { testName := "Versioning_GetObjectAttributes_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := versions[0] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := versions[0] - getObjAttrs := func(versionId *string) (*s3.GetObjectAttributesOutput, error) { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - ObjectAttributes: []types.ObjectAttributes{ - types.ObjectAttributesEtag, - }, - }) - cancel() - return res, err - } + getObjAttrs := func(versionId *string) (*s3.GetObjectAttributesOutput, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesEtag, + }, + }) + cancel() + return res, err + } - // By specifying the versionId - res, err := getObjAttrs(version.VersionId) - if err != nil { - return err - } + // By specifying the versionId + res, err := getObjAttrs(version.VersionId) + if err != nil { + return err + } - if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { - return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", - strings.Trim(*version.ETag, "\""), getString(res.ETag)) - } - if getString(res.VersionId) != *version.VersionId { - return fmt.Errorf("expected the uploaded versionId to be %v, instead got %v", - *version.VersionId, getString(res.VersionId)) - } + if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { + return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", + strings.Trim(*version.ETag, "\""), getString(res.ETag)) + } + if getString(res.VersionId) != *version.VersionId { + return fmt.Errorf("expected the uploaded versionId to be %v, instead got %v", + *version.VersionId, getString(res.VersionId)) + } - // Without versionId - res, err = getObjAttrs(nil) - if err != nil { - return err - } + // Without versionId + res, err = getObjAttrs(nil) + if err != nil { + return err + } - if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { - return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", - strings.Trim(*version.ETag, "\""), getString(res.ETag)) - } - if getString(res.VersionId) != *version.VersionId { - return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", - *version.VersionId, getString(res.VersionId)) - } + if getString(res.ETag) != strings.Trim(*version.ETag, "\"") { + return fmt.Errorf("expected the uploaded object ETag to be %v, instead got %v", + strings.Trim(*version.ETag, "\""), getString(res.ETag)) + } + if getString(res.VersionId) != *version.VersionId { + return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", + *version.VersionId, getString(res.VersionId)) + } - return nil + return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObjectAttributes_delete_marker(s *S3Conf) error { testName := "Versioning_GetObjectAttributes_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: res.VersionId, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesEtag, + }, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: res.VersionId, - ObjectAttributes: []types.ObjectAttributes{ - types.ObjectAttributesEtag, - }, - }) - cancel() - if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1150,75 +1970,402 @@ func Versioning_DeleteObject_invalid_versionId(s *S3Conf) error { func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { testName := "Versioning_DeleteObject_delete_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - oLen := int64(1000) - obj := "my-obj" - r, err := putObjectWithData(oLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + oLen := objDataLen(obj, 1000) + r, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + versionId := r.res.VersionId + if versionId == nil || *versionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + _, err = putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil { + return fmt.Errorf("expected non nil versionId") + } + if *out.VersionId != *versionId { + return fmt.Errorf("expected deleted object versionId to be %v, instead got %v", + *versionId, *out.VersionId) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_dir_object_latest_version(s *S3Conf) error { + testName := "Versioning_DeleteObject_dir_object_latest_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + versionIds := []string{} + for i := range 3 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: map[string]string{ + "version": fmt.Sprint(i), + }, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + versionIds = append(versionIds, *out.VersionId) } - versionId := r.res.VersionId - if versionId == nil || *versionId == "" { - return fmt.Errorf("expected non empty versionId") - } + // deleting the latest version makes the previous one the current + for i := 2; i > 0; i-- { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &versionIds[i], + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) != versionIds[i] { + return fmt.Errorf("expected the deleted versionId to be %v, instead got %v", + versionIds[i], getString(out.VersionId)) + } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be false") + } - _, err = putObjects(s3client, []string{obj}, bucket) - if err != nil { - return err + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != versionIds[i-1] { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + versionIds[i-1], getString(res.VersionId)) + } + expectedMeta := map[string]string{ + "version": fmt.Sprint(i - 1), + } + if !areMapsSame(res.Metadata, expectedMeta) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + expectedMeta, res.Metadata) + } + if getString(res.ContentType) != directoryContentType { + return fmt.Errorf("expected the content type to be %v, instead got %v", + directoryContentType, getString(res.ContentType)) + } + if res.ContentLength == nil || *res.ContentLength != 0 { + return fmt.Errorf("expected zero content length, instead got %v", + res.ContentLength) + } } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, Key: &obj, - VersionId: versionId, + VersionId: &versionIds[0], }) cancel() if err != nil { return err } - if out.VersionId == nil { - return fmt.Errorf("expected non nil versionId") + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err == nil { + return fmt.Errorf("expected NotFound, instead got nil") } - if *out.VersionId != *versionId { - return fmt.Errorf("expected deleted object versionId to be %v, instead got %v", - *versionId, *out.VersionId) + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Versions) != 0 { + return fmt.Errorf("expected empty object versions, instead got %v", res.Versions) + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", res.DeleteMarkers) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Versions) != 1 { + return fmt.Errorf("expected 1 object version, instead got %v", len(res.Versions)) + } + if getString(res.Versions[0].Key) != obj { + return fmt.Errorf("expected the version key to be %v, instead got %v", + obj, getString(res.Versions[0].Key)) + } + if getString(res.Versions[0].VersionId) != *out.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *out.VersionId, getString(res.Versions[0].VersionId)) + } + if res.Versions[0].IsLatest == nil || !*res.Versions[0].IsLatest { + return fmt.Errorf("expected the version to be the latest") + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", res.DeleteMarkers) } return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } +// Versioning_DeleteObject_latest_version_with_null_version deletes the current +// version of an object whose history also holds a null version, created while +// versioning was suspended. The version that becomes current has to be the one +// created right before the deleted version, not the older null version. +func Versioning_DeleteObject_latest_version_with_null_version(s *S3Conf) error { + testName := "Versioning_DeleteObject_latest_version_with_null_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // the versions are told apart by a metadata entry, as a + // directory object carries no data + put := func(marker string) (*s3.PutObjectOutput, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + return s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: map[string]string{"marker": marker}, + }) + } + + if _, err := put("v1"); err != nil { + return err + } + if _, err := put("v2"); err != nil { + return err + } + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + // the null version sits in the middle of the version history + if _, err := put("null"); err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + third, err := put("v3") + if err != nil { + return err + } + latest, err := put("v4") + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: latest.VersionId, + }) + cancel() + if err != nil { + return err + } + if getString(out.VersionId) != getString(latest.VersionId) { + return fmt.Errorf("expected the deleted versionId to be %v, instead got %v", + getString(latest.VersionId), getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + expectedMeta := map[string]string{"marker": "v3"} + if !areMapsSame(res.Metadata, expectedMeta) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + expectedMeta, res.Metadata) + } + if getString(res.VersionId) != getString(third.VersionId) { + return fmt.Errorf("expected the current versionId to be %v, instead got %v", + getString(third.VersionId), getString(res.VersionId)) + } + + // the null version has to be left untouched by the delete + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + versions, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + nullFound := false + for _, v := range versions.Versions { + if getString(v.VersionId) == nullVersionId { + nullFound = true + if v.IsLatest != nil && *v.IsLatest { + return fmt.Errorf("expected the null version not to be the latest") + } + } + } + if !nullFound { + return fmt.Errorf("expected the null version to be kept in %v", + versions.Versions) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { testName := "Versioning_DeleteObject_non_existing_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + ctx, canel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + canel() + if err != nil { + return err + } - ctx, canel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, canel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("non_existing_version_id"), + }) + canel() + if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, "non_existing_version_id")); err != nil { + return err + } + + return nil }) - canel() + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_implicit_dir(s *S3Conf) error { + testName := "Versioning_DeleteObject_implicit_dir" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, obj := "my-dir/", "my-dir/obj" + versions, err := createObjVersions(s3client, bucket, obj, 1) if err != nil { return err } - ctx, canel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("non_existing_version_id"), + // "my-dir/" is only the parent directory of "my-dir/obj", not an object + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, }) - canel() - if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, "non_existing_version_id")); err != nil { + cancel() + if err != nil { return err } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be false") + } + if getString(out.VersionId) != "" { + return fmt.Errorf("expected empty versionId, instead got %v", + getString(out.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if len(res.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", + res.DeleteMarkers) + } + return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1226,20 +2373,113 @@ func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { testName := "Versioning_DeleteObject_delete_a_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - oLen := int64(1000) - obj := "my-obj" - _, err := putObjectWithData(oLen, &s3.PutObjectInput{ + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + oLen := objDataLen(obj, 1000) + _, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.DeleteMarker == nil || !*out.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be true") + } + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.DeleteMarker == nil || !*res.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be true") + } + if res.VersionId == nil { + return fmt.Errorf("expected non empty versionId") + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *out.VersionId, *res.VersionId) + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_dir_object_with_children(s *S3Conf) error { + testName := "Versioning_DeleteObject_dir_object_with_children" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dir, child := "my-dir/", "my-dir/child" + dirVersions, err := createObjVersions(s3client, bucket, dir, 1) + if err != nil { + return err + } + childData, err := putObjectWithData(100, &s3.PutObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &child, }, s3client) if err != nil { return err } + // checkObjects checks that the child object is readable and that + // the bucket lists exactly keys + checkObjects := func(keys ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &child, + }) + cancel() + if err != nil { + return err + } + body, err := io.ReadAll(out.Body) + out.Body.Close() + if err != nil { + return err + } + if sha256.Sum256(body) != childData.csum { + return fmt.Errorf("expected the %v data checksum to match", child) + } + + objs, _, err := listObjects(s3client, bucket, "", "", 1000) + if err != nil { + return err + } + if !hasObjNames(objs, keys) { + return fmt.Errorf("expected the listed objects to be %v, instead got %v", + keys, objs) + } + + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &dir, }) cancel() if err != nil { @@ -1249,14 +2489,19 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { if out.DeleteMarker == nil || !*out.DeleteMarker { return fmt.Errorf("expected the response DeleteMarker to be true") } - if out.VersionId == nil || *out.VersionId == "" { + if getString(out.VersionId) == "" { return fmt.Errorf("expected non empty versionId") } + err = checkObjects(child) + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, - Key: &obj, + Key: &dir, VersionId: out.VersionId, }) cancel() @@ -1267,12 +2512,158 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { if res.DeleteMarker == nil || !*res.DeleteMarker { return fmt.Errorf("expected the response DeleteMarker to be true") } - if res.VersionId == nil { - return fmt.Errorf("expected non empty versionId") - } - if *res.VersionId != *out.VersionId { + if getString(res.VersionId) != *out.VersionId { return fmt.Errorf("expected the versionId to be %v, instead got %v", - *out.VersionId, *res.VersionId) + *out.VersionId, getString(res.VersionId)) + } + + // removing the delete marker restores the directory object + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + head, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err != nil { + return err + } + if getString(head.VersionId) != *dirVersions[0].VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *dirVersions[0].VersionId, getString(head.VersionId)) + } + + err = checkObjects(dir, child) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &dir, + VersionId: dirVersions[0].VersionId, + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) != *dirVersions[0].VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + *dirVersions[0].VersionId, getString(res.VersionId)) + } + + // with no versions left the directory is only the parent of the child + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + if err == nil { + return fmt.Errorf("expected NotFound, instead got nil") + } + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + err = checkObjects(child) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + versions, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(versions.Versions) != 1 || getString(versions.Versions[0].Key) != child { + return fmt.Errorf("expected only %v versions, instead got %v", + child, versions.Versions) + } + if len(versions.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", + versions.DeleteMarkers) + } + + return nil + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_DeleteObject_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_DeleteObject_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + expected := []types.ObjectVersion{} + // deleting the key with or without a trailing slash doesn't delete + // the object or any of its versions + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + versions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + expected = append(expected, versions...) + + for _, versionId := range []*string{versions[0].VersionId, versions[1].VersionId} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + VersionId: versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + if getString(out.VersionId) != *versionId { + return fmt.Errorf("%v: expected the versionId to be %v, instead got %v", + other, *versionId, getString(out.VersionId)) + } + if out.DeleteMarker != nil && *out.DeleteMarker { + return fmt.Errorf("%v: expected the response DeleteMarker to be false", other) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if getString(res.VersionId) != getString(versions[0].VersionId) { + return fmt.Errorf("%v: expected the versionId to be %v, instead got %v", + obj, getString(versions[0].VersionId), getString(res.VersionId)) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(expected, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + expected, res.Versions) } return nil @@ -1282,87 +2673,93 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { func Versioning_Delete_null_versionId_object(s *S3Conf) error { testName := "Versioning_Delete_null_versionId_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj, nObjLgth := "my-obj", int64(3211) - _, err := putObjectWithData(nObjLgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + objs, nObjLgth := []string{"my-obj", "my-dir/"}, int64(3211) + // the null versions are created before versioning is enabled + for _, obj := range objs { + _, err := putObjectWithData(objDataLen(obj, nObjLgth), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + } + + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) if err != nil { return err } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) - if err != nil { - return err - } + return forEachKey(objs, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - _, err = createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, getString(res.VersionId)) + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr(nullVersionId), + return nil }) - cancel() - if err != nil { - return err - } - if getString(res.VersionId) != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - nullVersionId, getString(res.VersionId)) - } - - return nil }) } func Versioning_DeleteObject_nested_dir_object(s *S3Conf) error { testName := "Versioning_DeleteObject_nested_dir_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "foo/bar/baz" - out, err := putObjectWithData(1000, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"foo/bar/baz", "foo/bar/baz/"}, func(obj string) error { + out, err := putObjectWithData(objDataLen(obj, 1000), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.res.VersionId, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if getString(res.VersionId) != getString(out.res.VersionId) { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + getString(out.res.VersionId), getString(res.VersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + // Then create the bucket back to not get error on teardown + if err := setup(s, bucket, withLock()); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } - - if getString(res.VersionId) != getString(out.res.VersionId) { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - getString(out.res.VersionId), getString(res.VersionId)) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - // Then create the bucket back to not get error on teardown - if err := setup(s, bucket); err != nil { - return err - } - - return nil }, withLock()) } @@ -1378,6 +2775,15 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { } versionId := getString(out.res.VersionId) + out, err = putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("my-dir/"), + }, s3client) + if err != nil { + return err + } + dirVersionId := getString(out.res.VersionId) + for _, test := range []struct { key string versionId string @@ -1386,8 +2792,14 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { {"foo/bar/baz", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"hello", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"hello/world", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"foo/bar/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"hello/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, + {"hello/world/", "01KF2YVN948NAZ4JJR4X1AAVRA"}, {"foo/bar/baz/quxx", versionId}, + {"foo/bar/baz/quxx/", versionId}, + {"my-dir/hello/", dirVersionId}, {"foo", versionId}, + {"my-dir/", dirVersionId}, } { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ @@ -1428,72 +2840,80 @@ func Versioning_DeleteObject_non_existing_objects(s *S3Conf) error { func Versioning_DeleteObject_suspended(s *S3Conf) error { testName := "Versioning_DeleteObject_suspended" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - versions[0].IsLatest = getBoolPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + // the object version is created while versioning is enabled + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } - err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) - if err != nil { - return err - } + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + versions[0].IsLatest = getBoolPtr(false) + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + for range 5 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.DeleteMarker == nil { + return fmt.Errorf("expected the delete marker to be true") + } + if !*res.DeleteMarker { + return fmt.Errorf("expected the delete marker to be true, instead got %v", + *res.DeleteMarker) + } + if res.VersionId == nil { + return fmt.Errorf("expected non nil versionId") + } + if *res.VersionId != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", + nullVersionId, *res.VersionId) + } + } - for range 5 { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ Bucket: &bucket, - Key: &obj, + Prefix: &obj, }) cancel() if err != nil { return err } - if res.DeleteMarker == nil { - return fmt.Errorf("expected the delete marker to be true") + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getBoolPtr(true), + Key: &obj, + VersionId: &nullVersionId, + }, } - if !*res.DeleteMarker { - return fmt.Errorf("expected the delete marker to be true, instead got %v", - *res.DeleteMarker) - } - if res.VersionId == nil { - return fmt.Errorf("expected non nil versionId") - } - if *res.VersionId != nullVersionId { - return fmt.Errorf("expected the versionId to be %v, instead got %v", - nullVersionId, *res.VersionId) - } - } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %v, instead got %v", + versions, res.Versions) + } + if !compareDelMarkers(res.DeleteMarkers, delMarkers) { + return fmt.Errorf("expected the delete markers to be %v, instead got %v", + delMarkers, res.DeleteMarkers) + } + + return nil }) - cancel() - if err != nil { - return err - } - - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getBoolPtr(true), - Key: &obj, - VersionId: &nullVersionId, - }, - } - - if !compareVersions(versions, res.Versions) { - return fmt.Errorf("expected the versions to be %v, instead got %v", - versions, res.Versions) - } - if !compareDelMarkers(res.DeleteMarkers, delMarkers) { - return fmt.Errorf("expected the delete markers to be %v, instead got %v", - delMarkers, res.DeleteMarkers) - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -1501,6 +2921,7 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { testName := "Versioning_DeleteObjects_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj1, obj2, obj3 := "foo", "bar", "baz" + dir1, dir2 := "foo-dir/", "baz-dir/" obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) if err != nil { @@ -1514,6 +2935,14 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { if err != nil { return err } + dir1Version, err := createObjVersions(s3client, bucket, dir1, 1) + if err != nil { + return err + } + dir2Version, err := createObjVersions(s3client, bucket, dir2, 1) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ @@ -1530,6 +2959,13 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { { Key: obj3Version[0].Key, }, + { + Key: dir1Version[0].Key, + VersionId: dir1Version[0].VersionId, + }, + { + Key: dir2Version[0].Key, + }, }, }, }) @@ -1552,6 +2988,15 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { Key: obj3Version[0].Key, DeleteMarker: getBoolPtr(true), }, + { + Key: dir1Version[0].Key, + VersionId: dir1Version[0].VersionId, + DeleteMarker: getBoolPtr(false), + }, + { + Key: dir2Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, } if len(out.Errors) != 0 { @@ -1574,7 +3019,9 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { obj2Version[0].IsLatest = getBoolPtr(false) obj3Version[0].IsLatest = getBoolPtr(false) + dir2Version[0].IsLatest = getBoolPtr(false) versions := append(obj2Version, obj3Version...) + versions = append(versions, dir2Version...) delMarkers := []types.DeleteMarkerEntry{ { @@ -1587,6 +3034,11 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { Key: out.Deleted[2].Key, VersionId: out.Deleted[2].DeleteMarkerVersionId, }, + { + IsLatest: getBoolPtr(true), + Key: out.Deleted[4].Key, + VersionId: out.Deleted[4].DeleteMarkerVersionId, + }, } if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", @@ -1605,6 +3057,7 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { testName := "Versioning_DeleteObjects_delete_deleteMarkers" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj1, obj2 := "foo", "bar" + dir1, dir2 := "foo-dir/", "bar-dir/" obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) if err != nil { @@ -1614,6 +3067,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { if err != nil { return err } + dir1Version, err := createObjVersions(s3client, bucket, dir1, 1) + if err != nil { + return err + } + dir2Version, err := createObjVersions(s3client, bucket, dir2, 1) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ @@ -1626,6 +3087,12 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { { Key: obj2Version[0].Key, }, + { + Key: dir1Version[0].Key, + }, + { + Key: dir2Version[0].Key, + }, }, }, }) @@ -1643,6 +3110,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { Key: obj2Version[0].Key, DeleteMarker: getBoolPtr(true), }, + { + Key: dir1Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, + { + Key: dir2Version[0].Key, + DeleteMarker: getBoolPtr(true), + }, } if len(out.Errors) != 0 { @@ -1667,6 +3142,14 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { Key: out.Deleted[1].Key, VersionId: out.Deleted[1].VersionId, }, + { + Key: out.Deleted[2].Key, + VersionId: out.Deleted[2].VersionId, + }, + { + Key: out.Deleted[3].Key, + VersionId: out.Deleted[3].VersionId, + }, }, }, }) @@ -1692,6 +3175,18 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { DeleteMarkerVersionId: out.Deleted[1].VersionId, VersionId: out.Deleted[1].VersionId, }, + { + Key: out.Deleted[2].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[2].VersionId, + VersionId: out.Deleted[2].VersionId, + }, + { + Key: out.Deleted[3].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[3].VersionId, + VersionId: out.Deleted[3].VersionId, + }, } if !compareDelObjects(delResult, res.Deleted) { @@ -1961,127 +3456,212 @@ func Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId(s * func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error { testName := "Versioning_UploadPartCopy_non_existing_versionId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dstBucket, dstObj, srcObj := getBucketName(), "dst-obj", "src-obj" + return forEachKey([]string{"src-obj", "src-dir/"}, func(srcObj string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" - lgth := int64(100) - _, err := putObjectWithData(lgth, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &srcObj, - }, s3client) - if err != nil { - return err - } + lgth := objDataLen(srcObj, 100) + _, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } - if err := setup(s, dstBucket); err != nil { - return err - } + if err := setup(s, dstBucket); err != nil { + return err + } - mp, err := createMp(s3client, dstBucket, dstObj) - if err != nil { - return err - } + mp, err := createMp(s3client, dstBucket, dstObj) + if err != nil { + return err + } - pNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &dstBucket, - Key: &dstObj, - UploadId: mp.UploadId, - PartNumber: &pNumber, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVS0", - bucket, srcObj)), + pNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &pNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=01BX5ZZKBKACTAV9WEVGEMMVS0", + bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - if err := teardown(s, dstBucket); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { testName := "Versioning_UploadPartCopy_from_an_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - srcObj, dstBucket, obj := "my-obj", getBucketName(), "dst-obj" - err := setup(s, dstBucket) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstBucket, obj := getBucketName(), "dst-obj" + err := setup(s, dstBucket) + if err != nil { + return err + } - srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } - srcObjVersion := srcObjVersions[0] + // the latest version and a noncurrent one + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 2) + if err != nil { + return err + } - out, err := createMp(s3client, dstBucket, obj) - if err != nil { - return err - } + for _, srcObjVersion := range srcObjVersions { + out, err := createMp(s3client, dstBucket, obj) + if err != nil { + return err + } - partNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &dstBucket, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, *srcObjVersion.VersionId)), - UploadId: out.UploadId, - Key: &obj, - PartNumber: &partNumber, + partNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, *srcObjVersion.VersionId)), + UploadId: out.UploadId, + Key: &obj, + PartNumber: &partNumber, + }) + cancel() + if err != nil { + return err + } + + if getString(copyOut.CopySourceVersionId) != getString(srcObjVersion.VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersion.VersionId), getString(copyOut.CopySourceVersionId)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &dstBucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", + len(res.Parts)) + } + if res.Parts[0].PartNumber == nil { + return fmt.Errorf("expected part-number to be non nil") + } + if *res.Parts[0].PartNumber != partNumber { + return fmt.Errorf("expected part-number to be %v, instead got %v", + partNumber, res.Parts[0].PartNumber) + } + if res.Parts[0].Size == nil { + return fmt.Errorf("expected part size to be non nil") + } + if *res.Parts[0].Size != *srcObjVersion.Size { + return fmt.Errorf("expected part size to be %v, instead got %v", + *srcObjVersion.Size, res.Parts[0].Size) + } + if getString(res.Parts[0].ETag) != getString(copyOut.CopyPartResult.ETag) { + return fmt.Errorf("expected part etag to be %v, instead got %v", + getString(copyOut.CopyPartResult.ETag), getString(res.Parts[0].ETag)) + } + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - return err - } + }, withVersioning(types.BucketVersioningStatusEnabled)) +} - if getString(copyOut.CopySourceVersionId) != getString(srcObjVersion.VersionId) { - return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", - getString(srcObjVersion.VersionId), getString(copyOut.CopySourceVersionId)) - } +// A copy source that resolves to a delete marker is rejected: the key has no +// current version when the marker is the latest, and naming the marker by +// version id is an invalid request. Versions the marker hides stay copyable. +func Versioning_UploadPartCopy_from_a_delete_marker(s *S3Conf) error { + testName := "Versioning_UploadPartCopy_from_a_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(srcObj string) error { + dstBucket, dstObj := getBucketName(), "dst-obj" + if err := setup(s, dstBucket); err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &dstBucket, - Key: &obj, - UploadId: out.UploadId, + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + + delMarker, err := createDeleteMarker(s3client, bucket, srcObj) + if err != nil { + return err + } + + mp, err := createMp(s3client, dstBucket, dstObj) + if err != nil { + return err + } + + partNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, delMarker)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", + bucket, srcObj, getString(srcObjVersions[0].VersionId))), + }) + cancel() + if err != nil { + return err + } + + if getString(out.CopySourceVersionId) != getString(srcObjVersions[0].VersionId) { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", + getString(srcObjVersions[0].VersionId), getString(out.CopySourceVersionId)) + } + + return teardown(s, dstBucket) }) - cancel() - if err != nil { - return err - } - - if len(res.Parts) != 1 { - return fmt.Errorf("expected parts to be 1, instead got %v", - len(res.Parts)) - } - if res.Parts[0].PartNumber == nil { - return fmt.Errorf("expected part-number to be non nil") - } - if *res.Parts[0].PartNumber != partNumber { - return fmt.Errorf("expected part-number to be %v, instead got %v", - partNumber, res.Parts[0].PartNumber) - } - if res.Parts[0].Size == nil { - return fmt.Errorf("expected part size to be non nil") - } - if *res.Parts[0].Size != *srcObjVersion.Size { - return fmt.Errorf("expected part size to be %v, instead got %v", - *srcObjVersion.Size, res.Parts[0].Size) - } - if getString(res.Parts[0].ETag) != getString(copyOut.CopyPartResult.ETag) { - return fmt.Errorf("expected part etag to be %v, instead got %v", - getString(copyOut.CopyPartResult.ETag), getString(res.Parts[0].ETag)) - } - - if err := teardown(s, dstBucket); err != nil { - return err - } - - return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2167,29 +3747,30 @@ func Versioning_PutObjectRetention_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectRetention_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectRetention_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2216,121 +3797,124 @@ func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectRetention_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectRetention_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Put_GetObjectRetention_delete_marker(s *S3Conf) error { testName := "Versioning_Put_GetObjectRetention_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectRetention + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectRetention + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectRetention - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectRetention - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } func Versioning_Put_GetObjectRetention_success(s *S3Conf) error { testName := "Versioning_Put_GetObjectRetention_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } - objVersion := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + objVersion := objVersions[1] - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.Retention.Mode != types.ObjectLockRetentionModeGovernance { + return fmt.Errorf("expected the object retention mode to be %v, instead got %v", + types.ObjectLockRetentionModeGovernance, res.Retention.Mode) + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: getString(objVersion.Key), versionId: getString(objVersion.VersionId)}}) }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - }) - cancel() - if err != nil { - return err - } - - if res.Retention.Mode != types.ObjectLockRetentionModeGovernance { - return fmt.Errorf("expected the object retention mode to be %v, instead got %v", - types.ObjectLockRetentionModeGovernance, res.Retention.Mode) - } - - return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: getString(objVersion.Key), versionId: getString(objVersion.VersionId)}}) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2360,27 +3944,28 @@ func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectLegalHold_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectLegalHold_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2407,123 +3992,126 @@ func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectLegalHold_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectLegalHold_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - _, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + _, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01G65Z755AFWAKHE12NY0CQ9FH"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + return nil }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { - return err - } - - return nil }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetObjectLegalHold_delete_marker(s *S3Conf) error { testName := "Versioning_PutGetObjectLegalHold_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectLegalHold + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectLegalHold + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectLegalHold - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectLegalHold - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { testName := "Versioning_Put_GetObjectLegalHold_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } - objVersion := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + objVersion := objVersions[1] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objVersion.VersionId, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objVersion.VersionId, + }) + cancel() + if err != nil { + return err + } - if res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { - return fmt.Errorf("expected the object version legal hold status to be %v, instead got %v", - types.ObjectLockLegalHoldStatusOn, res.LegalHold.Status) - } + if res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the object version legal hold status to be %v, instead got %v", + types.ObjectLockLegalHoldStatusOn, res.LegalHold.Status) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: getString(objVersion.Key), - versionId: getString(objVersion.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: getString(objVersion.Key), + versionId: getString(objVersion.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2531,90 +4119,200 @@ func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_legal_hold" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[1] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[1] - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } +func Versioning_WORM_dir_object_lock_headers(s *S3Conf) error { + testName := "Versioning_WORM_dir_object_lock_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-dir/" + rDate := time.Now().Add(time.Hour * 48) + lockedObjs := []objToDelete{} + + for i, test := range []struct { + legalHold types.ObjectLockLegalHoldStatus + mode types.ObjectLockMode + retainUntilDate *time.Time + }{ + {legalHold: types.ObjectLockLegalHoldStatusOn}, + {mode: types.ObjectLockModeGovernance, retainUntilDate: &rDate}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: test.legalHold, + ObjectLockMode: test.mode, + ObjectLockRetainUntilDate: test.retainUntilDate, + }, s3client) + if err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: getString(res.res.VersionId), + removeOnlyLeglHold: test.legalHold == types.ObjectLockLegalHoldStatusOn, + }) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + + if out.ObjectLockLegalHoldStatus != test.legalHold { + return fmt.Errorf("test %v: expected the object legal hold status to be %q, instead got %q", + i+1, test.legalHold, out.ObjectLockLegalHoldStatus) + } + if out.ObjectLockMode != test.mode { + return fmt.Errorf("test %v: expected the object lock mode to be %q, instead got %q", + i+1, test.mode, out.ObjectLockMode) + } + if test.retainUntilDate == nil { + if out.ObjectLockRetainUntilDate != nil { + return fmt.Errorf("test %v: expected nil object lock retain until date, instead got %v", + i+1, *out.ObjectLockRetainUntilDate) + } + } else if out.ObjectLockRetainUntilDate == nil || + out.ObjectLockRetainUntilDate.Unix() != test.retainUntilDate.Unix() { + return fmt.Errorf("test %v: expected the object lock retain until date to be %v, instead got %v", + i+1, test.retainUntilDate.Format(time.RFC3339), out.ObjectLockRetainUntilDate) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: res.res.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("test %v: %w", i+1, err) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.DeleteMarker == nil || !*out.DeleteMarker { + return fmt.Errorf("expected the delete marker to be true, instead got %v", out.DeleteMarker) + } + if getString(out.VersionId) == "" { + return fmt.Errorf("expected non empty delete marker versionId") + } + + // the noncurrent versions keep their lock settings + for _, lockedObj := range lockedObjs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &lockedObj.versionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("version %v: %w", lockedObj.versionId, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_governance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[0] + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[0] - rDate := time.Now().Add(time.Hour * 48) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &rDate, - }, - }) - cancel() - if err != nil { - return err - } + rDate := time.Now().Add(time.Hour * 48) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + }, + }) }) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } @@ -2622,114 +4320,124 @@ func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) err func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) error { testName := "Versioning_WORM_obj_version_locked_with_compliance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 2) - if err != nil { - return err - } - version := objVersions[0] + // COMPLIANCE retentions can only be waited out, so the locked + // versions of all the keys are cleaned up together + lockedObjs := []objToDelete{} + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + version := objVersions[0] - rDate := time.Now().Add(2 * complianceTestRetention) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &rDate, - }, - }) - cancel() - if err != nil { - return err - } + rDate := time.Now().Add(2 * complianceTestRetention) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: version.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: version.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { + lockedObjs = append(lockedObjs, objToDelete{ key: obj, versionId: getString(version.VersionId), isCompliance: true, - }, + }) + return nil }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_WORM_delete_marker_locked_object_legal_hold(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_legal_hold" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -2737,68 +4445,70 @@ func Versioning_WORM_delete_marker_locked_object_legal_hold(s *S3Conf) error { func Versioning_WORM_delete_marker_locked_object_governance_retention(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_governance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(version.VersionId), - isCompliance: false, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(version.VersionId), + isCompliance: false, + }, + }) }) }, withLock()) } @@ -2806,140 +4516,151 @@ func Versioning_WORM_delete_marker_locked_object_governance_retention(s *S3Conf) func Versioning_WORM_delete_marker_locked_object_compliance_retention(s *S3Conf) error { testName := "Versioning_WORM_delete_marker_locked_object_compliance_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objVersions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } - version := objVersions[0] - objVersions[0].IsLatest = getPtr(false) + // COMPLIANCE retentions can only be waited out, so the locked + // versions of all the keys are cleaned up together + lockedObjs := []objToDelete{} + err := forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objVersions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + version := objVersions[0] + objVersions[0].IsLatest = getPtr(false) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: getPtr(time.Now().Add(complianceTestRetention)), - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: getPtr(time.Now().Add(complianceTestRetention)), + }, + }) + cancel() + if err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } - delMarkers := []types.DeleteMarkerEntry{ - { - IsLatest: getPtr(true), - Key: &obj, - VersionId: out.VersionId, - }, - } + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getPtr(true), + Key: &obj, + VersionId: out.VersionId, + }, + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(objVersions, resp.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) - } - if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { - return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) - } + if !compareVersions(objVersions, resp.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", objVersions, resp.Versions) + } + if !compareDelMarkers(delMarkers, resp.DeleteMarkers) { + return fmt.Errorf("expected the object delete markers to be %v, instead got %v", delMarkers, resp.DeleteMarkers) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { + lockedObjs = append(lockedObjs, objToDelete{ key: obj, versionId: getString(version.VersionId), isCompliance: true, - }, + }) + return nil }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) }, withLock()) } func Versioning_WORM_PutObject_overwrite_locked_object(s *S3Conf) error { testName := "Versioning_WORM_PutObject_overwrite_locked_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - v := versions[0] - v.IsLatest = getPtr(false) + v := versions[0] + v.IsLatest = getPtr(false) - // lock the object with legal hold - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, - }) - cancel() - if err != nil { - return err - } + // lock the object with legal hold + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - dataLen := int64(10) + dataLen := objDataLen(obj, 10) - // overwrite the locked object with a new version - r, err := putObjectWithData(dataLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + // overwrite the locked object with a new version + r, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - version := types.ObjectVersion{ - ETag: r.res.ETag, - IsLatest: getPtr(true), - Key: &obj, - Size: &dataLen, - VersionId: r.res.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: r.res.ChecksumType, - } + version := types.ObjectVersion{ + ETag: r.res.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: &dataLen, + VersionId: r.res.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: r.res.ChecksumType, + } - result := []types.ObjectVersion{version, v} + result := []types.ObjectVersion{version, v} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - if !compareVersions(result, out.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) - } + if !compareVersions(result, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(v.VersionId), - removeOnlyLeglHold: true, - }, + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(v.VersionId), + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -2947,81 +4668,188 @@ func Versioning_WORM_PutObject_overwrite_locked_object(s *S3Conf) error { func Versioning_WORM_CopyObject_overwrite_locked_object(s *S3Conf) error { testName := "Versioning_WORM_CopyObject_overwrite_locked_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - versions, err := createObjVersions(s3client, bucket, obj, 1) - if err != nil { - return err - } + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - v := versions[0] - v.IsLatest = getPtr(false) + v := versions[0] + v.IsLatest = getPtr(false) - // lock the object with legal hold - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ - Bucket: &bucket, - Key: &obj, - LegalHold: &types.ObjectLockLegalHold{ - Status: types.ObjectLockLegalHoldStatusOn, - }, + // lock the object with legal hold + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } + + // create a source object version, a directory object for a + // directory object destination + srcObj := "source-object" + if strings.HasSuffix(obj, "/") { + srcObj = "source-dir/" + } + srcVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + + srcVersion := srcVersions[0] + + // overwrite the locked object with a new version with CopyObject + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + version := types.ObjectVersion{ + ETag: copyResult.CopyObjectResult.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: srcVersion.Size, + VersionId: copyResult.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: copyResult.CopyObjectResult.ChecksumType, + } + + // the destination and the source versions are listed separately, + // as the bucket also holds the versions of the other keys + for _, result := range [][]types.ObjectVersion{{version, v}, {srcVersion}} { + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: result[0].Key, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(result, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) + } + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: getString(v.VersionId), + removeOnlyLeglHold: true, + }, + }) }) - cancel() - if err != nil { - return err - } + }, withLock()) +} - // create a source object version - srcObj := "source-object" - srcVersions, err := createObjVersions(s3client, bucket, srcObj, 1) - if err != nil { - return err - } +// A copy of a locked object onto itself creates a new version, leaving the +// locked one and its legal hold in place. +func Versioning_WORM_CopyObject_to_itself_locked_object(s *S3Conf) error { + testName := "Versioning_WORM_CopyObject_to_itself_locked_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } - srcVersion := srcVersions[0] + v := versions[0] + v.IsLatest = getPtr(false) + lockedVersionId := getString(v.VersionId) - // overwrite the locked object with a new version with CopyObject - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &obj, - CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)), - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } - version := types.ObjectVersion{ - ETag: copyResult.CopyObjectResult.ETag, - IsLatest: getPtr(true), - Key: &obj, - Size: srcVersion.Size, - VersionId: copyResult.VersionId, - StorageClass: types.ObjectVersionStorageClassStandard, - ChecksumType: copyResult.CopyObjectResult.ChecksumType, - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyResult, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + MetadataDirective: types.MetadataDirectiveReplace, + ContentType: getPtr("application/json"), + }) + cancel() + if err != nil { + return err + } - result := []types.ObjectVersion{version, v, srcVersion} + if getString(copyResult.VersionId) == lockedVersionId { + return fmt.Errorf("expected a new versionId, instead got %v", + getString(copyResult.VersionId)) + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } + version := types.ObjectVersion{ + ETag: copyResult.CopyObjectResult.ETag, + IsLatest: getPtr(true), + Key: &obj, + Size: v.Size, + VersionId: copyResult.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: copyResult.CopyObjectResult.ChecksumType, + } - if !compareVersions(result, out.Versions) { - return fmt.Errorf("expected the object versions to be %v, instead got %v", result, out.Versions) - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } - return cleanupLockedObjects(s3client, bucket, []objToDelete{ - { - key: obj, - versionId: getString(v.VersionId), - removeOnlyLeglHold: true, - }, + if !compareVersions([]types.ObjectVersion{version, v}, out.Versions) { + return fmt.Errorf("expected the object versions to be %v, instead got %v", + []types.ObjectVersion{version, v}, out.Versions) + } + + // the legal hold stays on the version it was set on + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lhRes, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &lockedVersionId, + }) + cancel() + if err != nil { + return err + } + + if lhRes.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %v, instead got %v", + types.ObjectLockLegalHoldStatusOn, lhRes.LegalHold.Status) + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + { + key: obj, + versionId: lockedVersionId, + removeOnlyLeglHold: true, + }, + }) }) }, withLock()) } @@ -3140,328 +4968,202 @@ func Versioning_WORM_remove_delete_marker_under_bucket_default_retention(s *S3Co return err } - obj := "my-object" - versions, err := createObjVersions(s3client, bucket, obj, 3) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } - // Create a delete marker - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - // Delete the delete marker - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - return err - } - - if !compareVersions(versions, resp.Versions) { - return fmt.Errorf("expected the object vresions to be %v, instead got %v", versions, resp.Versions) - } - if len(resp.DeleteMarkers) != 0 { - return fmt.Errorf("expected empty delete markers list, instead got %v", resp.DeleteMarkers) - } - - // - lockedVersions := make([]objToDelete, 0, len(versions)) - for _, v := range versions { - lockedVersions = append(lockedVersions, objToDelete{ - key: obj, - versionId: getString(v.VersionId), - isCompliance: false, + // Create a delete marker + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, }) - } - return cleanupLockedObjects(s3client, bucket, lockedVersions) + cancel() + if err != nil { + return err + } + + // Delete the delete marker + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Prefix: &obj, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, resp.Versions) { + return fmt.Errorf("expected the object vresions to be %v, instead got %v", versions, resp.Versions) + } + if len(resp.DeleteMarkers) != 0 { + return fmt.Errorf("expected empty delete markers list, instead got %v", resp.DeleteMarkers) + } + + // + lockedVersions := make([]objToDelete, 0, len(versions)) + for _, v := range versions { + lockedVersions = append(lockedVersions, objToDelete{ + key: obj, + versionId: getString(v.VersionId), + isCompliance: false, + }) + } + return cleanupLockedObjects(s3client, bucket, lockedVersions) + }) }, withLock()) } +func Versioning_WORM_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_WORM_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rDate := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + lockedObjs := []objToDelete{} + + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &rDate, + }, s3client) + if err != nil { + return err + } + versionId := getString(res.res.VersionId) + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: versionId, + removeOnlyLeglHold: true, + }) + + // the version belongs to the object, not to the other key + err = checkObjectLockErr(s3client, bucket, other, versionId, s3err.GetAPIError(s3err.ErrNoSuchVersion)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &other, + VersionId: &versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } + + err = checkObjectLock(s3client, bucket, obj, versionId, rDate) + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }, withLock()) +} + +func Versioning_WORM_null_version_locked_with_legal_hold(s *S3Conf) error { + testName := "Versioning_WORM_null_version_locked_with_legal_hold" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []string{"my-obj", "my-dir/"} + // the objects are put before versioning is enabled: their + // current versions are the null versions + _, err := putObjects(s3client, objs, bucket) + if err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{ + Bucket: &bucket, + ObjectLockConfiguration: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + }, + }) + cancel() + if err != nil { + return err + } + + lockedObjs := []objToDelete{} + err = forEachKey(objs, func(obj string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + cancel() + if err != nil { + return err + } + lockedObjs = append(lockedObjs, objToDelete{ + key: obj, + versionId: nullVersionId, + removeOnlyLeglHold: true, + }) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + if err != nil { + return err + } + if res.LegalHold == nil || res.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %q, instead got %v", + types.ObjectLockLegalHoldStatusOn, res.LegalHold) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)) + }) + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, lockedObjs) + }) +} + func Versioning_AccessControl_GetObjectVersion(s *S3Conf) error { testName := "Versioning_AccessControl_GetObjectVersion" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objData, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - // querying with versionId should return access denied - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - defer cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - - // grant the user s3:GetObjectVersion - doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - defer cancel() - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_HeadObjectVersion(s *S3Conf) error { - testName := "Versioning_AccessControl_HeadObjectVersion" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - objData, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - // querying with versionId should return access denied - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - cancel() - if err := checkSdkApiErr(err, http.StatusText(http.StatusForbidden)); err != nil { - return err - } - - // grant the user s3:GetObjectVersion - doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ - Bucket: &bucket, - Policy: &doc, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: objData.res.VersionId, - }) - cancel() - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_object_tagging_policy(s *S3Conf) error { - testName := "Versioning_AccessControl_PutObjectTagging_policy" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - object := "my-object" - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &object, - }, s3client) - if err != nil { - return err - } - - testuser := getUser("user") - err = createUsers(s, []user{testuser}) - if err != nil { - return err - } - - userClient := s.getUserClient(testuser) - - putGetDeleteObjectTagging := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - Tagging: &types.Tagging{ - TagSet: []types.Tag{ - {Key: getPtr("key"), Value: getPtr("value")}, - }, - }, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userClient.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &object, - VersionId: versionId, - }) - cancel() - if denyAccess { - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { - return err - } - } else { - if err != nil { - return err - } - } - - return nil - } - - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectVersionTagging", "s3:GetObjectVersionTagging", "s3:DeleteObjectVersionTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny without versionId - err = putGetDeleteObjectTagging(nil, true) - if err != nil { - return err - } - - // allow with versionId - err = putGetDeleteObjectTagging(res.res.VersionId, false) - if err != nil { - return err - } - - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectTagging", "s3:GetObjectTagging", "s3:DeleteObjectTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // allow without versionId - err = putGetDeleteObjectTagging(nil, false) - if err != nil { - return err - } - - // deny with versionId - err = putGetDeleteObjectTagging(res.res.VersionId, true) - if err != nil { - return err - } - - return nil - }, withVersioning(types.BucketVersioningStatusEnabled)) -} - -func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { - testName := "Versioning_AccessControl_DeleteObject_policy" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" testuser := getUser("user") err := createUsers(s, []user{testuser}) if err != nil { @@ -3470,150 +5172,416 @@ func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { userClient := s.getUserClient(testuser) - delObject := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if denyAccess { - return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objData, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err } - return err - } + doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) + // querying with versionId should return access denied + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + defer cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + + // grant the user s3:GetObjectVersion + doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + defer cancel() + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_HeadObjectVersion(s *S3Conf) error { + testName := "Versioning_AccessControl_HeadObjectVersion" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) + userClient := s.getUserClient(testuser) + + return forEachKey([]string{"my-obj", "my-dir/"}, func(obj string) error { + objData, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + doc := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + // querying with versionId should return access denied + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, http.StatusText(http.StatusForbidden)); err != nil { + return err + } + + // grant the user s3:GetObjectVersion + doc = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: objData.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_object_tagging_policy(s *S3Conf) error { + testName := "Versioning_AccessControl_PutObjectTagging_policy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - // deny with versionId - err = delObject(res.res.VersionId, true) + userClient := s.getUserClient(testuser) + + return forEachKey([]string{"my-object", "my-dir/"}, func(object string) error { + res, err := putObjectWithData(objDataLen(object, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &object, + }, s3client) + if err != nil { + return err + } + + putGetDeleteObjectTagging := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + {Key: getPtr("key"), Value: getPtr("value")}, + }, + }, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &object, + VersionId: versionId, + }) + cancel() + if denyAccess { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + } else { + if err != nil { + return err + } + } + + return nil + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectVersionTagging", "s3:GetObjectVersionTagging", "s3:DeleteObjectVersionTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny without versionId + err = putGetDeleteObjectTagging(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = putGetDeleteObjectTagging(res.res.VersionId, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `["s3:PutObjectTagging", "s3:GetObjectTagging", "s3:DeleteObjectTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // allow without versionId + err = putGetDeleteObjectTagging(nil, false) + if err != nil { + return err + } + + // deny with versionId + err = putGetDeleteObjectTagging(res.res.VersionId, true) + if err != nil { + return err + } + + return nil + }) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_AccessControl_DeleteObject_policy(s *S3Conf) error { + testName := "Versioning_AccessControl_DeleteObject_policy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + err := createUsers(s, []user{testuser}) if err != nil { return err } - // allow without versionId - err = delObject(nil, false) - if err != nil { - return err - } + userClient := s.getUserClient(testuser) - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + delObject := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if denyAccess { + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + } - // recreate the object - res, err = putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return err + } - // deny without versionId - err = delObject(nil, true) - if err != nil { - return err - } + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - // allow with versionId - err = delObject(res.res.VersionId, false) - if err != nil { - return err - } + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObject"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } - return nil + // deny with versionId + err = delObject(res.res.VersionId, true) + if err != nil { + return err + } + + // allow without versionId + err = delObject(nil, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:DeleteObjectVersion"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // recreate the object + res, err = putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + // deny without versionId + err = delObject(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = delObject(res.res.VersionId, false) + if err != nil { + return err + } + + return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_AccessControl_GetObjectAttributes_policy(s *S3Conf) error { testName := "Versioning_AccessControl_GetObjectAttributes_policy" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - res, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - testuser := getUser("user") - err = createUsers(s, []user{testuser}) + err := createUsers(s, []user{testuser}) if err != nil { return err } userClient := s.getUserClient(testuser) - getObjectAttr := func(versionId *string, denyAccess bool) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := userClient.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - ObjectAttributes: types.ObjectAttributesChecksum.Values(), - }) - cancel() - if denyAccess { - return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + getObjectAttr := func(versionId *string, denyAccess bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + ObjectAttributes: types.ObjectAttributesChecksum.Values(), + }) + cancel() + if denyAccess { + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + } + + return nil + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny with versionId + err = getObjectAttr(res.res.VersionId, true) + if err != nil { + return err + } + + // allow without versionId + err = getObjectAttr(nil, false) + if err != nil { + return err + } + + policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersionAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + err = putBucketPolicy(s3client, bucket, policy) + if err != nil { + return err + } + + // deny without versionId + err = getObjectAttr(nil, true) + if err != nil { + return err + } + + // allow with versionId + err = getObjectAttr(res.res.VersionId, false) + if err != nil { + return err } return nil - } - - policy := genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny with versionId - err = getObjectAttr(res.res.VersionId, true) - if err != nil { - return err - } - - // allow without versionId - err = getObjectAttr(nil, false) - if err != nil { - return err - } - - policy = genPolicyDoc("Allow", fmt.Sprintf(`"%s"`, testuser.access), `"s3:GetObjectVersionAttributes"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) - err = putBucketPolicy(s3client, bucket, policy) - if err != nil { - return err - } - - // deny without versionId - err = getObjectAttr(nil, true) - if err != nil { - return err - } - - // allow with versionId - err = getObjectAttr(res.res.VersionId, false) - if err != nil { - return err - } - - return nil + }) }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -3754,88 +5722,90 @@ func Versioning_GetObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_PutObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_PutObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &types.Tagging{ - TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, - }, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, + }, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetDeleteObjectTagging_delete_marker(s *S3Conf) error { testName := "Versioning_PutGetDeleteObjectTagging_delete_marker" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + // PutObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // GetObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + // DeleteObjectTagging + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }) - cancel() - if err != nil { - return err - } - - // PutObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - Tagging: &types.Tagging{ - TagSet: []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}}, - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // GetObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { - return err - } - - // DeleteObjectTagging - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: out.VersionId, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) }, withLock()) } @@ -3868,23 +5838,24 @@ func Versioning_PutObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_GetObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_GetObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } @@ -3914,111 +5885,156 @@ func Versioning_DeleteObjectTagging_invalid_versionId(s *S3Conf) error { func Versioning_DeleteObjectTagging_non_existing_object_version(s *S3Conf) error { testName := "Versioning_DeleteObjectTagging_non_existing_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - _, err := putObjectWithData(4, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + _, err := putObjectWithData(objDataLen(obj, 4), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("01K97XE6PJQ1A4X5TJFDHK4EMC"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }) - cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutGetDeleteObjectTagging_success(s *S3Conf) error { testName := "Versioning_PutGetDeleteObjectTagging_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-object" - versions, err := createObjVersions(s3client, bucket, obj, 5) - if err != nil { - return err - } - versionId := versions[2].VersionId + return forEachKey([]string{"my-object", "my-dir/"}, func(obj string) error { + versions, err := createObjVersions(s3client, bucket, obj, 5) + if err != nil { + return err + } + versionId := versions[2].VersionId - tagging := types.Tagging{ - TagSet: []types.Tag{ - {Key: getPtr("key"), Value: getPtr("value")}, - }, - } + tagging := types.Tagging{ + TagSet: []types.Tag{ + {Key: getPtr("key"), Value: getPtr("value")}, + }, + } - compareVersionId := func(expected, input *string) error { - if getString(expected) != getString(input) { - return fmt.Errorf("expected the response versionId to be %s, instead got %s", getString(expected), getString(input)) + compareVersionId := func(expected, input *string) error { + if getString(expected) != getString(input) { + return fmt.Errorf("expected the response versionId to be %s, instead got %s", getString(expected), getString(input)) + } + + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if err := compareVersionId(versionId, res.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + if !areTagsSame(tagging.TagSet, out.TagSet) { + return fmt.Errorf("expected the object version tags to be %v, instead got %v", tagging.TagSet, out.TagSet) + } + if err := compareVersionId(versionId, out.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if err := compareVersionId(versionId, resp.VersionId); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + r, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if len(r.TagSet) != 0 { + return fmt.Errorf("expected empty tag set, instead got %v", r.TagSet) } return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &tagging, - VersionId: versionId, }) - cancel() - if err != nil { - return err - } + }, withVersioning(types.BucketVersioningStatusEnabled)) +} - if err := compareVersionId(versionId, res.VersionId); err != nil { - return err - } +func Versioning_ObjectTagging_trailing_slash_counterpart(s *S3Conf) error { + testName := "Versioning_ObjectTagging_trailing_slash_counterpart" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + tagSet := []types.Tag{{Key: getPtr("key"), Value: getPtr("value")}} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } - if !areTagsSame(tagging.TagSet, out.TagSet) { - return fmt.Errorf("expected the object version tags to be %v, instead got %v", tagging.TagSet, out.TagSet) - } - if err := compareVersionId(versionId, out.VersionId); err != nil { - return err - } + for _, keys := range [][2]string{{"my-dir/", "my-dir"}, {"my-obj", "my-obj/"}} { + obj, other := keys[0], keys[1] + res, err := putObjectWithData(objDataLen(obj, 10), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Tagging: getPtr("key=value"), + }, s3client) + if err != nil { + return err + } + versionId := getString(res.res.VersionId) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } + // the version belongs to the object, not to the other key + err = checkObjectTaggingErr(s3client, bucket, other, versionId, s3err.GetAPIError(s3err.ErrNoSuchVersion)) + if err != nil { + return fmt.Errorf("%v: %w", other, err) + } - if err := compareVersionId(versionId, resp.VersionId); err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - r, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - VersionId: versionId, - }) - cancel() - if err != nil { - return err - } - - if len(r.TagSet) != 0 { - return fmt.Errorf("expected empty tag set, instead got %v", r.TagSet) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + VersionId: &versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: %w", obj, err) + } + if !areTagsSame(out.TagSet, tagSet) { + return fmt.Errorf("%v: expected the tag set to be %v, instead got %v", + obj, tagSet, out.TagSet) + } } return nil