mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-04-21 05:00:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #886 from versity/fix/versioning-null-versionid

Browse Source 
null versionId objects
This commit is contained in:
Ben McClelland
2024-10-11 15:20:01 -07:00
committed by GitHub
parent 6925aae48a f03d600b56
commit 79ece46eae
5 changed files with 524 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
backend/posix/posix.go
View File

@@ -532,20 +532,30 @@ func (p *Posix) GetBucketVersioning(_ context.Context, bucket string) (s3respons
}
// Returns the specified bucket versioning status
func (p *Posix) isBucketVersioningEnabled(ctx context.Context, bucket string) (bool, error) {
func (p *Posix) getBucketVersioningStatus(ctx context.Context, bucket string) (types.BucketVersioningStatus, error) {
res, err := p.GetBucketVersioning(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)) {
return false, nil
return "", nil
}
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)) {
return false, err
return "", err
}
if res.Status != nil {
return *res.Status == types.BucketVersioningStatusEnabled, nil
if res.Status == nil {
return "", nil
}
return false, nil
return *res.Status, nil
}
// Checks if the given bucket versioning status is 'Enabled'
func (p *Posix) isBucketVersioningEnabled(s types.BucketVersioningStatus) bool {
return s == types.BucketVersioningStatusEnabled
}
// Checks if the given bucket versioning status is 'Suspended'
func (p *Posix) isBucketVersioningSuspended(s types.BucketVersioningStatus) bool {
return s == types.BucketVersioningStatusSuspended
}
// Generates the object version path in the versioning directory
@@ -560,6 +570,18 @@ func genObjVersionKey(key string) string {
return filepath.Join(sum[:2], sum[2:4], sum[4:6], sum)
}
// Removes the null versionId object from versioning directory
func (p *Posix) deleteNullVersionIdObject(bucket, key string) error {
versionPath := filepath.Join(p.genObjVersionPath(bucket, key), nullVersionId)
err := os.Remove(versionPath)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}
// Creates a new copy(version) of an object in the versioning directory
func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account) (versionPath string, err error) {
sf, err := os.Open(filepath.Join(bucket, key))
@@ -570,10 +592,13 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun
var versionId string
data, err := p.meta.RetrieveAttribute(sf, bucket, key, versionIdKey)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return versionPath, fmt.Errorf("get object versionId: %w", err)
}
if err == nil {
versionId = string(data)
} else {
versionId = ulid.Make().String()
versionId = nullVersionId
}
attrs, err := p.meta.ListAttributes(bucket, key)
@@ -844,8 +869,81 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
}, nil
}
// First find the null versionId object(if exists)
// before starting the object versions listing
var nullVersionIdObj *types.ObjectVersion
var nullObjDelMarker *types.DeleteMarkerEntry
nf, err := os.Stat(filepath.Join(versionPath, nullVersionId))
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
if err == nil {
isDel, err := p.isObjDeleteMarker(versionPath, nullVersionId)
if err != nil {
return nil, err
}
// Check to see if the null versionId object is delete marker or not
if isDel {
nullObjDelMarker = &types.DeleteMarkerEntry{
VersionId: backend.GetStringPtr("null"),
LastModified: backend.GetTimePtr(nf.ModTime()),
Key: &path,
IsLatest: getBoolPtr(false),
}
} else {
etagBytes, err := p.meta.RetrieveAttribute(nil, versionPath, nullVersionId, etagkey)
if errors.Is(err, fs.ErrNotExist) {
return nil, backend.ErrSkipObj
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get etag: %w", err)
}
// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
// so this will just set etag to "" if its not already set
etag := string(etagBytes)
size := nf.Size()
nullVersionIdObj = &types.ObjectVersion{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(nf.ModTime()),
Size: &size,
VersionId: backend.GetStringPtr("null"),
IsLatest: getBoolPtr(false),
StorageClass: types.ObjectVersionStorageClassStandard,
}
}
}
isNullVersionIdObjFound := nullVersionIdObj != nil || nullObjDelMarker != nil
if len(dirEnts) == 1 && (isNullVersionIdObjFound) {
if nullObjDelMarker != nil {
delMarkers = append(delMarkers, *nullObjDelMarker)
}
if nullVersionIdObj != nil {
objects = append(objects, *nullVersionIdObj)
}
if availableObjCount == 1 {
return &backend.ObjVersionFuncResult{
ObjectVersions: objects,
DelMarkers: delMarkers,
Truncated: true,
NextVersionIdMarker: nullVersionId,
}, nil
}
}
isNullVersionIdObjAdded := false
for i := len(dirEnts) - 1; i >= 0; i-- {
dEntry := dirEnts[i]
// Skip the null versionId object to not
// break the object versions list
if dEntry.Name() == nullVersionId {
continue
}
f, err := dEntry.Info()
if errors.Is(err, fs.ErrNotExist) {
@@ -855,6 +953,29 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
return nil, fmt.Errorf("get fileinfo: %w", err)
}
// If the null versionId object is found, first push it
// by checking its creation date, then continue the adding
if isNullVersionIdObjFound && !isNullVersionIdObjAdded {
if nf.ModTime().After(f.ModTime()) {
if nullVersionIdObj != nil {
objects = append(objects, *nullVersionIdObj)
}
if nullObjDelMarker != nil {
delMarkers = append(delMarkers, *nullObjDelMarker)
}
isNullVersionIdObjAdded = true
if availableObjCount--; availableObjCount == 0 {
return &backend.ObjVersionFuncResult{
ObjectVersions: objects,
DelMarkers: delMarkers,
Truncated: true,
NextVersionIdMarker: nullVersionId,
}, nil
}
}
}
versionId := f.Name()
size := f.Size()
@@ -912,6 +1033,26 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
}
}
// If null versionId object is found but not yet pushed,
// push it after the listing, as it's the oldest object version
if isNullVersionIdObjFound && !isNullVersionIdObjAdded {
if nullVersionIdObj != nil {
objects = append(objects, *nullVersionIdObj)
}
if nullObjDelMarker != nil {
delMarkers = append(delMarkers, *nullObjDelMarker)
}
if availableObjCount--; availableObjCount == 0 {
return &backend.ObjVersionFuncResult{
ObjectVersions: objects,
DelMarkers: delMarkers,
Truncated: true,
NextVersionIdMarker: nullVersionId,
}, nil
}
}
return &backend.ObjVersionFuncResult{
ObjectVersions: objects,
DelMarkers: delMarkers,
@@ -1215,10 +1356,11 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
}
}
vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket)
vStatus, err := p.getBucketVersioningStatus(ctx, bucket)
if err != nil {
return nil, err
}
vEnabled := p.isBucketVersioningEnabled(vStatus)
d, err := os.Stat(objname)
@@ -1830,10 +1972,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err)
}
vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket)
vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket)
if err != nil {
return s3response.CopyObjectResult{}, err
}
vEnabled := p.isBucketVersioningEnabled(vStatus)
if srcVersionId != "" {
if !p.versioningEnabled() || !vEnabled {
@@ -2019,10 +2162,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
}, nil
}
vEnabled, err := p.isBucketVersioningEnabled(ctx, *po.Bucket)
vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket)
if err != nil {
return s3response.PutObjectOutput{}, err
}
vEnabled := p.isBucketVersioningEnabled(vStatus)
// object is file
d, err := os.Stat(name)
@@ -2031,10 +2175,20 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
}
// if the versioninng is enabled first create the file object version
if p.versioningEnabled() && vEnabled && err == nil {
_, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err)
if p.versioningEnabled() && vStatus != "" && err == nil {
var isVersionIdMissing bool
if p.isBucketVersioningSuspended(vStatus) {
vIdBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, versionIdKey)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return s3response.PutObjectOutput{}, fmt.Errorf("get object versionId: %w", err)
}
isVersionIdMissing = len(vIdBytes) == 0
}
if !isVersionIdMissing {
_, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err)
}
}
}
if errors.Is(err, syscall.ENAMETOOLONG) {
@@ -2084,6 +2238,17 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
versionID = ulid.Make().String()
}
// Before finaliazing the object creation remove
// null versionId object from versioning directory
// if it exists and the versioning status is Suspended
if p.isBucketVersioningSuspended(vStatus) {
err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key)
if err != nil {
return s3response.PutObjectOutput{}, err
}
versionID = nullVersionId
}
for k, v := range po.Metadata {
err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key,
fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
@@ -2115,7 +2280,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
}
}
if versionID != "" {
if versionID != "" && versionID != nullVersionId {
err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, versionIdKey, []byte(versionID))
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err)
@@ -2199,10 +2364,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
objpath := filepath.Join(bucket, object)
vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket)
vStatus, err := p.getBucketVersioningStatus(ctx, bucket)
if err != nil {
return nil, err
}
vEnabled := p.isBucketVersioningEnabled(vStatus)
// Directory objects can't have versions
if !isDir && p.versioningEnabled() && vEnabled {
@@ -2254,6 +2420,9 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("get obj versionId: %w", err)
}
if errors.Is(err, meta.ErrNoSuchKey) {
vId = []byte(nullVersionId)
}
if string(vId) == *input.VersionId {
// if the specified VersionId is the same as in the latest version,
@@ -2922,10 +3091,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
return nil, fmt.Errorf("stat bucket: %w", err)
}
vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket)
vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket)
if err != nil {
return nil, err
}
vEnabled := p.isBucketVersioningEnabled(vStatus)
if srcVersionId != "" {
if !p.versioningEnabled() || !vEnabled {

11
cmd/versitygw/test.go
View File

@@ -337,11 +337,22 @@ func extractIntTests() (commands []*cli.Command) {
if debug {
opts = append(opts, integration.WithDebug())
}
if versioningEnabled {
opts = append(opts, integration.WithVersioningEnabled())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
return err
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test the bucket object versioning, if the versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
},
})
}
return

14
tests/integration/group-tests.go
View File

@@ -431,7 +431,8 @@ func TestGetObjectLegalHold(s *S3Conf) {
func TestWORMProtection(s *S3Conf) {
WORMProtection_bucket_object_lock_configuration_compliance_mode(s)
WORMProtection_bucket_object_lock_configuration_governance_mode(s)
WORMProtection_bucket_object_lock_governance_bypass_delete(s)
// WORMProtection_bucket_object_lock_governance_bypass_delete(s)
// WORMProtection_bucket_object_lock_governance_bypass_delete_multiple
WORMProtection_object_lock_retention_compliance_locked(s)
WORMProtection_object_lock_retention_governance_locked(s)
WORMProtection_object_lock_retention_governance_bypass_overwrite(s)
@@ -542,6 +543,10 @@ func TestVersioning(s *S3Conf) {
GetBucketVersioning_non_existing_bucket(s)
GetBucketVersioning_empty_response(s)
GetBucketVersioning_success(s)
// PutObject action
Versioning_PutObject_suspended_null_versionId_obj(s)
Versioning_PutObject_null_versionId_obj(s)
Versioning_PutObject_overwrite_null_versionId_obj(s)
Versioning_PutObject_success(s)
// CopyObject action
Versioning_CopyObject_success(s)
@@ -560,6 +565,7 @@ func TestVersioning(s *S3Conf) {
Versioning_DeleteObject_delete_object_version(s)
Versioning_DeleteObject_non_existing_object(s)
Versioning_DeleteObject_delete_a_delete_marker(s)
Versioning_Delete_null_versionId_object(s)
Versioning_DeleteObjects_success(s)
Versioning_DeleteObjects_delete_deleteMarkers(s)
// ListObjectVersions
@@ -568,6 +574,7 @@ func TestVersioning(s *S3Conf) {
ListObjectVersions_list_multiple_object_versions(s)
ListObjectVersions_multiple_object_versions_truncated(s)
ListObjectVersions_with_delete_markers(s)
ListObjectVersions_containing_null_versionId_obj(s)
// Multipart upload
Versioning_Multipart_Upload_success(s)
Versioning_Multipart_Upload_overwrite_an_object(s)
@@ -922,6 +929,9 @@ func GetIntTests() IntTests {
"GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket,
"GetBucketVersioning_empty_response": GetBucketVersioning_empty_response,
"GetBucketVersioning_success": GetBucketVersioning_success,
"Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj,
"Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj,
"Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj,
"Versioning_PutObject_success": Versioning_PutObject_success,
"Versioning_CopyObject_success": Versioning_CopyObject_success,
"Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id,
@@ -936,6 +946,7 @@ func GetIntTests() IntTests {
"Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version,
"Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object,
"Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker,
"Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object,
"Versioning_DeleteObjects_success": Versioning_DeleteObjects_success,
"Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers,
"ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket,
@@ -943,6 +954,7 @@ func GetIntTests() IntTests {
"ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions,
"ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated,
"ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers,
"ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj,
"Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success,
"Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object,
"Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId,

377
tests/integration/tests.go
View File

@@ -42,6 +42,7 @@ import (
var (
shortTimeout = 10 * time.Second
iso8601Format = "20060102T150405Z"
nullVersionId = "null"
)
func Authentication_empty_auth_header(s *S3Conf) error {
@@ -10602,14 +10603,7 @@ func PutObject_name_too_long(s *S3Conf) error {
func PutBucketVersioning_non_existing_bucket(s *S3Conf) error {
testName := "PutBucketVersioning_non_existing_bucket"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: getPtr(getBucketName()),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
cancel()
err := putBucketVersioningStatus(s3client, getBucketName(), types.BucketVersioningStatusSuspended)
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
return err
}
@@ -10638,14 +10632,7 @@ func HeadObject_name_too_long(s *S3Conf) error {
func PutBucketVersioning_invalid_status(s *S3Conf) error {
testName := "PutBucketVersioning_invalid_status"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatus("invalid_status"),
},
})
cancel()
err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatus("invalid_status"))
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil {
return err
}
@@ -10657,14 +10644,7 @@ func PutBucketVersioning_invalid_status(s *S3Conf) error {
func PutBucketVersioning_success_enabled(s *S3Conf) error {
testName := "PutBucketVersioning_success_enabled"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
cancel()
err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err != nil {
return err
}
@@ -10676,14 +10656,7 @@ func PutBucketVersioning_success_enabled(s *S3Conf) error {
func PutBucketVersioning_success_suspended(s *S3Conf) error {
testName := "PutBucketVersioning_success_suspended"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusSuspended,
},
})
cancel()
err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended)
if err != nil {
return err
}
@@ -10747,7 +10720,148 @@ func GetBucketVersioning_success(s *S3Conf) error {
return fmt.Errorf("expected bucket versioning status to be %v, instead got %v", types.BucketVersioningStatusEnabled, res.Status)
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_PutObject_suspended_null_versionId_obj(s *S3Conf) error {
testName := "Versioning_PutObject_suspended_null_versionId_obj"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
out, err := putObjectWithData(1222, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
if getString(out.res.VersionId) != nullVersionId {
return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", nullVersionId, getString(out.res.VersionId))
}
return nil
}, withVersioning(types.BucketVersioningStatusSuspended))
}
func Versioning_PutObject_null_versionId_obj(s *S3Conf) error {
testName := "Versioning_PutObject_null_versionId_obj"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, lgth := "my-obj", int64(1234)
out, err := putObjectWithData(lgth, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
// Enable bucket versioning
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err != nil {
return err
}
versions, err := createObjVersions(s3client, bucket, obj, 4)
if err != nil {
return err
}
versions = append(versions, types.ObjectVersion{
ETag: out.res.ETag,
IsLatest: getBoolPtr(false),
Key: &obj,
Size: &lgth,
VersionId: &nullVersionId,
StorageClass: types.ObjectVersionStorageClassStandard,
})
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
if !compareVersions(versions, res.Versions) {
return fmt.Errorf("expected the listed versions to be %v, instead got %v", versions, res.Versions)
}
return nil
})
}
func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error {
testName := "Versioning_PutObject_overwrite_null_versionId_obj"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
_, err := putObjectWithData(int64(1233), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
// Enable bucket versioning
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err != nil {
return err
}
versions, err := createObjVersions(s3client, bucket, obj, 4)
if err != nil {
return err
}
// Set bucket versioning status to Suspended
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended)
if err != nil {
return err
}
lgth := int64(3200)
out, err := putObjectWithData(lgth, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
if getString(out.res.VersionId) != nullVersionId {
return fmt.Errorf("expected the uploaded object versionId to be %v, insted got %v", nullVersionId, getString(out.res.VersionId))
}
versions[0].IsLatest = getBoolPtr(false)
versions = append([]types.ObjectVersion{
{
ETag: out.res.ETag,
IsLatest: getBoolPtr(true),
Key: &obj,
Size: &lgth,
VersionId: &nullVersionId,
StorageClass: types.ObjectVersionStorageClassStandard,
},
}, versions...)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
if !compareVersions(versions, res.Versions) {
return fmt.Errorf("expected the listed versions to be %v, instead got %v", versions, res.Versions)
}
return nil
})
}
func Versioning_PutObject_success(s *S3Conf) error {
@@ -10768,7 +10882,7 @@ func Versioning_PutObject_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_CopyObject_success(s *S3Conf) error {
@@ -10840,7 +10954,7 @@ func Versioning_CopyObject_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error {
@@ -10874,14 +10988,14 @@ func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_CopyObject_from_an_object_version(s *S3Conf) error {
testName := "Versioning_CopyObject_from_an_object_version"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
srcBucket, srcObj, dstObj := getBucketName(), "my-obj", "my-dst-obj"
if err := setup(s, srcBucket, withVersioning()); err != nil {
if err := setup(s, srcBucket, withVersioning(types.BucketVersioningStatusEnabled)); err != nil {
return err
}
@@ -10932,7 +11046,7 @@ func Versioning_CopyObject_from_an_object_version(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_CopyObject_special_chars(s *S3Conf) error {
@@ -10985,7 +11099,7 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_HeadObject_invalid_versionId(s *S3Conf) error {
@@ -11063,7 +11177,7 @@ func Versioning_HeadObject_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_HeadObject_delete_marker(s *S3Conf) error {
@@ -11105,7 +11219,7 @@ func Versioning_HeadObject_delete_marker(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_GetObject_invalid_versionId(s *S3Conf) error {
@@ -11133,7 +11247,7 @@ func Versioning_GetObject_invalid_versionId(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_GetObject_success(s *S3Conf) error {
@@ -11209,7 +11323,7 @@ func Versioning_GetObject_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_GetObject_delete_marker(s *S3Conf) error {
@@ -11251,7 +11365,7 @@ func Versioning_GetObject_delete_marker(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_DeleteObject_delete_object_version(s *S3Conf) error {
@@ -11293,7 +11407,7 @@ func Versioning_DeleteObject_delete_object_version(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_DeleteObject_non_existing_object(s *S3Conf) error {
@@ -11323,7 +11437,7 @@ func Versioning_DeleteObject_non_existing_object(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error {
@@ -11375,7 +11489,47 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Delete_null_versionId_object(s *S3Conf) error {
testName := "Versioning_Delete_null_versionId_object"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, nObjLgth := "my-obj", int64(3211)
_, err := putObjectWithData(nObjLgth, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err != nil {
return err
}
_, err = createObjVersions(s3client, bucket, obj, 3)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: &bucket,
Key: &obj,
VersionId: getPtr(nullVersionId),
})
cancel()
if err != nil {
return err
}
if getString(res.VersionId) != nullVersionId {
return fmt.Errorf("expected the versionId to be %v, instead got %v", nullVersionId, getString(res.VersionId))
}
return nil
})
}
func Versioning_DeleteObjects_success(s *S3Conf) error {
@@ -11475,7 +11629,7 @@ func Versioning_DeleteObjects_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error {
@@ -11573,7 +11727,7 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_non_existing_bucket(s *S3Conf) error {
@@ -11589,7 +11743,7 @@ func ListObjectVersions_non_existing_bucket(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_list_single_object_versions(s *S3Conf) error {
@@ -11615,7 +11769,7 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error {
@@ -11652,7 +11806,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
@@ -11735,7 +11889,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_with_delete_markers(s *S3Conf) error {
@@ -11783,7 +11937,76 @@ func ListObjectVersions_with_delete_markers(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error {
testName := "ListObjectVersions_containing_null_versionId_obj"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
versions, err := createObjVersions(s3client, bucket, obj, 3)
if err != nil {
return err
}
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended)
if err != nil {
return err
}
objLgth := int64(543)
out, err := putObjectWithData(objLgth, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
if getString(out.res.VersionId) != nullVersionId {
return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", nullVersionId, getString(out.res.VersionId))
}
versions[0].IsLatest = getBoolPtr(false)
versions = append([]types.ObjectVersion{
{
ETag: out.res.ETag,
IsLatest: getBoolPtr(false),
Key: &obj,
Size: &objLgth,
VersionId: &nullVersionId,
StorageClass: types.ObjectVersionStorageClassStandard,
},
}, versions...)
err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err != nil {
return err
}
newVersions, err := createObjVersions(s3client, bucket, obj, 4)
if err != nil {
return err
}
versions = append(newVersions, versions...)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
if !compareVersions(res.Versions, versions) {
return fmt.Errorf("expected the listed object versions to be %v, instead got %v", versions, res.Versions)
}
return nil
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Multipart_Upload_success(s *S3Conf) error {
@@ -11858,7 +12081,7 @@ func Versioning_Multipart_Upload_success(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error {
@@ -11944,7 +12167,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error {
@@ -11989,7 +12212,7 @@ func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error {
@@ -12059,7 +12282,7 @@ func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Enable_object_lock(s *S3Conf) error {
@@ -12085,14 +12308,7 @@ func Versioning_Enable_object_lock(s *S3Conf) error {
func Versioning_status_switch_to_suspended_with_object_lock(s *S3Conf) error {
testName := "Versioning_status_switch_to_suspended_with_object_lock"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusSuspended,
},
})
cancel()
err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended)
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrSuspendedVersioningNotAllowed)); err != nil {
return err
}
@@ -12127,7 +12343,7 @@ func Versioning_PutObjectRetention_invalid_versionId(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error {
@@ -12151,7 +12367,7 @@ func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Put_GetObjectRetention_success(s *S3Conf) error {
@@ -12200,7 +12416,7 @@ func Versioning_Put_GetObjectRetention_success(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error {
@@ -12227,7 +12443,7 @@ func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error {
@@ -12251,7 +12467,7 @@ func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error {
@@ -12298,7 +12514,7 @@ func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error {
@@ -12341,7 +12557,7 @@ func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error {
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) error {
@@ -12386,7 +12602,7 @@ func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) err
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) error {
@@ -12431,20 +12647,13 @@ func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) err
}
return nil
}, withLock(), withVersioning())
}, withLock(), withVersioning(types.BucketVersioningStatusEnabled))
}
func VersioningDisabled_GetBucketVersioning_not_configured(s *S3Conf) error {
testName := "VersioningDisabled_GetBucketVersioning_not_configured"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
cancel()
err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled)
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)); err != nil {
return err
}
@@ -12528,5 +12737,5 @@ func Versioning_concurrent_upload_object(s *S3Conf) error {
}
return nil
}, withVersioning())
}, withVersioning(types.BucketVersioningStatusEnabled))
}

27
tests/integration/utils.go
View File

@@ -74,12 +74,12 @@ func setup(s *S3Conf, bucket string, opts ...setupOpt) error {
return err
}
if cfg.VersioningEnabled {
if cfg.VersioningStatus != "" {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
Status: cfg.VersioningStatus,
},
})
cancel()
@@ -172,9 +172,9 @@ func teardown(s *S3Conf, bucket string) error {
}
type setupCfg struct {
LockEnabled bool
VersioningEnabled bool
Ownership types.ObjectOwnership
LockEnabled bool
VersioningStatus types.BucketVersioningStatus
Ownership types.ObjectOwnership
}
type setupOpt func(*setupCfg)
@@ -185,8 +185,8 @@ func withLock() setupOpt {
func withOwnership(o types.ObjectOwnership) setupOpt {
return func(s *setupCfg) { s.Ownership = o }
}
func withVersioning() setupOpt {
return func(s *setupCfg) { s.VersioningEnabled = true }
func withVersioning(v types.BucketVersioningStatus) setupOpt {
return func(s *setupCfg) { s.VersioningStatus = v }
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error {
@@ -885,6 +885,19 @@ func changeBucketObjectLockStatus(client *s3.Client, bucket string, status bool)
return nil
}
func putBucketVersioningStatus(client *s3.Client, bucket string, status types.BucketVersioningStatus) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: status,
},
})
cancel()
return err
}
func checkWORMProtection(client *s3.Client, bucket, object string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := client.PutObject(ctx, &s3.PutObjectInput{