diff --git a/auth/bucket_policy_actions.go b/auth/bucket_policy_actions.go index 22510dc7..7e468fd2 100644 --- a/auth/bucket_policy_actions.go +++ b/auth/bucket_policy_actions.go @@ -36,6 +36,7 @@ const ( ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads" PutObjectAction Action = "s3:PutObject" GetObjectAction Action = "s3:GetObject" + GetObjectVersionAction Action = "s3:GetObjectVersion" DeleteObjectAction Action = "s3:DeleteObject" GetObjectAclAction Action = "s3:GetObjectAcl" GetObjectAttributesAction Action = "s3:GetObjectAttributes" @@ -75,6 +76,7 @@ var supportedActionList = map[Action]struct{}{ ListBucketMultipartUploadsAction: {}, PutObjectAction: {}, GetObjectAction: {}, + GetObjectVersionAction: {}, DeleteObjectAction: {}, GetObjectAclAction: {}, GetObjectAttributesAction: {}, @@ -103,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{ ListMultipartUploadPartsAction: {}, PutObjectAction: {}, GetObjectAction: {}, + GetObjectVersionAction: {}, DeleteObjectAction: {}, GetObjectAclAction: {}, GetObjectAttributesAction: {}, diff --git a/backend/azure/azure.go b/backend/azure/azure.go index c0c13cd6..00551950 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -284,10 +284,10 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership)) } -func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { +func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { tags, err := parseTags(po.Tagging) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } opts := &blockblob.UploadStreamOptions{ @@ -312,14 +312,14 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts) if err != nil { - return "", azureErrToS3Err(err) + return s3response.PutObjectOutput{}, azureErrToS3Err(err) } // Set object legal hold if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } @@ -331,15 +331,17 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, } retParsed, err := json.Marshal(retention) if err != nil { - return "", fmt.Errorf("parse object lock retention: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) } err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } - return string(*uploadResp.ETag), nil + return s3response.PutObjectOutput{ + ETag: string(*uploadResp.ETag), + }, nil } func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error { @@ -675,22 +677,22 @@ Pager: }, nil } -func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { +func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) if err != nil { azerr, ok := err.(*azcore.ResponseError) if ok && azerr.StatusCode == 404 { // if the object does not exist, S3 returns success - return nil + return &s3.DeleteObjectOutput{}, nil } } - return azureErrToS3Err(err) + return &s3.DeleteObjectOutput{}, azureErrToS3Err(err) } func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { delResult, errs := []types.DeletedObject{}, []types.Error{} for _, obj := range input.Delete.Objects { - err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ + _, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: input.Bucket, Key: obj.Key, }) diff --git a/backend/backend.go b/backend/backend.go index 3976be21..0e52dfb6 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -38,7 +38,7 @@ type Backend interface { CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error PutBucketAcl(_ context.Context, bucket string, data []byte) error DeleteBucket(context.Context, *s3.DeleteBucketInput) error - PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error + PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) @@ -57,7 +57,7 @@ type Backend interface { UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) // standard object operations - PutObject(context.Context, *s3.PutObjectInput) (string, error) + PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) @@ -65,10 +65,10 @@ type Backend interface { CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) - DeleteObject(context.Context, *s3.DeleteObjectInput) error + DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error - ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) + ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) // special case object operations RestoreObject(context.Context, *s3.RestoreObjectInput) error @@ -126,7 +126,7 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data [] func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error { +func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { @@ -173,8 +173,8 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) { - return "", s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -197,8 +197,8 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3 func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error { - return s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -225,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel } } -func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) { diff --git a/backend/common.go b/backend/common.go index 1f421c7e..c4d9132a 100644 --- a/backend/common.go +++ b/backend/common.go @@ -102,6 +102,32 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) { return startOffset, endOffset - startOffset + 1, nil } +// ParseCopySource parses x-amz-copy-source header and returns source bucket, +// source object, versionId, error respectively +func ParseCopySource(copySourceHeader string) (string, string, string, error) { + if copySourceHeader[0] == '/' { + copySourceHeader = copySourceHeader[1:] + } + + cSplitted := strings.Split(copySourceHeader, "?") + copySource := cSplitted[0] + var versionId string + if len(cSplitted) > 1 { + versionIdParts := strings.Split(cSplitted[1], "=") + if len(versionIdParts) != 2 || versionIdParts[0] != "versionId" { + return "", "", "", s3err.GetAPIError(s3err.ErrInvalidRequest) + } + versionId = versionIdParts[1] + } + + srcBucket, srcObject, ok := strings.Cut(copySource, "/") + if !ok { + return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + return srcBucket, srcObject, versionId, nil +} + func CreateExceedingRangeErr(objSize int64) s3err.APIError { return s3err.APIError{ Code: "InvalidArgument", diff --git a/backend/posix/posix.go b/backend/posix/posix.go index ad56b363..be6a91b3 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -35,6 +35,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" + "github.com/oklog/ulid/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/backend/meta" @@ -64,6 +65,9 @@ type Posix struct { // bucketlinks is a flag to enable symlinks to directories at the top // level gateway directory to be treated as buckets the same as directories bucketlinks bool + + // bucket versioning directory path + versioningDir string } var _ backend.Backend = &Posix{} @@ -84,15 +88,19 @@ const ( bucketLockKey = "bucket-lock" objectRetentionKey = "object-retention" objectLegalHoldKey = "object-legal-hold" + versioningKey = "versioning" + deleteMarkerKey = "delete-marker" + versionIdKey = "version-id" doFalloc = true skipFalloc = false ) type PosixOpts struct { - ChownUID bool - ChownGID bool - BucketLinks bool + ChownUID bool + ChownGID bool + BucketLinks bool + VersioningDir string } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { @@ -106,15 +114,57 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro return nil, fmt.Errorf("open %v: %w", rootdir, err) } + var verioningdirAbs string + + // Ensure the versioning directory isn't within the root directory + if opts.VersioningDir != "" { + rootdirAbs, err := filepath.Abs(rootdir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err) + } + + verioningdirAbs, err = filepath.Abs(opts.VersioningDir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", opts.VersioningDir, err) + } + + // Ensure the paths end with a separator + if !strings.HasSuffix(rootdirAbs, string(filepath.Separator)) { + rootdirAbs += string(filepath.Separator) + } + + if !strings.HasSuffix(verioningdirAbs, string(filepath.Separator)) { + verioningdirAbs += string(filepath.Separator) + } + + // Ensure the posix root directory doesn't contain the versioning directory + if strings.HasPrefix(verioningdirAbs, rootdirAbs) { + return nil, fmt.Errorf("the root directory %v contains the versioning directory %v", rootdir, opts.VersioningDir) + } + + vDir, err := os.Stat(verioningdirAbs) + if err != nil { + return nil, fmt.Errorf("stat versioning dir: %w", err) + } + + // Check the versioning path to be a directory + if !vDir.IsDir() { + return nil, fmt.Errorf("versioning path should be a directory") + } + + fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs) + } + return &Posix{ - meta: meta, - rootfd: f, - rootdir: rootdir, - euid: os.Geteuid(), - egid: os.Getegid(), - chownuid: opts.ChownUID, - chowngid: opts.ChownGID, - bucketlinks: opts.BucketLinks, + meta: meta, + rootfd: f, + rootdir: rootdir, + euid: os.Geteuid(), + egid: os.Getegid(), + chownuid: opts.ChownUID, + chowngid: opts.ChownGID, + bucketlinks: opts.BucketLinks, + versioningDir: verioningdirAbs, }, nil } @@ -126,6 +176,11 @@ func (p *Posix) String() string { return "Posix Gateway" } +// returns the versioning state +func (p *Posix) versioningEnabled() bool { + return p.versioningDir != "" +} + func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) { entries, err := os.ReadDir(".") if err != nil { @@ -380,6 +435,451 @@ func (p *Posix) DeleteBucketOwnershipControls(_ context.Context, bucket string) return nil } +func (p *Posix) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil + } + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return fmt.Errorf("stat bucket: %w", err) + } + + // Store 1 bit for bucket versioning state + var versioning []byte + switch status { + case types.BucketVersioningStatusEnabled: + // '1' maps to 'Enabled' + versioning = []byte{1} + case types.BucketVersioningStatusSuspended: + // '0' maps to 'Suspended' + versioning = []byte{0} + } + + if err := p.meta.StoreAttribute(bucket, "", versioningKey, versioning); err != nil { + return fmt.Errorf("set versioning: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { + if !p.versioningEnabled() { + // AWS returns empty response, if versioning is not set + //TODO: Maybe we need to return our custom error here? + return &s3.GetBucketVersioningOutput{}, nil + } + + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + vData, err := p.meta.RetrieveAttribute(bucket, "", versioningKey) + if errors.Is(err, meta.ErrNoSuchKey) { + return &s3.GetBucketVersioningOutput{}, nil + } + + switch vData[0] { + case 1: + return &s3.GetBucketVersioningOutput{ + Status: types.BucketVersioningStatusEnabled, + }, nil + case 0: + return &s3.GetBucketVersioningOutput{ + Status: types.BucketVersioningStatusSuspended, + }, nil + } + + return &s3.GetBucketVersioningOutput{}, nil +} + +// Returns the specified bucket versioning status +func (p *Posix) isBucketVersioningEnabled(ctx context.Context, bucket string) (bool, error) { + res, err := p.GetBucketVersioning(ctx, bucket) + if err != nil { + return false, err + } + + return res.Status == types.BucketVersioningStatusEnabled, nil +} + +// Generates the object version path in the versioning directory +func (p *Posix) genObjVersionPath(bucket, key string) string { + return filepath.Join(p.versioningDir, bucket, genObjVersionKey(key)) +} + +// Generates the versioning path for the given object key +func genObjVersionKey(key string) string { + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) + + return filepath.Join(sum[:2], sum[2:4], sum[4:6], sum) +} + +// Creates a new copy(version) of an object in the versioning directory +func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account) (versionPath string, err error) { + sf, err := os.Open(filepath.Join(bucket, key)) + if err != nil { + return "", err + } + defer sf.Close() + + var versionId string + data, err := p.meta.RetrieveAttribute(bucket, key, versionIdKey) + if err == nil { + versionId = string(data) + } else { + versionId = ulid.Make().String() + } + + attrs, err := p.meta.ListAttributes(bucket, key) + if err != nil { + return versionPath, fmt.Errorf("load object attributes: %w", err) + } + + versionBucketPath := filepath.Join(p.versioningDir, bucket) + versioningKey := filepath.Join(genObjVersionKey(key), versionId) + versionTmpPath := filepath.Join(versionBucketPath, metaTmpDir) + f, err := p.openTmpFile(versionTmpPath, versionBucketPath, versioningKey, size, acc, doFalloc) + if err != nil { + return versionPath, err + } + defer f.cleanup() + + _, err = io.Copy(f.File(), sf) + if err != nil { + return versionPath, err + } + + versionPath = filepath.Join(versionBucketPath, versioningKey) + + err = os.MkdirAll(filepath.Join(versionBucketPath, genObjVersionKey(key)), defaultDirPerm) + if err != nil { + return versionPath, err + } + + if err := f.link(); err != nil { + return versionPath, err + } + + // Copy the object attributes(metadata) + for _, attr := range attrs { + data, err := p.meta.RetrieveAttribute(bucket, key, attr) + if err != nil { + return versionPath, fmt.Errorf("list %v attribute: %w", attr, err) + } + + if err := p.meta.StoreAttribute(versionPath, "", attr, data); err != nil { + return versionPath, fmt.Errorf("store %v attribute: %w", attr, err) + } + } + + return versionPath, nil +} + +func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + if !p.versioningEnabled() { + return s3response.ListVersionsResult{}, nil + } + bucket := *input.Bucket + var prefix, delim, keyMarker, versionIdMarker string + var max int + + if input.Prefix != nil { + prefix = *input.Prefix + } + if input.Delimiter != nil { + delim = *input.Delimiter + } + if input.KeyMarker != nil { + keyMarker = *input.KeyMarker + } + if input.VersionIdMarker != nil { + versionIdMarker = *input.VersionIdMarker + } + if input.MaxKeys != nil { + max = int(*input.MaxKeys) + } + + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return s3response.ListVersionsResult{}, fmt.Errorf("stat bucket: %w", err) + } + + fileSystem := os.DirFS(bucket) + results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max, + p.fileToObjVersions(bucket), []string{metaTmpDir}) + if err != nil { + return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err) + } + + return s3response.ListVersionsResult{ + CommonPrefixes: results.CommonPrefixes, + DeleteMarkers: results.DelMarkers, + Delimiter: &delim, + IsTruncated: &results.Truncated, + KeyMarker: &keyMarker, + MaxKeys: input.MaxKeys, + Name: input.Bucket, + NextKeyMarker: &results.NextMarker, + NextVersionIdMarker: &results.NextVersionIdMarker, + Prefix: &prefix, + VersionIdMarker: &versionIdMarker, + Versions: results.ObjectVersions, + }, nil +} + +func getBoolPtr(b bool) *bool { + return &b +} + +// Check if the given object is a delete marker +func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { + _, err := p.meta.RetrieveAttribute(bucket, object, deleteMarkerKey) + if errors.Is(err, fs.ErrNotExist) { + return false, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get object delete-marker: %w", err) + } + + return true, nil +} + +// Converts the file to object version. Finds all the object versions, +// delete markers from the versioning directory and returns +func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { + return func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*backend.ObjVersionFuncResult, error) { + var objects []types.ObjectVersion + var delMarkers []types.DeleteMarkerEntry + // if the number of available objects is 0, return truncated response + if availableObjCount <= 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + }, nil + } + if d.IsDir() { + // directory object only happens if directory empty + // check to see if this is a directory object by checking etag + etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) + if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil { + return nil, fmt.Errorf("get etag: %w", err) + } + etag := string(etagBytes) + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + key := path + "/" + // Directory objects don't contain data + size := int64(0) + versionId := "null" + + objects = append(objects, types.ObjectVersion{ + ETag: &etag, + Key: &key, + LastModified: backend.GetTimePtr(fi.ModTime()), + IsLatest: getBoolPtr(true), + Size: &size, + VersionId: &versionId, + }) + + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: availableObjCount == 1, + }, nil + } + + // file object, get object info and fill out object data + etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + // If the object doesn't have versionId, it's 'null' + versionId := "null" + versionIdBytes, err := p.meta.RetrieveAttribute(bucket, path, versionIdKey) + if err == nil { + versionId = string(versionIdBytes) + } + if versionId == versionIdMarker { + *pastVersionIdMarker = true + } + if *pastVersionIdMarker { + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + size := fi.Size() + + isDel, err := p.isObjDeleteMarker(bucket, path) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + IsLatest: getBoolPtr(true), + VersionId: &versionId, + LastModified: backend.GetTimePtr(fi.ModTime()), + Key: &path, + }) + } else { + objects = append(objects, types.ObjectVersion{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(true), + }) + } + + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + } + + if !p.versioningEnabled() { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + + // List all the versions of the object in the versioning directory + versionPath := p.genObjVersionPath(bucket, path) + dirEnts, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(dirEnts) == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + + for i := len(dirEnts) - 1; i >= 0; i-- { + dEntry := dirEnts[i] + + f, err := dEntry.Info() + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + versionId := f.Name() + size := f.Size() + + if !*pastVersionIdMarker { + if versionId == versionIdMarker { + *pastVersionIdMarker = true + } + continue + } + + etagBytes, err := p.meta.RetrieveAttribute(versionPath, versionId, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + isDel, err := p.isObjDeleteMarker(versionPath, versionId) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + VersionId: &versionId, + LastModified: backend.GetTimePtr(f.ModTime()), + Key: &path, + IsLatest: getBoolPtr(false), + }) + } else { + objects = append(objects, types.ObjectVersion{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(f.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(false), + }) + } + + // if the available object count reaches to 0, return truncated response with nextVersionIdMarker + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + } + + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } +} + func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if mpu.Bucket == nil { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) @@ -673,11 +1173,37 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM return nil, err } } + + vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket) + if err != nil { + return nil, err + } + + d, err := os.Stat(objname) + + // if the versioninng is enabled first create the file object version + if p.versioningEnabled() && vEnabled && err == nil && !d.IsDir() { + _, err := p.createObjVersion(bucket, object, d.Size(), acct) + if err != nil { + return nil, fmt.Errorf("create object version: %w", err) + } + } + err = f.link() if err != nil { return nil, fmt.Errorf("link object in namespace: %w", err) } + // if the versioning is enabled, generate a new versionID for the object + var versionID string + if p.versioningEnabled() && vEnabled { + versionID = ulid.Make().String() + + if err := p.meta.StoreAttribute(bucket, object, versionIdKey, []byte(versionID)); err != nil { + return nil, fmt.Errorf("set versionId attr: %w", err) + } + } + for k, v := range userMetaData { err = p.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) if err != nil { @@ -761,9 +1287,10 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM os.Remove(filepath.Join(bucket, objdir)) return &s3.CompleteMultipartUploadOutput{ - Bucket: &bucket, - ETag: &s3MD5, - Key: &object, + Bucket: &bucket, + ETag: &s3MD5, + Key: &object, + VersionId: &versionID, }, nil } @@ -1258,14 +1785,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) - substrs := strings.SplitN(*upi.CopySource, "/", 2) - if len(substrs) != 2 { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidCopySource) + srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) + if err != nil { + return s3response.CopyObjectResult{}, err } - srcBucket := substrs[0] - srcObject := substrs[1] - _, err = os.Stat(srcBucket) if errors.Is(err, fs.ErrNotExist) { return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1274,9 +1798,35 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) } + vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket) + if err != nil { + return s3response.CopyObjectResult{}, err + } + + if srcVersionId != "" { + if !p.versioningEnabled() || !vEnabled { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + vId, err := p.meta.RetrieveAttribute(srcBucket, srcObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectResult{}, fmt.Errorf("get src object version id: %w", err) + } + + if string(vId) != srcVersionId { + srcBucket = filepath.Join(p.versioningDir, srcBucket) + srcObject = filepath.Join(genObjVersionKey(srcObject), srcVersionId) + } + } + objPath := filepath.Join(srcBucket, srcObject) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { + if p.versioningEnabled() && vEnabled { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchVersion) + } return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -1348,32 +1898,33 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } return s3response.CopyObjectResult{ - ETag: etag, - LastModified: fi.ModTime(), + ETag: etag, + LastModified: fi.ModTime(), + CopySourceVersionId: srcVersionId, }, nil } -func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { +func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if po.Bucket == nil { - return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if po.Key == nil { - return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } tagsStr := getString(po.Tagging) tags := make(map[string]string) _, err := os.Stat(*po.Bucket) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return "", fmt.Errorf("stat bucket: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("stat bucket: %w", err) } if tagsStr != "" { @@ -1381,10 +1932,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e for _, prt := range tagParts { p := strings.Split(prt, "=") if len(p) != 2 { - return "", s3err.GetAPIError(s3err.ErrInvalidTag) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidTag) } if len(p[0]) > 128 || len(p[1]) > 256 { - return "", s3err.GetAPIError(s3err.ErrInvalidTag) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidTag) } tags[p[0]] = p[1] } @@ -1404,22 +1955,22 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e // posix directories can't contain data, send error // if reuests has a data payload associated with a // directory object - return "", s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) } err = backend.MkdirAll(name, uid, gid, doChown) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", err + return s3response.PutObjectOutput{}, err } for k, v := range po.Metadata { err := p.meta.StoreAttribute(*po.Bucket, *po.Key, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) if err != nil { - return "", fmt.Errorf("set user attr %q: %w", k, err) + return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err) } } @@ -1427,31 +1978,47 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(emptyMD5)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } - return emptyMD5, nil + // for directory object no version is created + return s3response.PutObjectOutput{ + ETag: emptyMD5, + }, nil + } + + vEnabled, err := p.isBucketVersioningEnabled(ctx, *po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err } // object is file d, err := os.Stat(name) if err == nil && d.IsDir() { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // if the versioninng is enabled first create the file object version + if p.versioningEnabled() && vEnabled && err == nil { + _, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err) + } } if errors.Is(err, syscall.ENAMETOOLONG) { - return "", s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil && !errors.Is(err, fs.ErrNotExist) { - return "", fmt.Errorf("stat object: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("stat object: %w", err) } f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), *po.Bucket, *po.Key, contentLength, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("open temp file: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() @@ -1460,28 +2027,28 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e _, err = io.Copy(f, rdr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("write object data: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("write object data: %w", err) } dir := filepath.Dir(name) if dir != "" { err = backend.MkdirAll(dir, uid, gid, doChown) if err != nil { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } } err = f.link() if err != nil { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } for k, v := range po.Metadata { err := p.meta.StoreAttribute(*po.Bucket, *po.Key, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) if err != nil { - return "", fmt.Errorf("set user attr %q: %w", k, err) + return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err) } } @@ -1489,15 +2056,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e if tagsStr != "" { err := p.PutObjectTagging(ctx, *po.Bucket, *po.Key, tags) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } // Set object legal hold if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) - if err != nil { - return "", err + if err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil { + return s3response.PutObjectOutput{}, err } } @@ -1509,11 +2075,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } retParsed, err := json.Marshal(retention) if err != nil { - return "", fmt.Errorf("parse object lock retention: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) } - err = p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed) - if err != nil { - return "", err + if err := p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil { + return s3response.PutObjectOutput{}, err } } @@ -1521,7 +2086,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e etag := hex.EncodeToString(dataSum[:]) err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(etag)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } ctype := getString(po.ContentType) @@ -1529,7 +2094,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentTypeHdr, []byte(*po.ContentType)) if err != nil { - return "", fmt.Errorf("set content-type attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) } } @@ -1538,64 +2103,246 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentEncHdr, []byte(*po.ContentEncoding)) if err != nil { - return "", fmt.Errorf("set content-encoding attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set content-encoding attr: %w", err) } } - return etag, nil + // if the versioning is enabled, generate a new versionID for the object + var versionID string + if p.versioningEnabled() && vEnabled { + versionID = ulid.Make().String() + + if err := p.meta.StoreAttribute(*po.Bucket, *po.Key, versionIdKey, []byte(versionID)); err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + return s3response.PutObjectOutput{ + ETag: etag, + VersionID: versionID, + }, nil } -func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error { +func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { if input.Bucket == nil { - return s3err.GetAPIError(s3err.ErrInvalidBucketName) + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if input.Key == nil { - return s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } bucket := *input.Bucket object := *input.Key + isDir := strings.HasSuffix(object, "/") _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return s3err.GetAPIError(s3err.ErrNoSuchBucket) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return fmt.Errorf("stat bucket: %w", err) + return nil, fmt.Errorf("stat bucket: %w", err) } objpath := filepath.Join(bucket, object) - fi, err := os.Stat(objpath) + vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket) if err != nil { - // AWS returns success if the object does not exist or - // is invalid somehow. - // TODO: log if !errors.Is(err, fs.ErrNotExist) somewhere? + return nil, err + } - return nil + // Directory objects can't have versions + if !isDir && p.versioningEnabled() && vEnabled { + if getString(input.VersionId) == "" { + // if the versionId is not specified, make the current version a delete marker + fi, err := os.Stat(objpath) + if errors.Is(err, syscall.ENAMETOOLONG) { + return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) + } + if err != nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + // Creates a new version in the versioning directory + _, err = p.createObjVersion(bucket, object, fi.Size(), acct) + if err != nil { + return nil, err + } + + // Mark the object as a delete marker + if err := p.meta.StoreAttribute(bucket, object, deleteMarkerKey, []byte{}); err != nil { + return nil, fmt.Errorf("set delete marker: %w", err) + } + // Generate & set a unique versionId for the delete marker + versionId := ulid.Make().String() + if err := p.meta.StoreAttribute(bucket, object, versionIdKey, []byte(versionId)); err != nil { + return nil, fmt.Errorf("set versionId: %w", err) + } + + return &s3.DeleteObjectOutput{ + VersionId: &versionId, + }, nil + } else { + versionPath := p.genObjVersionPath(bucket, object) + + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + + if string(vId) == *input.VersionId { + // if the specified VersionId is the same as in the latest version, + // remove the latest version, find the latest version from the versioning + // directory and move to the place of the deleted object, to make it the latest + + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + err = os.Remove(objpath) + if err != nil { + return nil, fmt.Errorf("remove obj version: %w", err) + } + + ents, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + + srcObjVersion, err := ents[len(ents)-1].Info() + if err != nil { + return nil, fmt.Errorf("get file info: %w", err) + } + srcVersionId := srcObjVersion.Name() + sf, err := os.Open(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return nil, fmt.Errorf("open obj version: %w", err) + } + defer sf.Close() + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, srcObjVersion.Size(), acct, doFalloc) + if err != nil { + return nil, fmt.Errorf("open tmp file: %w", err) + } + defer f.cleanup() + + _, err = io.Copy(f, sf) + if err != nil { + return nil, fmt.Errorf("copy object %w", err) + } + + if err := f.link(); err != nil { + return nil, fmt.Errorf("link tmp file: %w", err) + } + + attrs, err := p.meta.ListAttributes(versionPath, srcVersionId) + if err != nil { + return nil, fmt.Errorf("list object attributes: %w", err) + } + + for _, attr := range attrs { + data, err := p.meta.RetrieveAttribute(versionPath, srcVersionId, attr) + if err != nil { + return nil, fmt.Errorf("load %v attribute", attr) + } + + if err := p.meta.StoreAttribute(bucket, object, attr, data); err != nil { + return nil, fmt.Errorf("store %v attribute", attr) + } + } + + if err := os.Remove(filepath.Join(versionPath, srcVersionId)); err != nil { + return nil, fmt.Errorf("remove obj version %w", err) + } + + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + + isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId) + + err = os.Remove(filepath.Join(versionPath, *input.VersionId)) + if errors.Is(err, syscall.ENAMETOOLONG) { + return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) + } + if errors.Is(err, fs.ErrNotExist) { + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + if err != nil { + return nil, fmt.Errorf("delete object: %w", err) + } + + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + } + + fi, err := os.Stat(objpath) + if errors.Is(err, syscall.ENAMETOOLONG) { + return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) + } + if errors.Is(err, fs.ErrNotExist) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + if err != nil { + return &s3.DeleteObjectOutput{}, fmt.Errorf("stat object: %w", err) } if strings.HasSuffix(object, "/") && !fi.IsDir() { // requested object is expecting a directory with a trailing // slash, but the object is not a directory. treat this as // a non-existent object. // AWS returns success if the object does not exist - return nil + return &s3.DeleteObjectOutput{}, nil } err = os.Remove(objpath) if errors.Is(err, fs.ErrNotExist) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { - return fmt.Errorf("delete object: %w", err) + return nil, fmt.Errorf("delete object: %w", err) } err = p.meta.DeleteAttributes(bucket, object) if err != nil { - return fmt.Errorf("delete object attributes: %w", err) + return nil, fmt.Errorf("delete object attributes: %w", err) } - return p.removeParents(bucket, object) + err = p.removeParents(bucket, object) + if err != nil { + return nil, err + } + + return &s3.DeleteObjectOutput{}, nil } func (p *Posix) removeParents(bucket, object string) error { @@ -1608,7 +2355,7 @@ func (p *Posix) removeParents(bucket, object string) error { for { parent := filepath.Dir(objPath) - if parent == "." { + if parent == string(filepath.Separator) || parent == "." { // stop removing parents if we hit the bucket directory. break } @@ -1636,12 +2383,23 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) delResult, errs := []types.DeletedObject{}, []types.Error{} for _, obj := range input.Delete.Objects { //TODO: Make the delete operation concurrent - err := p.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: input.Bucket, - Key: obj.Key, + res, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: input.Bucket, + Key: obj.Key, + VersionId: obj.VersionId, }) if err == nil { - delResult = append(delResult, types.DeletedObject{Key: obj.Key}) + delEntity := types.DeletedObject{ + Key: obj.Key, + DeleteMarker: res.DeleteMarker, + } + if delEntity.DeleteMarker != nil && *delEntity.DeleteMarker { + delEntity.DeleteMarkerVersionId = res.VersionId + } else { + delEntity.VersionId = res.VersionId + } + + delResult = append(delResult, delEntity) } else { serr, ok := err.(s3err.APIError) if ok { @@ -1677,6 +2435,11 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, s3err.GetAPIError(s3err.ErrInvalidRange) } + if !p.versioningEnabled() && *input.VersionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + bucket := *input.Bucket _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1687,10 +2450,32 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } object := *input.Key + if *input.VersionId != "" { + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + + if string(vId) != *input.VersionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + } + objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { + if *input.VersionId != "" { + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -1704,6 +2489,21 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if *input.VersionId != "" { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + return &s3.GetObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fi.ModTime()), + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } + } + acceptRange := *input.Range startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange) if err != nil { @@ -1764,6 +2564,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO TagCount: tagCount, ContentRange: &contentRange, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, }, nil } @@ -1808,6 +2609,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO TagCount: tagCount, ContentRange: &contentRange, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, Body: &backend.FileSectionReadCloser{R: rdr, F: f}, }, nil } @@ -1819,6 +2621,12 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. if input.Key == nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + + if !p.versioningEnabled() && *input.VersionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + bucket := *input.Bucket object := *input.Key @@ -1874,10 +2682,32 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, fmt.Errorf("stat bucket: %w", err) } + if *input.VersionId != "" { + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + + if string(vId) != *input.VersionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + } + objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { + if *input.VersionId != "" { + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -1890,6 +2720,21 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if *input.VersionId != "" { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + return &s3.HeadObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fi.ModTime()), + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } + } + userMetaData := make(map[string]string) contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) @@ -1939,13 +2784,15 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ObjectLockMode: objectLockMode, ObjectLockRetainUntilDate: objectLockRetainUntilDate, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, }, nil } func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) { data, err := p.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: input.Bucket, - Key: input.Key, + Bucket: input.Bucket, + Key: input.Key, + VersionId: input.VersionId, }) if err != nil { return s3response.GetObjectAttributesResult{}, nil @@ -1972,14 +2819,15 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. if input.ExpectedBucketOwner == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) } - srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/") - if !ok { - return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) + + srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*input.CopySource) + if err != nil { + return nil, err } dstBucket := *input.Bucket dstObject := *input.Key - _, err := os.Stat(srcBucket) + _, err = os.Stat(srcBucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } @@ -1987,6 +2835,29 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return nil, fmt.Errorf("stat bucket: %w", err) } + vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket) + if err != nil { + return nil, err + } + + if srcVersionId != "" { + if !p.versioningEnabled() || !vEnabled { + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + vId, err := p.meta.RetrieveAttribute(srcBucket, srcObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get src object version id: %w", err) + } + + if string(vId) != srcVersionId { + srcBucket = filepath.Join(p.versioningDir, srcBucket) + srcObject = filepath.Join(genObjVersionKey(srcObject), srcVersionId) + } + } + _, err = os.Stat(dstBucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1998,6 +2869,9 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. objPath := filepath.Join(srcBucket, srcObject) f, err := os.Open(objPath) if errors.Is(err, fs.ErrNotExist) { + if p.versioningEnabled() && vEnabled { + return nil, s3err.GetAPIError(s3err.ErrNoSuchVersion) + } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -2020,6 +2894,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. p.loadUserMetaData(srcBucket, srcObject, mdmap) var etag string + var version *string dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -2044,10 +2919,14 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. b, _ := p.meta.RetrieveAttribute(dstBucket, dstObject, etagkey) etag = string(b) + vId, _ := p.meta.RetrieveAttribute(dstBucket, dstObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + version = backend.GetStringPtr(string(vId)) } else { contentLength := fi.Size() - - etag, err = p.PutObject(ctx, + res, err := p.PutObject(ctx, &s3.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, @@ -2058,6 +2937,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. if err != nil { return nil, err } + etag = res.ETag + version = &res.VersionID } fi, err = os.Stat(dstObjdPath) @@ -2070,6 +2951,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), }, + VersionId: version, + CopySourceVersionId: &srcVersionId, }, nil } @@ -2157,6 +3040,12 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { }, nil } + // If the object is a delete marker, skip + isDel, _ := p.isObjDeleteMarker(bucket, path) + if isDel { + return s3response.Object{}, backend.ErrSkipObj + } + // file object, get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index a3f2f1a4..30864272 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -161,6 +161,48 @@ func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket stri return handleError(err) } +func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error { + _, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: status, + }, + }) + + return handleError(err) +} + +func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { + out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + + return out, handleError(err) +} + +func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + out, err := s.client.ListObjectVersions(ctx, input) + if err != nil { + return s3response.ListVersionsResult{}, handleError(err) + } + + return s3response.ListVersionsResult{ + CommonPrefixes: out.CommonPrefixes, + DeleteMarkers: out.DeleteMarkers, + Delimiter: out.Delimiter, + EncodingType: out.EncodingType, + IsTruncated: out.IsTruncated, + KeyMarker: out.KeyMarker, + MaxKeys: out.MaxKeys, + Name: out.Name, + NextKeyMarker: out.NextKeyMarker, + NextVersionIdMarker: out.NextVersionIdMarker, + Prefix: out.Prefix, + VersionIdMarker: input.VersionIdMarker, + Versions: out.Versions, + }, nil +} + func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { out, err := s.client.CreateMultipartUpload(ctx, input) if err != nil { @@ -304,17 +346,25 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn }, nil } -func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) { +func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) { // streaming backend is not seekable, // use unsigned payload for streaming ops output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions( v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware, )) if err != nil { - return "", handleError(err) + return s3response.PutObjectOutput{}, handleError(err) } - return *output.ETag, nil + var versionID string + if output.VersionId != nil { + versionID = *output.VersionId + } + + return s3response.PutObjectOutput{ + ETag: *output.ETag, + VersionID: versionID, + }, nil } func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { @@ -416,9 +466,9 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu }, nil } -func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { - _, err := s.client.DeleteObject(ctx, input) - return handleError(err) +func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := s.client.DeleteObject(ctx, input) + return res, handleError(err) } func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { diff --git a/backend/walk.go b/backend/walk.go index 047b5623..0cf36f65 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -246,3 +246,205 @@ func contains(a string, strs []string) bool { } return false } + +type WalkVersioningResults struct { + CommonPrefixes []types.CommonPrefix + ObjectVersions []types.ObjectVersion + DelMarkers []types.DeleteMarkerEntry + Truncated bool + NextMarker string + NextVersionIdMarker string +} + +type ObjVersionFuncResult struct { + ObjectVersions []types.ObjectVersion + DelMarkers []types.DeleteMarkerEntry + NextVersionIdMarker string + Truncated bool +} + +type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error) + +// WalkVersions walks the supplied fs.FS and returns results compatible with +// ListObjectVersions action response +func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) { + cpmap := make(map[string]struct{}) + var objects []types.ObjectVersion + var delMarkers []types.DeleteMarkerEntry + + var pastMarker bool + if keyMarker == "" { + pastMarker = true + } + var nextMarker string + var nextVersionIdMarker string + var truncated bool + + pastVersionIdMarker := versionIdMarker == "" + + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + // Ignore the root directory + if path == "." { + return nil + } + if contains(d.Name(), skipdirs) { + return fs.SkipDir + } + + if !pastMarker { + if path == keyMarker { + pastMarker = true + } + if path < keyMarker { + return nil + } + } + + if d.IsDir() { + // If prefix is defined and the directory does not match prefix, + // do not descend into the directory because nothing will + // match this prefix. Make sure to append the / at the end of + // directories since this is implied as a directory path name. + // If path is a prefix of prefix, then path could still be + // building to match. So only skip if path isn't a prefix of prefix + // and prefix isn't a prefix of path. + if prefix != "" && + !strings.HasPrefix(path+string(os.PathSeparator), prefix) && + !strings.HasPrefix(prefix, path+string(os.PathSeparator)) { + return fs.SkipDir + } + + res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d) + if err == ErrSkipObj { + return nil + } + if err != nil { + return fmt.Errorf("directory to object %q: %w", path, err) + } + objects = append(objects, res.ObjectVersions...) + delMarkers = append(delMarkers, res.DelMarkers...) + if res.Truncated { + truncated = true + nextMarker = path + nextVersionIdMarker = res.NextVersionIdMarker + return fs.SkipAll + } + + return nil + } + + // If object doesn't have prefix, don't include in results. + if prefix != "" && !strings.HasPrefix(path, prefix) { + return nil + } + + if delimiter == "" { + // If no delimiter specified, then all files with matching + // prefix are included in results + res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d) + if err == ErrSkipObj { + return nil + } + if err != nil { + return fmt.Errorf("file to object %q: %w", path, err) + } + objects = append(objects, res.ObjectVersions...) + delMarkers = append(delMarkers, res.DelMarkers...) + if res.Truncated { + truncated = true + nextMarker = path + nextVersionIdMarker = res.NextVersionIdMarker + return fs.SkipAll + } + + return nil + } + + // Since delimiter is specified, we only want results that + // do not contain the delimiter beyond the prefix. If the + // delimiter exists past the prefix, then the substring + // between the prefix and delimiter is part of common prefixes. + // + // For example: + // prefix = A/ + // delimiter = / + // and objects: + // A/file + // A/B/file + // B/C + // would return: + // objects: A/file + // common prefix: A/B/ + // + // Note: No objects are included past the common prefix since + // these are all rolled up into the common prefix. + // Note: The delimiter can be anything, so we have to operate on + // the full path without any assumptions on posix directory hierarchy + // here. Usually the delimiter will be "/", but thats not required. + suffix := strings.TrimPrefix(path, prefix) + before, _, found := strings.Cut(suffix, delimiter) + if !found { + res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d) + if err == ErrSkipObj { + return nil + } + if err != nil { + return fmt.Errorf("file to object %q: %w", path, err) + } + objects = append(objects, res.ObjectVersions...) + delMarkers = append(delMarkers, res.DelMarkers...) + + if res.Truncated { + truncated = true + nextMarker = path + nextVersionIdMarker = res.NextVersionIdMarker + return fs.SkipAll + } + return nil + } + + // Common prefixes are a set, so should not have duplicates. + // These are abstractly a "directory", so need to include the + // delimiter at the end. + cpmap[prefix+before+delimiter] = struct{}{} + if (len(objects) + len(cpmap)) == int(max) { + nextMarker = path + truncated = true + + return fs.SkipAll + } + + return nil + }) + if err != nil { + return WalkVersioningResults{}, err + } + + var commonPrefixStrings []string + for k := range cpmap { + commonPrefixStrings = append(commonPrefixStrings, k) + } + sort.Strings(commonPrefixStrings) + commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings)) + for _, cp := range commonPrefixStrings { + pfx := cp + commonPrefixes = append(commonPrefixes, types.CommonPrefix{ + Prefix: &pfx, + }) + } + + return WalkVersioningResults{ + CommonPrefixes: commonPrefixes, + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: truncated, + NextMarker: nextMarker, + NextVersionIdMarker: nextVersionIdMarker, + }, nil +} diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index b91fde59..3c8421de 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -25,6 +25,7 @@ import ( var ( chownuid, chowngid bool bucketlinks bool + versioningDir string ) func posixCommand() *cli.Command { @@ -61,6 +62,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, EnvVars: []string{"VGW_BUCKET_LINKS"}, Destination: &bucketlinks, }, + &cli.StringFlag{ + Name: "versioning-dir", + Usage: "the directory path to enable bucket versioning", + EnvVars: []string{"VGW_VERSIONING_DIR"}, + Destination: &versioningDir, + }, }, } } @@ -77,9 +84,10 @@ func runPosix(ctx *cli.Context) error { } be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{ - ChownUID: chownuid, - ChownGID: chowngid, - BucketLinks: bucketlinks, + ChownUID: chownuid, + ChownGID: chowngid, + BucketLinks: bucketlinks, + VersioningDir: versioningDir, }) if err != nil { return fmt.Errorf("init posix: %v", err) diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index b6c9ec24..b139f471 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -22,20 +22,21 @@ import ( ) var ( - awsID string - awsSecret string - endpoint string - prefix string - dstBucket string - partSize int64 - objSize int64 - concurrency int - files int - totalReqs int - upload bool - download bool - pathStyle bool - checksumDisable bool + awsID string + awsSecret string + endpoint string + prefix string + dstBucket string + partSize int64 + objSize int64 + concurrency int + files int + totalReqs int + upload bool + download bool + pathStyle bool + checksumDisable bool + versioningEnabled bool ) func testCommand() *cli.Command { @@ -87,6 +88,14 @@ func initTestCommands() []*cli.Command { Usage: "Tests the full flow of gateway.", Description: `Runs all the available tests to test the full flow of the gateway.`, Action: getAction(integration.TestFullFlow), + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "versioning-enabled", + Usage: "Test the bucket object versioning, if the versioning is enabled", + Destination: &versioningEnabled, + Aliases: []string{"vs"}, + }, + }, }, { Name: "posix", @@ -276,6 +285,9 @@ func getAction(tf testFunc) func(*cli.Context) error { if debug { opts = append(opts, integration.WithDebug()) } + if versioningEnabled { + opts = append(opts, integration.WithVersioningEnabled()) + } s := integration.NewS3Conf(opts...) tf(s) diff --git a/go.mod b/go.mod index bfe90d8d..0e471212 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/vault-client-go v0.4.3 github.com/nats-io/nats.go v1.37.0 + github.com/oklog/ulid/v2 v2.1.0 github.com/pkg/xattr v0.4.10 github.com/segmentio/kafka-go v0.4.47 github.com/smira/go-statsd v1.3.3 diff --git a/go.sum b/go.sum index 3b7a1b19..f77b3728 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,9 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= +github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= diff --git a/runtests.sh b/runtests.sh index abb63209..83d45116 100755 --- a/runtests.sh +++ b/runtests.sh @@ -5,9 +5,11 @@ rm -rf /tmp/gw mkdir /tmp/gw rm -rf /tmp/covdata mkdir /tmp/covdata +rm -rf /tmp/versioningdir +mkdir /tmp/versioningdir # run server in background -GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw & +GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw & GW_PID=$! # wait a second for server to start up @@ -21,7 +23,7 @@ fi # run tests # full flow tests -if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then +if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow -vs; then echo "full flow tests failed" kill $GW_PID exit 1 diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 0feaae5d..75c48049 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -53,7 +53,7 @@ var _ backend.Backend = &BackendMock{} // DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error { // panic("mock out the DeleteBucketTagging method") // }, -// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error { +// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { // panic("mock out the DeleteObject method") // }, // DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error { @@ -113,7 +113,7 @@ var _ backend.Backend = &BackendMock{} // ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { // panic("mock out the ListMultipartUploads method") // }, -// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { +// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { // panic("mock out the ListObjectVersions method") // }, // ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { @@ -137,10 +137,10 @@ var _ backend.Backend = &BackendMock{} // PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { // panic("mock out the PutBucketTagging method") // }, -// PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { +// PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { // panic("mock out the PutBucketVersioning method") // }, -// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) { +// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { // panic("mock out the PutObject method") // }, // PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error { @@ -214,7 +214,7 @@ type BackendMock struct { DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error // DeleteObjectFunc mocks the DeleteObject method. - DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error + DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) // DeleteObjectTaggingFunc mocks the DeleteObjectTagging method. DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error @@ -274,7 +274,7 @@ type BackendMock struct { ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) // ListObjectVersionsFunc mocks the ListObjectVersions method. - ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) + ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) // ListObjectsFunc mocks the ListObjects method. ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) @@ -298,10 +298,10 @@ type BackendMock struct { PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error // PutBucketVersioningFunc mocks the PutBucketVersioning method. - PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error + PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error // PutObjectFunc mocks the PutObject method. - PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) + PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) // PutObjectAclFunc mocks the PutObjectAcl method. PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error @@ -632,8 +632,10 @@ type BackendMock struct { PutBucketVersioning []struct { // ContextMoqParam is the contextMoqParam argument value. ContextMoqParam context.Context - // PutBucketVersioningInput is the putBucketVersioningInput argument value. - PutBucketVersioningInput *s3.PutBucketVersioningInput + // Bucket is the bucket argument value. + Bucket string + // Status is the status argument value. + Status types.BucketVersioningStatus } // PutObject holds details about calls to the PutObject method. PutObject []struct { @@ -1154,7 +1156,7 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct { } // DeleteObject calls DeleteObjectFunc. -func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error { +func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { if mock.DeleteObjectFunc == nil { panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called") } @@ -1898,7 +1900,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct { } // ListObjectVersions calls ListObjectVersionsFunc. -func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { +func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { if mock.ListObjectVersionsFunc == nil { panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called") } @@ -2202,21 +2204,23 @@ func (mock *BackendMock) PutBucketTaggingCalls() []struct { } // PutBucketVersioning calls PutBucketVersioningFunc. -func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { +func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { if mock.PutBucketVersioningFunc == nil { panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called") } callInfo := struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus }{ - ContextMoqParam: contextMoqParam, - PutBucketVersioningInput: putBucketVersioningInput, + ContextMoqParam: contextMoqParam, + Bucket: bucket, + Status: status, } mock.lockPutBucketVersioning.Lock() mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo) mock.lockPutBucketVersioning.Unlock() - return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput) + return mock.PutBucketVersioningFunc(contextMoqParam, bucket, status) } // PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning. @@ -2224,12 +2228,14 @@ func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, pu // // len(mockedBackend.PutBucketVersioningCalls()) func (mock *BackendMock) PutBucketVersioningCalls() []struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus } { var calls []struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus } mock.lockPutBucketVersioning.RLock() calls = mock.calls.PutBucketVersioning @@ -2238,7 +2244,7 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct { } // PutObject calls PutObjectFunc. -func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) { +func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { if mock.PutObjectFunc == nil { panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 93438dde..03ee2ccc 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -382,6 +382,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { }) } + action := auth.GetObjectAction + if versionId != "" { + action = auth.GetObjectVersionAction + } + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -390,7 +395,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Acc: acct, Bucket: bucket, Object: key, - Action: auth.GetObjectAction, + Action: action, }) if err != nil { return SendResponse(ctx, err, @@ -410,11 +415,23 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { VersionId: &versionId, }) if err != nil { + if res != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-delete-marker", + Value: "true", + }, + { + Key: "Last-Modified", + Value: res.LastModified.Format(timefmt), + }, + }) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, - Action: metrics.ActionGetObject, + Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }) } @@ -478,6 +495,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { utils.SetMetaHeaders(ctx, res.Metadata) // Set other response headers utils.SetResponseHeaders(ctx, hdrs) + // Set version id header + if getstring(res.VersionId) != "" { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }, + }) + } status := http.StatusOK if acceptRange != "" { @@ -981,8 +1007,8 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { objectOwnership := types.ObjectOwnership( ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), ) - mfa := ctx.Get("X-Amz-Mfa") - contentMD5 := ctx.Get("Content-MD5") + // mfa := ctx.Get("X-Amz-Mfa") + // contentMD5 := ctx.Get("Content-MD5") acct := ctx.Locals("account").(auth.Account) isRoot := ctx.Locals("isRoot").(bool) @@ -1136,13 +1162,21 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { }) } - err = c.be.PutBucketVersioning(ctx.Context(), - &s3.PutBucketVersioningInput{ - Bucket: &bucket, - MFA: &mfa, - VersioningConfiguration: &versioningConf, - ContentMD5: &contentMD5, - }) + if versioningConf.Status != types.BucketVersioningStatusEnabled && + versioningConf.Status != types.BucketVersioningStatusSuspended { + if c.debug { + log.Printf("invalid versioning configuration status: %v\n", versioningConf.Status) + } + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }) + } + + err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -1816,6 +1850,14 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ExpectedBucketOwner: &bucketOwner, CopySourceRange: ©SrcRange, }) + if err == nil && resp.CopySourceVersionId != "" { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-copy-source-version-id", + Value: resp.CopySourceVersionId, + }, + }) + } return SendXMLResponse(ctx, resp, err, &MetaOpts{ Logger: c.logger, @@ -2141,6 +2183,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { StorageClass: types.StorageClass(storageClass), }) if err == nil { + hdrs := []utils.CustomHeader{} + if getstring(res.VersionId) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }) + } + if getstring(res.CopySourceVersionId) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-copy-source-version-id", + Value: getstring(res.CopySourceVersionId), + }) + } + utils.SetResponseHeaders(ctx, hdrs) + return SendXMLResponse(ctx, res.CopyObjectResult, err, &MetaOpts{ Logger: c.logger, @@ -2232,7 +2289,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } ctx.Locals("logReqBody", false) - etag, err := c.be.PutObject(ctx.Context(), + res, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{ Bucket: &bucket, Key: &keyStart, @@ -2246,8 +2303,36 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ObjectLockMode: objLock.ObjectLockMode, ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, }) - ctx.Response().Header.Set("ETag", etag) - return SendResponse(ctx, err, + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + ContentLength: contentLength, + EvSender: c.evSender, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + ObjectSize: contentLength, + EventName: s3event.EventObjectCreatedPut, + }) + } + hdrs := []utils.CustomHeader{ + { + Key: "ETag", + Value: res.ETag, + }, + } + + if res.VersionID != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: res.VersionID, + }) + } + + utils.SetResponseHeaders(ctx, hdrs) + + return SendResponse(ctx, nil, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, @@ -2255,7 +2340,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { EvSender: c.evSender, Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, - ObjectETag: &etag, + ObjectETag: &res.ETag, ObjectSize: contentLength, EventName: s3event.EventObjectCreatedPut, }) @@ -2569,6 +2654,8 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { }) } + //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -2604,13 +2691,42 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { }) } - err = c.be.DeleteObject(ctx.Context(), + res, err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{ Bucket: &bucket, Key: &key, VersionId: &versionId, }) - return SendResponse(ctx, err, + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + EvSender: c.evSender, + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDelete, + Status: http.StatusNoContent, + }) + } + + hdrs := []utils.CustomHeader{} + if res.VersionId != nil && *res.VersionId != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: *res.VersionId, + }) + } + if res.DeleteMarker != nil && *res.DeleteMarker { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-delete-marker", + Value: "true", + }) + } + + utils.SetResponseHeaders(ctx, hdrs) + + return SendResponse(ctx, nil, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, @@ -2683,6 +2799,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) + versionId := ctx.Query("versionId") key := ctx.Params("key") keyEnd := ctx.Params("*1") if keyEnd != "" { @@ -2737,8 +2854,21 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &key, PartNumber: partNumber, + VersionId: &versionId, }) if err != nil { + if res != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-delete-marker", + Value: "true", + }, + { + Key: "Last-Modified", + Value: res.LastModified.Format(timefmt), + }, + }) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -2826,6 +2956,13 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: contentType, }) + if getstring(res.VersionId) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }) + } + utils.SetResponseHeaders(ctx, headers) return SendResponse(ctx, nil, @@ -3015,6 +3152,14 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }, }) if err == nil { + if getstring(res.VersionId) != "" { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }, + }) + } return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 0f1066b7..96ff2dd4 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -385,8 +385,8 @@ func TestS3ApiController_ListActions(t *testing.T) { GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { return &s3.GetBucketVersioningOutput{}, nil }, - ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { - return &s3.ListObjectVersionsOutput{}, nil + ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return s3response.ListVersionsResult{}, nil }, GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { return []byte{}, nil @@ -677,7 +677,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) { PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { return nil }, - PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { + PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { return nil }, PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error { @@ -968,8 +968,8 @@ func TestS3ApiController_PutActions(t *testing.T) { CopyObjectResult: &types.CopyObjectResult{}, }, nil }, - PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) { - return "ETag", nil + PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { + return s3response.PutObjectOutput{}, nil }, UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) { return "hello", nil @@ -1383,8 +1383,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) { GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error { - return nil + DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return &s3.DeleteObjectOutput{}, nil }, AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error { return nil @@ -1414,8 +1414,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) { GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error { - return s3err.GetAPIError(7) + DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) }, GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) diff --git a/s3err/s3err.go b/s3err/s3err.go index 2767d0ce..14fa70ae 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -132,6 +132,8 @@ const ( ErrMissingSecurityHeader ErrInvalidMetadataDirective ErrKeyTooLong + ErrInvalidVersionId + ErrNoSuchVersion // Non-AWS errors ErrExistingObjectIsDirectory @@ -517,8 +519,12 @@ var errorCodeResponse = map[ErrorCode]APIError{ HTTPStatusCode: http.StatusNotFound, }, ErrInvalidMetadataDirective: { + Code: "InvalidArgument", + Description: "Unknown metadata directive.", + }, + ErrInvalidVersionId: { Code: "InvalidArgument", - Description: "Unknown metadata directive.", + Description: "Invalid version id specified", HTTPStatusCode: http.StatusBadRequest, }, ErrKeyTooLong: { @@ -526,6 +532,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Your key is too long.", HTTPStatusCode: http.StatusBadRequest, }, + ErrNoSuchVersion: { + Code: "NoSuchVersion", + Description: "The specified version does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, // non aws errors ErrExistingObjectIsDirectory: { diff --git a/s3response/s3response.go b/s3response/s3response.go index ed54ad80..9d8b9b0f 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -23,6 +23,11 @@ import ( const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z" +type PutObjectOutput struct { + ETag string + VersionID string +} + // Part describes part metadata. type Part struct { PartNumber int @@ -302,9 +307,10 @@ type CanonicalUser struct { } type CopyObjectResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` - LastModified time.Time - ETag string + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` + LastModified time.Time + ETag string + CopySourceVersionId string `xml:"-"` } func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -360,3 +366,21 @@ type InitiateMultipartUploadResult struct { Key string UploadId string } + +type ListVersionsResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"` + CommonPrefixes []types.CommonPrefix + DeleteMarkers []types.DeleteMarkerEntry `xml:"DeleteMarker"` + Delimiter *string + EncodingType types.EncodingType + IsTruncated *bool + KeyMarker *string + MaxKeys *int32 + Name *string + NextKeyMarker *string + NextVersionIdMarker *string + Prefix *string + RequestCharged types.RequestCharged + VersionIdMarker *string + Versions []types.ObjectVersion `xml:"Version"` +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 12c7ef2d..c39b1914 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -469,6 +469,9 @@ func TestFullFlow(s *S3Conf) { TestGetObjectLegalHold(s) TestWORMProtection(s) TestAccessControl(s) + if s.versioningEnabled { + TestVersioning(s) + } } func TestPosix(s *S3Conf) { @@ -503,6 +506,45 @@ func TestAccessControl(s *S3Conf) { AccessControl_copy_object_with_starting_slash_for_user(s) } +func TestVersioning(s *S3Conf) { + // PutBucketVersioning action + PutBucketVersioning_non_existing_bucket(s) + PutBucketVersioning_invalid_status(s) + PutBucketVersioning_success(s) + // GetBucketVersioning action + GetBucketVersioning_non_existing_bucket(s) + GetBucketVersioning_success(s) + Versioning_PutObject_success(s) + // CopyObject action + Versioning_CopyObject_success(s) + Versioning_CopyObject_non_existing_version_id(s) + Versioning_CopyObject_from_an_object_version(s) + // HeadObject action + Versioning_HeadObject_invalid_versionId(s) + Versioning_HeadObject_success(s) + Versioning_HeadObject_delete_marker(s) + // GetObject action + Versioning_GetObject_invalid_versionId(s) + Versioning_GetObject_success(s) + Versioning_GetObject_delete_marker(s) + // DeleteObject(s) actions + Versioning_DeleteObject_delete_object_version(s) + Versioning_DeleteObject_delete_a_delete_marker(s) + Versioning_DeleteObjects_success(s) + Versioning_DeleteObjects_delete_deleteMarkers(s) + // ListObjectVersions + ListObjectVersions_non_existing_bucket(s) + ListObjectVersions_list_single_object_versions(s) + ListObjectVersions_list_multiple_object_versions(s) + ListObjectVersions_multiple_object_versions_truncated(s) + ListObjectVersions_with_delete_markers(s) + // Multipart upload + Versioning_Multipart_Upload_success(s) + Versioning_Multipart_Upload_overwrite_an_object(s) + Versioning_UploadPartCopy_non_existing_versionId(s) + Versioning_UploadPartCopy_from_an_object_version(s) +} + type IntTests map[string]func(s *S3Conf) error func GetIntTests() IntTests { @@ -812,5 +854,33 @@ func GetIntTests() IntTests { "AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl, "AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access, "AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user, + "PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket, + "PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status, + "PutBucketVersioning_success": PutBucketVersioning_success, + "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, + "GetBucketVersioning_success": GetBucketVersioning_success, + "Versioning_PutObject_success": Versioning_PutObject_success, + "Versioning_CopyObject_success": Versioning_CopyObject_success, + "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, + "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, + "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, + "Versioning_HeadObject_success": Versioning_HeadObject_success, + "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, + "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, + "Versioning_GetObject_success": Versioning_GetObject_success, + "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, + "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, + "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, + "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, + "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, + "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, + "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, + "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, + "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, + "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, + "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, + "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, } } diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index c547e8d2..49c362db 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -31,15 +31,16 @@ import ( ) type S3Conf struct { - awsID string - awsSecret string - awsRegion string - endpoint string - checksumDisable bool - pathStyle bool - PartSize int64 - Concurrency int - debug bool + awsID string + awsSecret string + awsRegion string + endpoint string + checksumDisable bool + pathStyle bool + PartSize int64 + Concurrency int + debug bool + versioningEnabled bool } func NewS3Conf(opts ...Option) *S3Conf { @@ -80,6 +81,9 @@ func WithConcurrency(c int) Option { func WithDebug() Option { return func(s *S3Conf) { s.debug = true } } +func WithVersioningEnabled() Option { + return func(s *S3Conf) { s.versioningEnabled = true } +} func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider { // TODO support token/IAM diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 016d6b37..08b07043 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -2985,7 +2985,7 @@ func HeadObject_non_existing_dir_object(s *S3Conf) error { "key2": "val2", } - _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, Metadata: meta, @@ -3018,7 +3018,7 @@ func HeadObject_with_contenttype(s *S3Conf) error { contentType := "text/plain" contentEncoding := "gzip" - _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, ContentType: &contentType, @@ -3075,7 +3075,7 @@ func HeadObject_success(s *S3Conf) error { } ctype := defaultContentType - _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, Metadata: meta, @@ -3234,7 +3234,7 @@ func GetObject_invalid_ranges(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(1234567), "my-obj" - _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -3291,7 +3291,7 @@ func GetObject_with_meta(s *S3Conf) error { "key2": "val2", } - _, _, err := putObjectWithData(0, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + _, err := putObjectWithData(0, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) if err != nil { return err } @@ -3320,7 +3320,7 @@ func GetObject_success(s *S3Conf) error { dataLength, obj := int64(1234567), "my-obj" ctype := defaultContentType - csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, ContentType: &ctype, @@ -3354,7 +3354,7 @@ func GetObject_success(s *S3Conf) error { } defer out.Body.Close() outCsum := sha256.Sum256(bdy) - if outCsum != csum { + if outCsum != r.csum { return fmt.Errorf("invalid object data") } return nil @@ -3368,7 +3368,7 @@ func GetObject_directory_success(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(0), "my-dir/" - _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -3405,7 +3405,7 @@ func GetObject_by_range_success(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(1234567), "my-obj" - _, data, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -3438,7 +3438,7 @@ func GetObject_by_range_success(s *S3Conf) error { } // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:201]) { + if !isEqual(b, r.data[100:201]) { return fmt.Errorf("data mismatch of range") } @@ -3462,7 +3462,7 @@ func GetObject_by_range_success(s *S3Conf) error { } // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:]) { + if !isEqual(b, r.data[100:]) { return fmt.Errorf("data mismatch of range") } return nil @@ -3473,7 +3473,7 @@ func GetObject_by_range_resp_status(s *S3Conf) error { testName := "GetObject_by_range_resp_status" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj, dLen := "my-obj", int64(4000) - _, _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -3521,7 +3521,7 @@ func GetObject_non_existing_dir_object(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(1234567), "my-obj" - _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -4308,9 +4308,11 @@ func DeleteObjects_success(s *S3Conf) error { } delObjects := []types.ObjectIdentifier{} + delResult := []types.DeletedObject{} for _, key := range objToDel { k := key delObjects = append(delObjects, types.ObjectIdentifier{Key: &k}) + delResult = append(delResult, types.DeletedObject{Key: &k}) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ @@ -4331,7 +4333,7 @@ func DeleteObjects_success(s *S3Conf) error { return fmt.Errorf("expected 2 errors, instead got %v", len(out.Errors)) } - if !compareDelObjects(objToDel, out.Deleted) { + if !compareDelObjects(delResult, out.Deleted) { return fmt.Errorf("unexpected deleted output") } @@ -4559,7 +4561,7 @@ func CopyObject_CopySource_starting_with_slash(s *S3Conf) error { return err } - csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -4588,7 +4590,7 @@ func CopyObject_CopySource_starting_with_slash(s *S3Conf) error { return err } if *out.ContentLength != dataLength { - return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, *out.ContentLength) } defer out.Body.Close() @@ -4598,7 +4600,7 @@ func CopyObject_CopySource_starting_with_slash(s *S3Conf) error { return err } outCsum := sha256.Sum256(bdy) - if outCsum != csum { + if outCsum != r.csum { return fmt.Errorf("invalid object data") } @@ -4620,7 +4622,7 @@ func CopyObject_non_existing_dir_object(s *S3Conf) error { return err } - _, _, err = putObjectWithData(dataLength, &s3.PutObjectInput{ + _, err = putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -4660,7 +4662,7 @@ func CopyObject_success(s *S3Conf) error { return err } - csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -4689,7 +4691,7 @@ func CopyObject_success(s *S3Conf) error { return err } if *out.ContentLength != dataLength { - return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, *out.ContentLength) } bdy, err := io.ReadAll(out.Body) @@ -4698,7 +4700,7 @@ func CopyObject_success(s *S3Conf) error { } defer out.Body.Close() outCsum := sha256.Sum256(bdy) - if outCsum != csum { + if outCsum != r.csum { return fmt.Errorf("invalid object data") } @@ -5724,7 +5726,7 @@ func UploadPartCopy_success(s *S3Conf) error { return err } objSize := 5 * 1024 * 1024 - _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ Bucket: &srcBucket, Key: &srcObj, }, s3client) @@ -5793,7 +5795,7 @@ func UploadPartCopy_by_range_invalid_range(s *S3Conf) error { return err } objSize := 5 * 1024 * 1024 - _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ Bucket: &srcBucket, Key: &srcObj, }, s3client) @@ -5839,7 +5841,7 @@ func UploadPartCopy_greater_range_than_obj_size(s *S3Conf) error { return err } srcObjSize := 5 * 1024 * 1024 - _, _, err = putObjectWithData(int64(srcObjSize), &s3.PutObjectInput{ + _, err = putObjectWithData(int64(srcObjSize), &s3.PutObjectInput{ Bucket: &srcBucket, Key: &srcObj, }, s3client) @@ -5885,7 +5887,7 @@ func UploadPartCopy_by_range_success(s *S3Conf) error { return err } objSize := 5 * 1024 * 1024 - _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ Bucket: &srcBucket, Key: &srcObj, }, s3client) @@ -10114,7 +10116,7 @@ func PutObject_overwrite_file_obj(s *S3Conf) error { func PutObject_dir_obj_with_data(s *S3Conf) error { testName := "PutObject_dir_obj_with_data" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - _, _, err := putObjectWithData(int64(20), &s3.PutObjectInput{ + _, err := putObjectWithData(int64(20), &s3.PutObjectInput{ Bucket: &bucket, Key: getPtr("obj/"), }, s3client) @@ -10155,6 +10157,26 @@ func PutObject_name_too_long(s *S3Conf) error { }) } +// Versioning tests +func PutBucketVersioning_non_existing_bucket(s *S3Conf) error { + testName := "PutBucketVersioning_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: getPtr(getBucketName()), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + func HeadObject_name_too_long(s *S3Conf) error { testName := "HeadObject_name_too_long" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -10167,6 +10189,291 @@ func HeadObject_name_too_long(s *S3Conf) error { if err := checkSdkApiErr(err, "BadRequest"); err != nil { return err } + + return nil + }) +} + +func PutBucketVersioning_invalid_status(s *S3Conf) error { + testName := "PutBucketVersioning_invalid_status" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatus("invalid_status"), + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil { + return err + } + + return nil + }) +} + +func PutBucketVersioning_success(s *S3Conf) error { + testName := "PutBucketVersioning_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + cancel() + if err != nil { + return err + } + + return nil + }) +} + +func GetBucketVersioning_non_existing_bucket(s *S3Conf) error { + testName := "GetBucketVersioning_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func GetBucketVersioning_success(s *S3Conf) error { + testName := "GetBucketVersioning_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if res.Status != types.BucketVersioningStatusEnabled { + return fmt.Errorf("expected bucket versioning status to be %v, instead got %v", types.BucketVersioningStatusEnabled, res.Status) + } + return nil + }, withVersioning()) +} + +func Versioning_PutObject_success(s *S3Conf) error { + testName := "Versioning_PutObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err != nil { + return err + } + + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected the versionId to be returned") + } + + return nil + }, withVersioning()) +} + +func Versioning_CopyObject_success(s *S3Conf) error { + testName := "Versioning_CopyObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dstObj := "dst-obj" + srcBucket, srcObj := getBucketName(), "src-obj" + + if err := setup(s, srcBucket); err != nil { + return err + } + + dstObjVersions, err := createObjVersions(s3client, bucket, dstObj, 1) + if err != nil { + return err + } + + srcObjLen := int64(2345) + _, err = putObjectWithData(srcObjLen, &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", srcBucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if err := teardown(s, srcBucket); err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId in the result") + } + + dstObjVersions[0].IsLatest = getBoolPtr(false) + versions := append([]types.ObjectVersion{ + { + ETag: out.CopyObjectResult.ETag, + IsLatest: getBoolPtr(true), + Key: &dstObj, + Size: &srcObjLen, + VersionId: out.VersionId, + }, + }, dstObjVersions...) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions) + } + + return nil + }, withVersioning()) +} + +func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { + testName := "Versioning_CopyObject_non_existing_version_id" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dstBucket, dstObj := getBucketName(), "my-obj" + srcObj := "my-obj" + + if err := setup(s, dstBucket); err != nil { + return err + } + + _, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=invalid_versionId", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { + testName := "Versioning_CopyObject_from_an_object_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcBucket, srcObj, dstObj := getBucketName(), "my-obj", "my-dst-obj" + if err := setup(s, srcBucket, withVersioning()); err != nil { + return err + } + + srcObjVersions, err := createObjVersions(s3client, srcBucket, srcObj, 1) + if err != nil { + return err + } + srcObjVersion := srcObjVersions[0] + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", srcBucket, srcObj, *srcObjVersion.VersionId)), + }) + cancel() + if err != nil { + return err + } + + if err := teardown(s, srcBucket); err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + if *out.CopySourceVersionId != *srcObjVersion.VersionId { + return fmt.Errorf("expected the SourceVersionId to be %v, instead got %v", *srcObjVersion.VersionId, *out.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if *res.ContentLength != *srcObjVersion.Size { + return fmt.Errorf("expected the copied object size to be %v, instead got %v", *srcObjVersion.Size, *res.ContentLength) + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the copied object versionId to be %v, instead got %v", *out.VersionId, *res.VersionId) + } + + return nil + }, withVersioning()) +} + +func Versioning_HeadObject_invalid_versionId(s *S3Conf) error { + testName := "Versioning_HeadObject_invalid_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("invalid_version_id"), + }) + cancel() + if err := checkSdkApiErr(err, "BadRequest"); err != nil { + return err + } return nil }) } @@ -10180,6 +10487,969 @@ func DeleteObject_name_too_long(s *S3Conf) error { Key: getPtr(genRandString(300)), }) cancel() - return err + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrKeyTooLong)); err != nil { + return err + } + return nil }) } + +func Versioning_HeadObject_success(s *S3Conf) error { + testName := "Versioning_HeadObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", *r.res.VersionId, *out.VersionId) + } + + return nil + }, withVersioning()) +} + +func Versioning_HeadObject_delete_marker(s *S3Conf) error { + testName := "Versioning_HeadObject_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkSdkApiErr(err, "MethodNotAllowed"); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func Versioning_GetObject_invalid_versionId(s *S3Conf) error { + testName := "Versioning_GetObject_invalid_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr("invalid_version_id"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidVersionId)); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func Versioning_GetObject_success(s *S3Conf) error { + testName := "Versioning_GetObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + r, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: r.res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if *out.ContentLength != dLen { + return fmt.Errorf("expected the object content-length to be %v, instead got %v", dLen, *out.ContentLength) + } + if *out.VersionId != *r.res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", *r.res.VersionId, *out.VersionId) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("incorrect output content") + } + + return nil + }, withVersioning()) +} + +func Versioning_GetObject_delete_marker(s *S3Conf) error { + testName := "Versioning_GetObject_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dLen := int64(2000) + obj := "my-obj" + _, err := putObjectWithData(dLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { + testName := "Versioning_DeleteObject_delete_object_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + oLen := int64(1000) + obj := "my-obj" + r, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + versionId := r.res.VersionId + if versionId == nil || *versionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + _, err = putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: versionId, + }) + cancel() + if err != nil { + return err + } + + if *out.VersionId != *versionId { + return fmt.Errorf("expected deleted object versionId to be %v, instead got %v", *versionId, *out.VersionId) + } + + return nil + }, withVersioning()) +} + +func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { + testName := "Versioning_DeleteObject_delete_a_delete_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + oLen := int64(1000) + obj := "my-obj" + _, err := putObjectWithData(oLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if out.VersionId == nil || *out.VersionId == "" { + return fmt.Errorf("expected non empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: out.VersionId, + }) + cancel() + if err != nil { + return err + } + + if res.DeleteMarker == nil || !*res.DeleteMarker { + return fmt.Errorf("expected the response DeleteMarker to be true") + } + if *res.VersionId != *out.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", *out.VersionId, *res.VersionId) + } + + return nil + }, withVersioning()) +} + +func Versioning_DeleteObjects_success(s *S3Conf) error { + testName := "Versioning_DeleteObjects_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2, obj3 := "foo", "bar", "baz" + + obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) + if err != nil { + return err + } + obj2Version, err := createObjVersions(s3client, bucket, obj2, 1) + if err != nil { + return err + } + obj3Version, err := createObjVersions(s3client, bucket, obj3, 1) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + { + Key: obj1Version[0].Key, + VersionId: obj1Version[0].VersionId, + }, + { + Key: obj2Version[0].Key, + }, + { + Key: obj3Version[0].Key, + }, + }, + }, + }) + cancel() + if err != nil { + return err + } + + delResult := []types.DeletedObject{ + { + Key: obj1Version[0].Key, + VersionId: obj1Version[0].VersionId, + }, + { + Key: obj2Version[0].Key, + }, + { + Key: obj3Version[0].Key, + }, + } + + if len(out.Errors) != 0 { + return fmt.Errorf("errors occurred during the deletion: %v", out.Errors) + } + if !compareDelObjects(delResult, out.Deleted) { + return fmt.Errorf("expected the deleted objects to be %v, instead got %v", delResult, out.Deleted) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + obj2Version[0].IsLatest = getBoolPtr(false) + obj3Version[0].IsLatest = getBoolPtr(false) + versions := append(obj2Version, obj3Version...) + + delMarkers := []types.DeleteMarkerEntry{ + { + IsLatest: getBoolPtr(true), + Key: out.Deleted[1].Key, + VersionId: out.Deleted[1].VersionId, + }, + { + IsLatest: getBoolPtr(true), + Key: out.Deleted[2].Key, + VersionId: out.Deleted[2].VersionId, + }, + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions) + } + if !compareDelMarkers(delMarkers, res.DeleteMarkers) { + return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", delMarkers, res.DeleteMarkers) + } + + return nil + }, withVersioning()) +} + +func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { + testName := "Versioning_DeleteObjects_delete_deleteMarkers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2 := "foo", "bar" + + obj1Version, err := createObjVersions(s3client, bucket, obj1, 1) + if err != nil { + return err + } + obj2Version, err := createObjVersions(s3client, bucket, obj2, 1) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + { + Key: obj1Version[0].Key, + }, + { + Key: obj2Version[0].Key, + }, + }, + }, + }) + cancel() + if err != nil { + return err + } + + delResult := []types.DeletedObject{ + { + Key: obj1Version[0].Key, + }, + { + Key: obj2Version[0].Key, + }, + } + + if len(out.Errors) != 0 { + return fmt.Errorf("errors occurred during the deletion: %v", out.Errors) + } + if !compareDelObjects(delResult, out.Deleted) { + return fmt.Errorf("expected the deleted objects to be %v, instead got %v", delResult, out.Deleted) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + { + Key: out.Deleted[0].Key, + VersionId: out.Deleted[0].VersionId, + }, + { + Key: out.Deleted[1].Key, + VersionId: out.Deleted[1].VersionId, + }, + }, + }, + }) + cancel() + if err != nil { + return err + } + if len(out.Errors) != 0 { + return fmt.Errorf("errors occurred during the deletion: %v", out.Errors) + } + + delResult = []types.DeletedObject{ + { + Key: out.Deleted[0].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[0].VersionId, + }, + { + Key: out.Deleted[1].Key, + DeleteMarker: getBoolPtr(true), + DeleteMarkerVersionId: out.Deleted[1].VersionId, + }, + } + + if !compareDelObjects(delResult, res.Deleted) { + return fmt.Errorf("expected the deleted objects to be %v, instead got %v", delResult, res.Deleted) + } + + return nil + }, withVersioning()) +} + +func ListObjectVersions_non_existing_bucket(s *S3Conf) error { + testName := "ListObjectVersions_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func ListObjectVersions_list_single_object_versions(s *S3Conf) error { + testName := "ListObjectVersions_list_single_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-obj" + versions, err := createObjVersions(s3client, bucket, object, 5) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(out.Versions, versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions) + } + + return nil + }, withVersioning()) +} + +func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { + testName := "ListObjectVersions_list_multiple_object_versions" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2, obj3 := "foo", "bar", "baz" + + obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) + if err != nil { + return err + } + obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + if err != nil { + return err + } + obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + if err != nil { + return err + } + + versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(out.Versions, versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions) + } + + return nil + }, withVersioning()) +} + +func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { + testName := "ListObjectVersions_multiple_object_versions_truncated" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2, obj3 := "foo", "bar", "baz" + + obj1Versions, err := createObjVersions(s3client, bucket, obj1, 4) + if err != nil { + return err + } + obj2Versions, err := createObjVersions(s3client, bucket, obj2, 3) + if err != nil { + return err + } + obj3Versions, err := createObjVersions(s3client, bucket, obj3, 5) + if err != nil { + return err + } + + versions := append(append(obj2Versions, obj3Versions...), obj1Versions...) + maxKeys := int32(5) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return err + } + + if *out.Name != bucket { + return fmt.Errorf("expected the bucket name to be %v, instead got %v", bucket, *out.Name) + } + if out.IsTruncated == nil || !*out.IsTruncated { + return fmt.Errorf("expected the output to be truncated") + } + if out.MaxKeys == nil || *out.MaxKeys != maxKeys { + return fmt.Errorf("expected the max-keys to be %v, instead got %v", maxKeys, *out.MaxKeys) + } + if *out.NextKeyMarker != *versions[maxKeys-1].Key { + return fmt.Errorf("expected the NextKeyMarker to be %v, instead got %v", *versions[maxKeys].Key, *out.NextKeyMarker) + } + if *out.NextVersionIdMarker != *versions[maxKeys-1].VersionId { + return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.NextVersionIdMarker) + } + + if !compareVersions(out.Versions, versions[:maxKeys]) { + return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[:maxKeys], out.Versions) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + KeyMarker: out.NextKeyMarker, + VersionIdMarker: out.NextVersionIdMarker, + }) + cancel() + if err != nil { + return err + } + + if *out.Name != bucket { + return fmt.Errorf("expected the bucket name to be %v, instead got %v", bucket, *out.Name) + } + if out.IsTruncated != nil && *out.IsTruncated { + return fmt.Errorf("expected the output not to be truncated") + } + if *out.KeyMarker != *versions[maxKeys-1].Key { + return fmt.Errorf("expected the KeyMarker to be %v, instead got %v", *versions[maxKeys].Key, *out.KeyMarker) + } + if *out.VersionIdMarker != *versions[maxKeys-1].VersionId { + return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.VersionIdMarker) + } + + if !compareVersions(out.Versions, versions[maxKeys:]) { + return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[maxKeys:], out.Versions) + } + + return nil + }, withVersioning()) +} + +func ListObjectVersions_with_delete_markers(s *S3Conf) error { + testName := "ListObjectVersions_with_delete_markers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + versions, err := createObjVersions(s3client, bucket, obj, 1) + if err != nil { + return err + } + + versions[0].IsLatest = getBoolPtr(false) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + delMarkers := []types.DeleteMarkerEntry{} + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + Key: &obj, + VersionId: out.VersionId, + IsLatest: getBoolPtr(true), + }) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(res.Versions, versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions) + } + if !compareDelMarkers(res.DeleteMarkers, delMarkers) { + return fmt.Errorf("expected the resulting delete markers to be %v, instead got %v", delMarkers, res.DeleteMarkers) + } + + return nil + }, withVersioning()) +} + +func Versioning_Multipart_Upload_success(s *S3Conf) error { + testName := "Versioning_Multipart_Upload_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + objSize := 5 * 1024 * 1024 + parts, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + if *res.Key != obj { + return fmt.Errorf("expected object key to be %v, instead got %v", obj, *res.Key) + } + if *res.Bucket != bucket { + return fmt.Errorf("expected the bucket name to be %v, instead got %v", bucket, *res.Bucket) + } + if res.ETag == nil || *res.ETag == "" { + return fmt.Errorf("expected non-empty ETag") + } + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected non-empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: res.VersionId, + }) + cancel() + if err != nil { + return err + } + + if *resp.ETag != *res.ETag { + return fmt.Errorf("expected the uploaded object etag to be %v, instead got %v", *res.ETag, *resp.ETag) + } + if *resp.ContentLength != int64(objSize) { + return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", objSize, resp.ContentLength) + } + if *resp.VersionId != *res.VersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", *res.VersionId, *resp.VersionId) + } + + return nil + }, withVersioning()) +} + +func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error { + testName := "Versioning_Multipart_Upload_overwrite_an_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + objVersions, err := createObjVersions(s3client, bucket, obj, 2) + if err != nil { + return err + } + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + objSize := 5 * 1024 * 1024 + parts, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + if *res.Key != obj { + return fmt.Errorf("expected object key to be %v, instead got %v", obj, *res.Key) + } + if *res.Bucket != bucket { + return fmt.Errorf("expected the bucket name to be %v, instead got %v", bucket, *res.Bucket) + } + if res.ETag == nil || *res.ETag == "" { + return fmt.Errorf("expected non-empty ETag") + } + if res.VersionId == nil || *res.VersionId == "" { + return fmt.Errorf("expected non-empty versionId") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + size := int64(objSize) + + objVersions[0].IsLatest = getBoolPtr(false) + versions := append([]types.ObjectVersion{ + { + Key: &obj, + VersionId: res.VersionId, + ETag: res.ETag, + IsLatest: getBoolPtr(true), + Size: &size, + }, + }, objVersions...) + + if !compareVersions(resp.Versions, versions) { + return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, resp.Versions) + } + + return nil + }, withVersioning()) +} + +func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error { + testName := "Versioning_UploadPartCopy_non_existing_versionId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dstBucket, dstObj, srcObj := getBucketName(), "dst-obj", "src-obj" + + lgth := int64(100) + _, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + if err := setup(s, dstBucket); err != nil { + return err + } + + mp, err := createMp(s3client, dstBucket, dstObj) + if err != nil { + return err + } + + pNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + Key: &dstObj, + UploadId: mp.UploadId, + PartNumber: &pNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=invalid_versionId", bucket, srcObj)), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)); err != nil { + return err + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil + }, withVersioning()) +} + +func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { + testName := "Versioning_UploadPartCopy_from_an_object_version" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj, dstBucket, obj := "my-obj", getBucketName(), "dst-obj" + err := setup(s, dstBucket) + if err != nil { + return err + } + + srcObjVersions, err := createObjVersions(s3client, bucket, srcObj, 1) + if err != nil { + return err + } + srcObjVersion := srcObjVersions[0] + + out, err := createMp(s3client, dstBucket, obj) + if err != nil { + return err + } + + partNumber := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &dstBucket, + CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, *srcObjVersion.VersionId)), + UploadId: out.UploadId, + Key: &obj, + PartNumber: &partNumber, + }) + cancel() + if err != nil { + return err + } + + if *copyOut.CopySourceVersionId != *srcObjVersion.VersionId { + return fmt.Errorf("expected the copy-source-version-id to be %v, instead got %v", *srcObjVersion.VersionId, *copyOut.CopySourceVersionId) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &dstBucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) + } + if *res.Parts[0].PartNumber != partNumber { + return fmt.Errorf("expected part-number to be %v, instead got %v", partNumber, res.Parts[0].PartNumber) + } + if *res.Parts[0].Size != *srcObjVersion.Size { + return fmt.Errorf("expected part size to be %v, instead got %v", *srcObjVersion.Size, res.Parts[0].Size) + } + if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { + return fmt.Errorf("expected part etag to be %v, instead got %v", *copyOut.CopyPartResult.ETag, *res.Parts[0].ETag) + } + + if err := teardown(s, dstBucket); err != nil { + return err + } + + return nil + }, withVersioning()) +} diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 2f6c01b1..8f10caec 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "math/big" rnd "math/rand" "net/http" "net/url" @@ -70,7 +71,25 @@ func setup(s *S3Conf, bucket string, opts ...setupOpt) error { ObjectOwnership: cfg.Ownership, }) cancel() - return err + if err != nil { + return err + } + + if cfg.VersioningEnabled { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + cancel() + if err != nil { + return err + } + } + + return nil } func teardown(s *S3Conf, bucket string) error { @@ -90,24 +109,31 @@ func teardown(s *S3Conf, bucket string) error { return nil } - in := &s3.ListObjectsV2Input{Bucket: &bucket} + in := &s3.ListObjectVersionsInput{Bucket: &bucket} for { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, in) + out, err := s3client.ListObjectVersions(ctx, in) cancel() if err != nil { return fmt.Errorf("failed to list objects: %w", err) } - for _, item := range out.Contents { - err = deleteObject(&bucket, item.Key, nil) + for _, item := range out.Versions { + err = deleteObject(&bucket, item.Key, item.VersionId) + if err != nil { + return err + } + } + for _, item := range out.DeleteMarkers { + err = deleteObject(&bucket, item.Key, item.VersionId) if err != nil { return err } } if out.IsTruncated != nil && *out.IsTruncated { - in.ContinuationToken = out.ContinuationToken + in.KeyMarker = out.KeyMarker + in.VersionIdMarker = out.NextVersionIdMarker } else { break } @@ -122,8 +148,9 @@ func teardown(s *S3Conf, bucket string) error { } type setupCfg struct { - LockEnabled bool - Ownership types.ObjectOwnership + LockEnabled bool + VersioningEnabled bool + Ownership types.ObjectOwnership } type setupOpt func(*setupCfg) @@ -134,6 +161,9 @@ func withLock() setupOpt { func withOwnership(o types.ObjectOwnership) setupOpt { return func(s *setupCfg) { s.Ownership = o } } +func withVersioning() setupOpt { + return func(s *setupCfg) { s.VersioningEnabled = true } +} func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error { runF(testName) @@ -383,18 +413,31 @@ func contains(s []string, e string) bool { return false } -func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) { - data = make([]byte, lgth) +type putObjectOutput struct { + csum [32]byte + data []byte + res *s3.PutObjectOutput +} + +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) { + data := make([]byte, lgth) rand.Read(data) - csum = sha256.Sum256(data) + csum := sha256.Sum256(data) r := bytes.NewReader(data) input.Body = r ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = client.PutObject(ctx, input) + res, err := client.PutObject(ctx, input) cancel() + if err != nil { + return nil, err + } - return + return &putObjectOutput{ + csum: csum, + data: data, + res: res, + }, nil } func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) { @@ -592,21 +635,40 @@ func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool { return true } -func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool { +func compareDelObjects(list1, list2 []types.DeletedObject) bool { if len(list1) != len(list2) { return false } - elementMap := make(map[string]bool) - - for _, elem := range list1 { - elementMap[elem] = true - } - - for _, elem := range list2 { - if _, found := elementMap[*elem.Key]; !found { + for i, obj := range list1 { + if *obj.Key != *list2[i].Key { return false } + + if obj.VersionId != nil { + if list2[i].VersionId == nil { + return false + } + if *obj.VersionId != *list2[i].VersionId { + return false + } + } + if obj.DeleteMarkerVersionId != nil { + if list2[i].DeleteMarkerVersionId == nil { + return false + } + if *obj.DeleteMarkerVersionId != *list2[i].DeleteMarkerVersionId { + return false + } + } + if obj.DeleteMarker != nil { + if list2[i].DeleteMarker == nil { + return false + } + if *obj.DeleteMarker != *list2[i].DeleteMarker { + return false + } + } } return true @@ -860,3 +922,124 @@ func pfxStrings(pfxs []types.CommonPrefix) []string { } return pfxStrs } + +func createObjVersions(client *s3.Client, bucket, object string, count int) ([]types.ObjectVersion, error) { + versions := []types.ObjectVersion{} + for i := 0; i < count; i++ { + rNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) + dataLength := rNumber.Int64() + if err != nil { + return nil, err + } + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &object, + }, client) + if err != nil { + return nil, err + } + + isLatest := i == count-1 + + versions = append(versions, types.ObjectVersion{ + ETag: r.res.ETag, + IsLatest: &isLatest, + Key: &object, + Size: &dataLength, + VersionId: r.res.VersionId, + }) + } + + versions = reverseSlice(versions) + + return versions, nil +} + +// ReverseSlice reverses a slice of any type +func reverseSlice[T any](s []T) []T { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } + return s +} + +func compareVersions(v1, v2 []types.ObjectVersion) bool { + if len(v1) != len(v2) { + return false + } + + for i, version := range v1 { + if version.Key == nil || v2[i].Key == nil { + return false + } + if *version.Key != *v2[i].Key { + return false + } + + if version.VersionId == nil || v2[i].VersionId == nil { + return false + } + if *version.VersionId != *v2[i].VersionId { + return false + } + + if version.IsLatest == nil || v2[i].IsLatest == nil { + return false + } + if *version.IsLatest != *v2[i].IsLatest { + return false + } + + if version.Size == nil || v2[i].Size == nil { + return false + } + if *version.Size != *v2[i].Size { + return false + } + + if version.ETag == nil || v2[i].ETag == nil { + return false + } + if *version.ETag != *v2[i].ETag { + return false + } + } + + return true +} + +func compareDelMarkers(d1, d2 []types.DeleteMarkerEntry) bool { + if len(d1) != len(d2) { + return false + } + + for i, dEntry := range d1 { + if dEntry.Key == nil || d2[i].Key == nil { + return false + } + if *dEntry.Key != *d2[i].Key { + return false + } + + if dEntry.IsLatest == nil || d2[i].IsLatest == nil { + return false + } + if *dEntry.IsLatest != *d2[i].IsLatest { + return false + } + + if dEntry.VersionId == nil || d2[i].VersionId == nil { + return false + } + if *dEntry.VersionId != *d2[i].VersionId { + return false + } + } + + return true +} + +func getBoolPtr(b bool) *bool { + return &b +}