diff --git a/auth/bucket_policy_actions.go b/auth/bucket_policy_actions.go index 22510dc..7e468fd 100644 --- a/auth/bucket_policy_actions.go +++ b/auth/bucket_policy_actions.go @@ -36,6 +36,7 @@ const ( ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads" PutObjectAction Action = "s3:PutObject" GetObjectAction Action = "s3:GetObject" + GetObjectVersionAction Action = "s3:GetObjectVersion" DeleteObjectAction Action = "s3:DeleteObject" GetObjectAclAction Action = "s3:GetObjectAcl" GetObjectAttributesAction Action = "s3:GetObjectAttributes" @@ -75,6 +76,7 @@ var supportedActionList = map[Action]struct{}{ ListBucketMultipartUploadsAction: {}, PutObjectAction: {}, GetObjectAction: {}, + GetObjectVersionAction: {}, DeleteObjectAction: {}, GetObjectAclAction: {}, GetObjectAttributesAction: {}, @@ -103,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{ ListMultipartUploadPartsAction: {}, PutObjectAction: {}, GetObjectAction: {}, + GetObjectVersionAction: {}, DeleteObjectAction: {}, GetObjectAclAction: {}, GetObjectAttributesAction: {}, diff --git a/backend/azure/azure.go b/backend/azure/azure.go index c0c13cd..0055195 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -284,10 +284,10 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership)) } -func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { +func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { tags, err := parseTags(po.Tagging) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } opts := &blockblob.UploadStreamOptions{ @@ -312,14 +312,14 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts) if err != nil { - return "", azureErrToS3Err(err) + return s3response.PutObjectOutput{}, azureErrToS3Err(err) } // Set object legal hold if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } @@ -331,15 +331,17 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, } retParsed, err := json.Marshal(retention) if err != nil { - return "", fmt.Errorf("parse object lock retention: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) } err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } - return string(*uploadResp.ETag), nil + return s3response.PutObjectOutput{ + ETag: string(*uploadResp.ETag), + }, nil } func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error { @@ -675,22 +677,22 @@ Pager: }, nil } -func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { +func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) if err != nil { azerr, ok := err.(*azcore.ResponseError) if ok && azerr.StatusCode == 404 { // if the object does not exist, S3 returns success - return nil + return &s3.DeleteObjectOutput{}, nil } } - return azureErrToS3Err(err) + return &s3.DeleteObjectOutput{}, azureErrToS3Err(err) } func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { delResult, errs := []types.DeletedObject{}, []types.Error{} for _, obj := range input.Delete.Objects { - err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ + _, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: input.Bucket, Key: obj.Key, }) diff --git a/backend/backend.go b/backend/backend.go index 3976be2..0e52dfb 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -38,7 +38,7 @@ type Backend interface { CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error PutBucketAcl(_ context.Context, bucket string, data []byte) error DeleteBucket(context.Context, *s3.DeleteBucketInput) error - PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error + PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) @@ -57,7 +57,7 @@ type Backend interface { UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) // standard object operations - PutObject(context.Context, *s3.PutObjectInput) (string, error) + PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) @@ -65,10 +65,10 @@ type Backend interface { CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) - DeleteObject(context.Context, *s3.DeleteObjectInput) error + DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error - ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) + ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) // special case object operations RestoreObject(context.Context, *s3.RestoreObjectInput) error @@ -126,7 +126,7 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data [] func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error { +func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { @@ -173,8 +173,8 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) { - return "", s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -197,8 +197,8 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3 func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error { - return s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -225,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel } } -func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index ad56b36..a331a3f 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -35,6 +35,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" + "github.com/oklog/ulid/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/backend/meta" @@ -64,6 +65,9 @@ type Posix struct { // bucketlinks is a flag to enable symlinks to directories at the top // level gateway directory to be treated as buckets the same as directories bucketlinks bool + + // bucket versioning directory path + versioningDir string } var _ backend.Backend = &Posix{} @@ -84,15 +88,19 @@ const ( bucketLockKey = "bucket-lock" objectRetentionKey = "object-retention" objectLegalHoldKey = "object-legal-hold" + versioningKey = "versioning" + deleteMarkerKey = "delete-marker" + versionIdKey = "version-id" doFalloc = true skipFalloc = false ) type PosixOpts struct { - ChownUID bool - ChownGID bool - BucketLinks bool + ChownUID bool + ChownGID bool + BucketLinks bool + VersioningDir string } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { @@ -106,15 +114,57 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro return nil, fmt.Errorf("open %v: %w", rootdir, err) } + var verioningdirAbs string + + // Ensure the versioning directory isn't within the root directory + if opts.VersioningDir != "" { + rootdirAbs, err := filepath.Abs(rootdir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err) + } + + verioningdirAbs, err = filepath.Abs(opts.VersioningDir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", opts.VersioningDir, err) + } + + // Ensure the paths end with a separator + if !strings.HasSuffix(rootdirAbs, string(filepath.Separator)) { + rootdirAbs += string(filepath.Separator) + } + + if !strings.HasSuffix(verioningdirAbs, string(filepath.Separator)) { + verioningdirAbs += string(filepath.Separator) + } + + // Ensure the posix root directory doesn't contain the versioning directory + if strings.HasPrefix(verioningdirAbs, rootdirAbs) { + return nil, fmt.Errorf("the root directory %v contains the versioning directory %v", rootdir, opts.VersioningDir) + } + + vDir, err := os.Stat(verioningdirAbs) + if err != nil { + return nil, fmt.Errorf("stat versioning dir: %w", err) + } + + // Check the versioning path to be a directory + if !vDir.IsDir() { + return nil, fmt.Errorf("versioning path should be a directory") + } + + fmt.Printf("bucket versioning enabled with directory: %v\n", verioningdirAbs) + } + return &Posix{ - meta: meta, - rootfd: f, - rootdir: rootdir, - euid: os.Geteuid(), - egid: os.Getegid(), - chownuid: opts.ChownUID, - chowngid: opts.ChownGID, - bucketlinks: opts.BucketLinks, + meta: meta, + rootfd: f, + rootdir: rootdir, + euid: os.Geteuid(), + egid: os.Getegid(), + chownuid: opts.ChownUID, + chowngid: opts.ChownGID, + bucketlinks: opts.BucketLinks, + versioningDir: verioningdirAbs, }, nil } @@ -126,6 +176,11 @@ func (p *Posix) String() string { return "Posix Gateway" } +// returns the versioning state +func (p *Posix) versioningEnabled() bool { + return p.versioningDir != "" +} + func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) { entries, err := os.ReadDir(".") if err != nil { @@ -380,6 +435,385 @@ func (p *Posix) DeleteBucketOwnershipControls(_ context.Context, bucket string) return nil } +func (p *Posix) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil + } + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return fmt.Errorf("stat bucket: %w", err) + } + + // Store 1 bit for bucket versioning state + var versioning []byte + switch status { + case types.BucketVersioningStatusEnabled: + // '1' maps to 'Enabled' + versioning = []byte{1} + case types.BucketVersioningStatusSuspended: + // '0' maps to 'Suspended' + versioning = []byte{0} + } + + if err := p.meta.StoreAttribute(bucket, "", versioningKey, versioning); err != nil { + return fmt.Errorf("set versioning: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { + if !p.versioningEnabled() { + // AWS returns empty response, if versioning is not set + //TODO: Maybe we need to return our custom error here? + return &s3.GetBucketVersioningOutput{}, nil + } + + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + vData, err := p.meta.RetrieveAttribute(bucket, "", versioningKey) + if errors.Is(err, meta.ErrNoSuchKey) { + return &s3.GetBucketVersioningOutput{}, nil + } + + switch vData[0] { + case 1: + return &s3.GetBucketVersioningOutput{ + Status: types.BucketVersioningStatusEnabled, + }, nil + case 0: + return &s3.GetBucketVersioningOutput{ + Status: types.BucketVersioningStatusSuspended, + }, nil + } + + return &s3.GetBucketVersioningOutput{}, nil +} + +// Generates the object version path in the versioning directory +func (p *Posix) genObjVersionPath(bucket, key string) string { + return filepath.Join(p.versioningDir, bucket, genObjVersionKey(key)) +} + +// Generates the versioning path for the given object key +func genObjVersionKey(key string) string { + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) + + return filepath.Join(sum[:2], sum[2:4], sum[4:6], sum) +} + +// Creates a new copy(version) of an object in the versioning directory +func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account) (versionPath string, err error) { + sf, err := os.Open(filepath.Join(bucket, key)) + if err != nil { + return "", err + } + defer sf.Close() + + var versionId string + data, err := p.meta.RetrieveAttribute(bucket, key, versionIdKey) + if err != nil { + versionId = string(data) + } else { + versionId = ulid.Make().String() + } + + attrs, err := p.meta.ListAttributes(bucket, key) + if err != nil { + return versionPath, fmt.Errorf("load object attributes: %w", err) + } + + versionBucketPath := filepath.Join(p.versioningDir, bucket) + versioningKey := filepath.Join(genObjVersionKey(key), versionId) + versionTmpPath := filepath.Join(versionBucketPath, metaTmpDir) + f, err := p.openTmpFile(versionTmpPath, versionBucketPath, versioningKey, size, acc, doFalloc) + if err != nil { + return versionPath, err + } + defer f.cleanup() + + _, err = io.Copy(f.File(), sf) + if err != nil { + return versionPath, err + } + + versionPath = filepath.Join(versionBucketPath, versioningKey) + + err = os.MkdirAll(filepath.Join(versionBucketPath, genObjVersionKey(key)), defaultDirPerm) + if err != nil { + return versionPath, err + } + + if err := f.link(); err != nil { + return versionPath, err + } + + // Copy the object attributes(metadata) + for _, attr := range attrs { + data, err := p.meta.RetrieveAttribute(bucket, key, attr) + if err != nil { + return versionPath, fmt.Errorf("list %v attribute: %w", attr, err) + } + + if err := p.meta.StoreAttribute(versionPath, "", attr, data); err != nil { + return versionPath, fmt.Errorf("store %v attribute: %w", attr, err) + } + } + + return versionPath, nil +} + +func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + if !p.versioningEnabled() { + return s3response.ListVersionsResult{}, nil + } + bucket := *input.Bucket + var prefix, delim, keyMarker, versionIdMarker string + var max int + + if input.Prefix != nil { + prefix = *input.Prefix + } + if input.Delimiter != nil { + delim = *input.Delimiter + } + if input.KeyMarker != nil { + keyMarker = *input.KeyMarker + } + if input.VersionIdMarker != nil { + versionIdMarker = *input.VersionIdMarker + } + if input.MaxKeys != nil { + max = int(*input.MaxKeys) + } + + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return s3response.ListVersionsResult{}, fmt.Errorf("stat bucket: %w", err) + } + + fileSystem := os.DirFS(bucket) + results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max, + p.fileToObjVersions(bucket), []string{metaTmpDir}) + if err != nil { + return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err) + } + + return s3response.ListVersionsResult{ + CommonPrefixes: results.CommonPrefixes, + DeleteMarkers: results.DelMarkers, + Delimiter: &delim, + IsTruncated: &results.Truncated, + KeyMarker: &keyMarker, + MaxKeys: input.MaxKeys, + Name: input.Bucket, + NextKeyMarker: &results.NextMarker, + NextVersionIdMarker: &results.NextVersionIdMarker, + Prefix: &prefix, + VersionIdMarker: &versionIdMarker, + Versions: results.ObjectVersions, + }, nil +} + +func getBoolPtr(b bool) *bool { + return &b +} + +// Check if the given object is a delete marker +func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { + _, err := p.meta.RetrieveAttribute(bucket, object, deleteMarkerKey) + if errors.Is(err, fs.ErrNotExist) { + return false, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get object delete-marker: %w", err) + } + + return true, nil +} + +// Converts the file to object version. Finds all the object versions, +// delete markers from the versioning directory and returns +func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { + return func(path, versionIdMarker string, availableObjCount int, d fs.DirEntry) (*backend.ObjVersionFuncResult, error) { + var objects []types.ObjectVersion + var delMarkers []types.DeleteMarkerEntry + // if the number of available objects is 0, return truncated response + if availableObjCount <= 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + }, nil + } + if d.IsDir() { + //TODO: directory objects can't have versions, but they are listed in object versions result? + return nil, backend.ErrSkipObj + } + + // file object, get object info and fill out object data + etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + // If the object doesn't have versionId, it's 'null' + versionId := "null" + versionIdBytes, err := p.meta.RetrieveAttribute(bucket, path, versionIdKey) + if err == nil { + versionId = string(versionIdBytes) + } + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + size := fi.Size() + + isDel, err := p.isObjDeleteMarker(bucket, path) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + IsLatest: getBoolPtr(true), + VersionId: &versionId, + LastModified: backend.GetTimePtr(fi.ModTime()), + Key: &path, + }) + } else { + objects = append(objects, types.ObjectVersion{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(true), + }) + } + + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + + // List all the versions of the object in the versioning directory + versionPath := p.genObjVersionPath(bucket, path) + dirEnts, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(dirEnts) == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + + for i := len(dirEnts) - 1; i >= 0; i-- { + dEntry := dirEnts[i] + + f, err := dEntry.Info() + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + versionId := f.Name() + size := f.Size() + + etagBytes, err := p.meta.RetrieveAttribute(versionPath, versionId, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + isDel, err := p.isObjDeleteMarker(versionPath, versionId) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, types.DeleteMarkerEntry{ + VersionId: &versionId, + LastModified: backend.GetTimePtr(fi.ModTime()), + Key: &path, + }) + } else { + objects = append(objects, types.ObjectVersion{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(f.ModTime()), + Size: &size, + VersionId: &versionId, + }) + } + + // if the available object count reaches to 0, return truncated response with nextVersionIdMarker + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + } + + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } +} + func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if mpu.Bucket == nil { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) @@ -1353,27 +1787,27 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) }, nil } -func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { +func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if po.Bucket == nil { - return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if po.Key == nil { - return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } tagsStr := getString(po.Tagging) tags := make(map[string]string) _, err := os.Stat(*po.Bucket) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return "", fmt.Errorf("stat bucket: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("stat bucket: %w", err) } if tagsStr != "" { @@ -1381,10 +1815,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e for _, prt := range tagParts { p := strings.Split(prt, "=") if len(p) != 2 { - return "", s3err.GetAPIError(s3err.ErrInvalidTag) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidTag) } if len(p[0]) > 128 || len(p[1]) > 256 { - return "", s3err.GetAPIError(s3err.ErrInvalidTag) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidTag) } tags[p[0]] = p[1] } @@ -1404,22 +1838,22 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e // posix directories can't contain data, send error // if reuests has a data payload associated with a // directory object - return "", s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) } err = backend.MkdirAll(name, uid, gid, doChown) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", err + return s3response.PutObjectOutput{}, err } for k, v := range po.Metadata { err := p.meta.StoreAttribute(*po.Bucket, *po.Key, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) if err != nil { - return "", fmt.Errorf("set user attr %q: %w", k, err) + return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err) } } @@ -1427,31 +1861,42 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(emptyMD5)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } - return emptyMD5, nil + // for directory object no version is created + return s3response.PutObjectOutput{ + ETag: emptyMD5, + }, nil } // object is file d, err := os.Stat(name) if err == nil && d.IsDir() { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // if the versioninng is enabled first create the file object version + if p.versioningEnabled() && err == nil { + _, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err) + } } if errors.Is(err, syscall.ENAMETOOLONG) { - return "", s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil && !errors.Is(err, fs.ErrNotExist) { - return "", fmt.Errorf("stat object: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("stat object: %w", err) } f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), *po.Bucket, *po.Key, contentLength, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("open temp file: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() @@ -1460,28 +1905,28 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e _, err = io.Copy(f, rdr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("write object data: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("write object data: %w", err) } dir := filepath.Dir(name) if dir != "" { err = backend.MkdirAll(dir, uid, gid, doChown) if err != nil { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } } err = f.link() if err != nil { - return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } for k, v := range po.Metadata { err := p.meta.StoreAttribute(*po.Bucket, *po.Key, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) if err != nil { - return "", fmt.Errorf("set user attr %q: %w", k, err) + return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err) } } @@ -1489,15 +1934,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e if tagsStr != "" { err := p.PutObjectTagging(ctx, *po.Bucket, *po.Key, tags) if err != nil { - return "", err + return s3response.PutObjectOutput{}, err } } // Set object legal hold if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) - if err != nil { - return "", err + if err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil { + return s3response.PutObjectOutput{}, err } } @@ -1509,11 +1953,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } retParsed, err := json.Marshal(retention) if err != nil { - return "", fmt.Errorf("parse object lock retention: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) } - err = p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed) - if err != nil { - return "", err + if err := p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil { + return s3response.PutObjectOutput{}, err } } @@ -1521,7 +1964,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e etag := hex.EncodeToString(dataSum[:]) err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(etag)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } ctype := getString(po.ContentType) @@ -1529,7 +1972,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentTypeHdr, []byte(*po.ContentType)) if err != nil { - return "", fmt.Errorf("set content-type attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) } } @@ -1538,19 +1981,32 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentEncHdr, []byte(*po.ContentEncoding)) if err != nil { - return "", fmt.Errorf("set content-encoding attr: %w", err) + return s3response.PutObjectOutput{}, fmt.Errorf("set content-encoding attr: %w", err) } } - return etag, nil + // if the versioning is enabled, generate a new versionID for the object + var versionID string + if p.versioningEnabled() { + versionID = ulid.Make().String() + + if err := p.meta.StoreAttribute(*po.Bucket, *po.Key, versionIdKey, []byte(versionID)); err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + return s3response.PutObjectOutput{ + ETag: etag, + VersionID: versionID, + }, nil } -func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error { +func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { if input.Bucket == nil { - return s3err.GetAPIError(s3err.ErrInvalidBucketName) + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if input.Key == nil { - return s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } bucket := *input.Bucket @@ -1558,44 +2014,179 @@ func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) err _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return s3err.GetAPIError(s3err.ErrNoSuchBucket) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return fmt.Errorf("stat bucket: %w", err) + return nil, fmt.Errorf("stat bucket: %w", err) } objpath := filepath.Join(bucket, object) - fi, err := os.Stat(objpath) - if err != nil { - // AWS returns success if the object does not exist or - // is invalid somehow. - // TODO: log if !errors.Is(err, fs.ErrNotExist) somewhere? + if p.versioningEnabled() { + if *input.VersionId == "" { + // if the versionId is not specified, make the current version a delete marker + _, err := os.Stat(objpath) + if err != nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } - return nil + if err := p.meta.StoreAttribute(bucket, object, deleteMarkerKey, []byte{}); err != nil { + return nil, fmt.Errorf("set delete marker: %w", err) + } + versionId := ulid.Make().String() + if err := p.meta.StoreAttribute(bucket, object, versionIdKey, []byte(versionId)); err != nil { + return nil, fmt.Errorf("set versionId: %w", err) + } + + return &s3.DeleteObjectOutput{ + VersionId: &versionId, + }, nil + } else { + delMarker := true + versionPath := p.genObjVersionPath(bucket, object) + + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + + if string(vId) == *input.VersionId { + // if the specified VersionId is the same as in the latest version, + // remove the latest version, find the latest version from the versioning + // directory and move to the place of the deleted object, to make it the latest + err := os.Remove(objpath) + if err != nil { + return nil, fmt.Errorf("remove obj version: %w", err) + } + + ents, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: input.VersionId, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + return &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: input.VersionId, + }, nil + } + + srcObjVersion, err := ents[len(ents)-1].Info() + if err != nil { + return nil, fmt.Errorf("get file info: %w", err) + } + srcVersionId := srcObjVersion.Name() + sf, err := os.Open(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return nil, fmt.Errorf("open obj version: %w", err) + } + defer sf.Close() + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, srcObjVersion.Size(), acct, doFalloc) + if err != nil { + return nil, fmt.Errorf("open tmp file: %w", err) + } + defer f.cleanup() + + _, err = io.Copy(f, sf) + if err != nil { + return nil, fmt.Errorf("copy object %w", err) + } + + if err := f.link(); err != nil { + return nil, fmt.Errorf("link tmp file: %w", err) + } + + attrs, err := p.meta.ListAttributes(versionPath, srcVersionId) + if err != nil { + return nil, fmt.Errorf("list object attributes: %w", err) + } + + for _, attr := range attrs { + data, err := p.meta.RetrieveAttribute(versionPath, srcVersionId, attr) + if err != nil { + return nil, fmt.Errorf("load %v attribute", attr) + } + + if err := p.meta.StoreAttribute(bucket, object, attr, data); err != nil { + return nil, fmt.Errorf("store %v attribute", attr) + } + } + + if err := os.Remove(filepath.Join(versionPath, srcVersionId)); err != nil { + return nil, fmt.Errorf("remove obj version %w", err) + } + + return &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: input.VersionId, + }, nil + } + + err = os.Remove(filepath.Join(versionPath, *input.VersionId)) + if errors.Is(err, fs.ErrNotExist) { + return &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: input.VersionId, + }, nil + } + if err != nil { + return nil, fmt.Errorf("delete object: %w", err) + } + + return &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: input.VersionId, + }, nil + } + + } + + fi, err := os.Stat(objpath) + if errors.Is(err, fs.ErrNotExist) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + if err != nil { + return &s3.DeleteObjectOutput{}, fmt.Errorf("stat object: %w", err) } if strings.HasSuffix(object, "/") && !fi.IsDir() { // requested object is expecting a directory with a trailing // slash, but the object is not a directory. treat this as // a non-existent object. // AWS returns success if the object does not exist - return nil + return &s3.DeleteObjectOutput{}, nil } err = os.Remove(objpath) if errors.Is(err, fs.ErrNotExist) { - return s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { - return fmt.Errorf("delete object: %w", err) + return nil, fmt.Errorf("delete object: %w", err) } err = p.meta.DeleteAttributes(bucket, object) if err != nil { - return fmt.Errorf("delete object attributes: %w", err) + return nil, fmt.Errorf("delete object attributes: %w", err) } - return p.removeParents(bucket, object) + err = p.removeParents(bucket, object) + if err != nil { + return nil, err + } + + return &s3.DeleteObjectOutput{}, nil } func (p *Posix) removeParents(bucket, object string) error { @@ -1636,12 +2227,16 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) delResult, errs := []types.DeletedObject{}, []types.Error{} for _, obj := range input.Delete.Objects { //TODO: Make the delete operation concurrent - err := p.DeleteObject(ctx, &s3.DeleteObjectInput{ + res, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: input.Bucket, Key: obj.Key, }) if err == nil { - delResult = append(delResult, types.DeletedObject{Key: obj.Key}) + delResult = append(delResult, types.DeletedObject{ + Key: obj.Key, + VersionId: res.VersionId, + DeleteMarker: res.DeleteMarker, + }) } else { serr, ok := err.(s3err.APIError) if ok { @@ -1677,6 +2272,11 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, s3err.GetAPIError(s3err.ErrInvalidRange) } + if !p.versioningEnabled() && *input.VersionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + bucket := *input.Bucket _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1687,6 +2287,25 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } object := *input.Key + if *input.VersionId != "" { + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + if err != nil { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + + if string(vId) != *input.VersionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + } + objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) @@ -1704,6 +2323,21 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if *input.VersionId != "" { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + return &s3.GetObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fi.ModTime()), + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } + } + acceptRange := *input.Range startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange) if err != nil { @@ -1764,6 +2398,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO TagCount: tagCount, ContentRange: &contentRange, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, }, nil } @@ -1808,6 +2443,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO TagCount: tagCount, ContentRange: &contentRange, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, Body: &backend.FileSectionReadCloser{R: rdr, F: f}, }, nil } @@ -1819,6 +2455,12 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. if input.Key == nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + + if !p.versioningEnabled() && *input.VersionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } + bucket := *input.Bucket object := *input.Key @@ -1866,6 +2508,25 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. }, nil } + if *input.VersionId != "" { + vId, err := p.meta.RetrieveAttribute(bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + if err != nil { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + + if string(vId) != *input.VersionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), *input.VersionId) + } + } + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1878,6 +2539,9 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { + if *input.VersionId != "" { + return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId) + } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -1890,6 +2554,21 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + if *input.VersionId != "" { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + return &s3.HeadObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fi.ModTime()), + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } + } + userMetaData := make(map[string]string) contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) @@ -1939,13 +2618,15 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ObjectLockMode: objectLockMode, ObjectLockRetainUntilDate: objectLockRetainUntilDate, StorageClass: types.StorageClassStandard, + VersionId: input.VersionId, }, nil } func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) { data, err := p.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: input.Bucket, - Key: input.Key, + Bucket: input.Bucket, + Key: input.Key, + VersionId: input.VersionId, }) if err != nil { return s3response.GetObjectAttributesResult{}, nil @@ -1972,7 +2653,20 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. if input.ExpectedBucketOwner == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) } - srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/") + + copySourceHdr := *input.CopySource + if copySourceHdr[0] == '/' { + copySourceHdr = copySourceHdr[1:] + } + + cSplitted := strings.Split(copySourceHdr, "?") + copySource := cSplitted[0] + var srcVersionId string + if len(cSplitted) > 1 { + srcVersionId = cSplitted[1] + } + + srcBucket, srcObject, ok := strings.Cut(copySource, "/") if !ok { return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) } @@ -1987,6 +2681,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return nil, fmt.Errorf("stat bucket: %w", err) } + if srcVersionId != "" { + srcBucket = filepath.Join(p.versioningDir, srcBucket) + srcObject = filepath.Join(genObjVersionKey(srcObject), srcVersionId) + } + _, err = os.Stat(dstBucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -2020,6 +2719,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. p.loadUserMetaData(srcBucket, srcObject, mdmap) var etag string + var version *string dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -2044,10 +2744,14 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. b, _ := p.meta.RetrieveAttribute(dstBucket, dstObject, etagkey) etag = string(b) + vId, _ := p.meta.RetrieveAttribute(dstBucket, dstObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + version = backend.GetStringPtr(string(vId)) } else { contentLength := fi.Size() - - etag, err = p.PutObject(ctx, + res, err := p.PutObject(ctx, &s3.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, @@ -2058,6 +2762,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. if err != nil { return nil, err } + etag = res.ETag + version = &res.VersionID } fi, err = os.Stat(dstObjdPath) @@ -2070,6 +2776,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), }, + VersionId: version, + CopySourceVersionId: &srcVersionId, }, nil } diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index a3f2f1a..3086427 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -161,6 +161,48 @@ func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket stri return handleError(err) } +func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error { + _, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: status, + }, + }) + + return handleError(err) +} + +func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { + out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + + return out, handleError(err) +} + +func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + out, err := s.client.ListObjectVersions(ctx, input) + if err != nil { + return s3response.ListVersionsResult{}, handleError(err) + } + + return s3response.ListVersionsResult{ + CommonPrefixes: out.CommonPrefixes, + DeleteMarkers: out.DeleteMarkers, + Delimiter: out.Delimiter, + EncodingType: out.EncodingType, + IsTruncated: out.IsTruncated, + KeyMarker: out.KeyMarker, + MaxKeys: out.MaxKeys, + Name: out.Name, + NextKeyMarker: out.NextKeyMarker, + NextVersionIdMarker: out.NextVersionIdMarker, + Prefix: out.Prefix, + VersionIdMarker: input.VersionIdMarker, + Versions: out.Versions, + }, nil +} + func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { out, err := s.client.CreateMultipartUpload(ctx, input) if err != nil { @@ -304,17 +346,25 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn }, nil } -func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) { +func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) { // streaming backend is not seekable, // use unsigned payload for streaming ops output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions( v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware, )) if err != nil { - return "", handleError(err) + return s3response.PutObjectOutput{}, handleError(err) } - return *output.ETag, nil + var versionID string + if output.VersionId != nil { + versionID = *output.VersionId + } + + return s3response.PutObjectOutput{ + ETag: *output.ETag, + VersionID: versionID, + }, nil } func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { @@ -416,9 +466,9 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu }, nil } -func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { - _, err := s.client.DeleteObject(ctx, input) - return handleError(err) +func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := s.client.DeleteObject(ctx, input) + return res, handleError(err) } func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { diff --git a/backend/walk.go b/backend/walk.go index 047b562..8df944e 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -246,3 +246,189 @@ func contains(a string, strs []string) bool { } return false } + +type WalkVersioningResults struct { + CommonPrefixes []types.CommonPrefix + ObjectVersions []types.ObjectVersion + DelMarkers []types.DeleteMarkerEntry + Truncated bool + NextMarker string + NextVersionIdMarker string +} + +type ObjVersionFuncResult struct { + ObjectVersions []types.ObjectVersion + DelMarkers []types.DeleteMarkerEntry + NextVersionIdMarker string + Truncated bool +} + +type GetVersionsFunc func(path, versionIdMarker string, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error) + +// WalkVersions walks the supplied fs.FS and returns results compatible with +// ListObjectVersions action response +func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) { + cpmap := make(map[string]struct{}) + var objects []types.ObjectVersion + var delMarkers []types.DeleteMarkerEntry + + var pastMarker bool + if keyMarker == "" { + pastMarker = true + } + var nextMarker string + var nextVersionIdMarker string + var truncated bool + + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + // Ignore the root directory + if path == "." { + return nil + } + if contains(d.Name(), skipdirs) { + return fs.SkipDir + } + + if d.IsDir() { + // If prefix is defined and the directory does not match prefix, + // do not descend into the directory because nothing will + // match this prefix. Make sure to append the / at the end of + // directories since this is implied as a directory path name. + // If path is a prefix of prefix, then path could still be + // building to match. So only skip if path isn't a prefix of prefix + // and prefix isn't a prefix of path. + if prefix != "" && + !strings.HasPrefix(path+string(os.PathSeparator), prefix) && + !strings.HasPrefix(prefix, path+string(os.PathSeparator)) { + return fs.SkipDir + } + + // skip directory objects, as they can't have versions + return nil + } + + if !pastMarker { + if path == keyMarker { + pastMarker = true + return nil + } + if path < keyMarker { + return nil + } + } + + // If object doesn't have prefix, don't include in results. + if prefix != "" && !strings.HasPrefix(path, prefix) { + return nil + } + + if delimiter == "" { + // If no delimiter specified, then all files with matching + // prefix are included in results + res, err := getObj(path, versionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d) + if err == ErrSkipObj { + return nil + } + if err != nil { + return fmt.Errorf("file to object %q: %w", path, err) + } + objects = append(objects, res.ObjectVersions...) + delMarkers = append(delMarkers, res.DelMarkers...) + if res.Truncated { + truncated = true + nextMarker = path + nextVersionIdMarker = res.NextVersionIdMarker + return fs.SkipAll + } + + return nil + } + + // Since delimiter is specified, we only want results that + // do not contain the delimiter beyond the prefix. If the + // delimiter exists past the prefix, then the substring + // between the prefix and delimiter is part of common prefixes. + // + // For example: + // prefix = A/ + // delimiter = / + // and objects: + // A/file + // A/B/file + // B/C + // would return: + // objects: A/file + // common prefix: A/B/ + // + // Note: No objects are included past the common prefix since + // these are all rolled up into the common prefix. + // Note: The delimiter can be anything, so we have to operate on + // the full path without any assumptions on posix directory hierarchy + // here. Usually the delimiter will be "/", but thats not required. + suffix := strings.TrimPrefix(path, prefix) + before, _, found := strings.Cut(suffix, delimiter) + if !found { + res, err := getObj(path, versionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d) + if err == ErrSkipObj { + return nil + } + if err != nil { + return fmt.Errorf("file to object %q: %w", path, err) + } + objects = append(objects, res.ObjectVersions...) + delMarkers = append(delMarkers, res.DelMarkers...) + + if res.Truncated { + truncated = true + nextMarker = path + nextVersionIdMarker = res.NextVersionIdMarker + return fs.SkipAll + } + return nil + } + + // Common prefixes are a set, so should not have duplicates. + // These are abstractly a "directory", so need to include the + // delimiter at the end. + cpmap[prefix+before+delimiter] = struct{}{} + if (len(objects) + len(cpmap)) == int(max) { + nextMarker = path + truncated = true + + return fs.SkipAll + } + + return nil + }) + if err != nil { + return WalkVersioningResults{}, err + } + + var commonPrefixStrings []string + for k := range cpmap { + commonPrefixStrings = append(commonPrefixStrings, k) + } + sort.Strings(commonPrefixStrings) + commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings)) + for _, cp := range commonPrefixStrings { + pfx := cp + commonPrefixes = append(commonPrefixes, types.CommonPrefix{ + Prefix: &pfx, + }) + } + + return WalkVersioningResults{ + CommonPrefixes: commonPrefixes, + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: truncated, + NextMarker: nextMarker, + NextVersionIdMarker: nextVersionIdMarker, + }, nil +} diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index b91fde5..3c8421d 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -25,6 +25,7 @@ import ( var ( chownuid, chowngid bool bucketlinks bool + versioningDir string ) func posixCommand() *cli.Command { @@ -61,6 +62,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, EnvVars: []string{"VGW_BUCKET_LINKS"}, Destination: &bucketlinks, }, + &cli.StringFlag{ + Name: "versioning-dir", + Usage: "the directory path to enable bucket versioning", + EnvVars: []string{"VGW_VERSIONING_DIR"}, + Destination: &versioningDir, + }, }, } } @@ -77,9 +84,10 @@ func runPosix(ctx *cli.Context) error { } be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{ - ChownUID: chownuid, - ChownGID: chowngid, - BucketLinks: bucketlinks, + ChownUID: chownuid, + ChownGID: chowngid, + BucketLinks: bucketlinks, + VersioningDir: versioningDir, }) if err != nil { return fmt.Errorf("init posix: %v", err) diff --git a/go.mod b/go.mod index bfe90d8..fc64b5c 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/oklog/ulid/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/ryanuber/go-glob v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3b7a1b1..f77b372 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,9 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= +github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 0feaae5..75c4804 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -53,7 +53,7 @@ var _ backend.Backend = &BackendMock{} // DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error { // panic("mock out the DeleteBucketTagging method") // }, -// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error { +// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { // panic("mock out the DeleteObject method") // }, // DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error { @@ -113,7 +113,7 @@ var _ backend.Backend = &BackendMock{} // ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { // panic("mock out the ListMultipartUploads method") // }, -// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { +// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { // panic("mock out the ListObjectVersions method") // }, // ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { @@ -137,10 +137,10 @@ var _ backend.Backend = &BackendMock{} // PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { // panic("mock out the PutBucketTagging method") // }, -// PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { +// PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { // panic("mock out the PutBucketVersioning method") // }, -// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) { +// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { // panic("mock out the PutObject method") // }, // PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error { @@ -214,7 +214,7 @@ type BackendMock struct { DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error // DeleteObjectFunc mocks the DeleteObject method. - DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error + DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) // DeleteObjectTaggingFunc mocks the DeleteObjectTagging method. DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error @@ -274,7 +274,7 @@ type BackendMock struct { ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) // ListObjectVersionsFunc mocks the ListObjectVersions method. - ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) + ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) // ListObjectsFunc mocks the ListObjects method. ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) @@ -298,10 +298,10 @@ type BackendMock struct { PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error // PutBucketVersioningFunc mocks the PutBucketVersioning method. - PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error + PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error // PutObjectFunc mocks the PutObject method. - PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) + PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) // PutObjectAclFunc mocks the PutObjectAcl method. PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error @@ -632,8 +632,10 @@ type BackendMock struct { PutBucketVersioning []struct { // ContextMoqParam is the contextMoqParam argument value. ContextMoqParam context.Context - // PutBucketVersioningInput is the putBucketVersioningInput argument value. - PutBucketVersioningInput *s3.PutBucketVersioningInput + // Bucket is the bucket argument value. + Bucket string + // Status is the status argument value. + Status types.BucketVersioningStatus } // PutObject holds details about calls to the PutObject method. PutObject []struct { @@ -1154,7 +1156,7 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct { } // DeleteObject calls DeleteObjectFunc. -func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error { +func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { if mock.DeleteObjectFunc == nil { panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called") } @@ -1898,7 +1900,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct { } // ListObjectVersions calls ListObjectVersionsFunc. -func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { +func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { if mock.ListObjectVersionsFunc == nil { panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called") } @@ -2202,21 +2204,23 @@ func (mock *BackendMock) PutBucketTaggingCalls() []struct { } // PutBucketVersioning calls PutBucketVersioningFunc. -func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { +func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { if mock.PutBucketVersioningFunc == nil { panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called") } callInfo := struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus }{ - ContextMoqParam: contextMoqParam, - PutBucketVersioningInput: putBucketVersioningInput, + ContextMoqParam: contextMoqParam, + Bucket: bucket, + Status: status, } mock.lockPutBucketVersioning.Lock() mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo) mock.lockPutBucketVersioning.Unlock() - return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput) + return mock.PutBucketVersioningFunc(contextMoqParam, bucket, status) } // PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning. @@ -2224,12 +2228,14 @@ func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, pu // // len(mockedBackend.PutBucketVersioningCalls()) func (mock *BackendMock) PutBucketVersioningCalls() []struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus } { var calls []struct { - ContextMoqParam context.Context - PutBucketVersioningInput *s3.PutBucketVersioningInput + ContextMoqParam context.Context + Bucket string + Status types.BucketVersioningStatus } mock.lockPutBucketVersioning.RLock() calls = mock.calls.PutBucketVersioning @@ -2238,7 +2244,7 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct { } // PutObject calls PutObjectFunc. -func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) { +func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { if mock.PutObjectFunc == nil { panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 93438dd..499cbb8 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -382,6 +382,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { }) } + action := auth.GetObjectAction + if versionId != "" { + action = auth.GetObjectVersionAction + } + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -390,7 +395,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Acc: acct, Bucket: bucket, Object: key, - Action: auth.GetObjectAction, + Action: action, }) if err != nil { return SendResponse(ctx, err, @@ -410,11 +415,23 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { VersionId: &versionId, }) if err != nil { + if res != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-delete-marker", + Value: "true", + }, + { + Key: "Last-Modified", + Value: res.LastModified.Format(timefmt), + }, + }) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, - Action: metrics.ActionGetObject, + Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }) } @@ -981,8 +998,8 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { objectOwnership := types.ObjectOwnership( ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), ) - mfa := ctx.Get("X-Amz-Mfa") - contentMD5 := ctx.Get("Content-MD5") + // mfa := ctx.Get("X-Amz-Mfa") + // contentMD5 := ctx.Get("Content-MD5") acct := ctx.Locals("account").(auth.Account) isRoot := ctx.Locals("isRoot").(bool) @@ -1136,13 +1153,21 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { }) } - err = c.be.PutBucketVersioning(ctx.Context(), - &s3.PutBucketVersioningInput{ - Bucket: &bucket, - MFA: &mfa, - VersioningConfiguration: &versioningConf, - ContentMD5: &contentMD5, - }) + if versioningConf.Status != types.BucketVersioningStatusEnabled && + versioningConf.Status != types.BucketVersioningStatusSuspended { + if c.debug { + log.Printf("invalid versioning configuration status: %v\n", versioningConf.Status) + } + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }) + } + + err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -2141,6 +2166,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { StorageClass: types.StorageClass(storageClass), }) if err == nil { + hdrs := []utils.CustomHeader{} + if getstring(res.VersionId) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }) + } + if getstring(res.CopySourceVersionId) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-copy-source-version-id", + Value: getstring(res.CopySourceVersionId), + }) + } + utils.SetResponseHeaders(ctx, hdrs) + return SendXMLResponse(ctx, res.CopyObjectResult, err, &MetaOpts{ Logger: c.logger, @@ -2232,7 +2272,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } ctx.Locals("logReqBody", false) - etag, err := c.be.PutObject(ctx.Context(), + res, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{ Bucket: &bucket, Key: &keyStart, @@ -2246,8 +2286,36 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ObjectLockMode: objLock.ObjectLockMode, ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, }) - ctx.Response().Header.Set("ETag", etag) - return SendResponse(ctx, err, + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + ContentLength: contentLength, + EvSender: c.evSender, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + ObjectSize: contentLength, + EventName: s3event.EventObjectCreatedPut, + }) + } + hdrs := []utils.CustomHeader{ + { + Key: "ETag", + Value: res.ETag, + }, + } + + if res.VersionID != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: res.VersionID, + }) + } + + utils.SetResponseHeaders(ctx, hdrs) + + return SendResponse(ctx, nil, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, @@ -2255,7 +2323,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { EvSender: c.evSender, Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, - ObjectETag: &etag, + ObjectETag: &res.ETag, ObjectSize: contentLength, EventName: s3event.EventObjectCreatedPut, }) @@ -2569,6 +2637,8 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { }) } + //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -2604,13 +2674,42 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { }) } - err = c.be.DeleteObject(ctx.Context(), + res, err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{ Bucket: &bucket, Key: &key, VersionId: &versionId, }) - return SendResponse(ctx, err, + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + EvSender: c.evSender, + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDelete, + Status: http.StatusNoContent, + }) + } + + hdrs := []utils.CustomHeader{} + if res.VersionId != nil && *res.VersionId != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: *res.VersionId, + }) + } + if res.DeleteMarker != nil && *res.DeleteMarker { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-delete-marker", + Value: "true", + }) + } + + utils.SetResponseHeaders(ctx, hdrs) + + return SendResponse(ctx, nil, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, @@ -2683,6 +2782,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) + versionId := ctx.Query("versionId") key := ctx.Params("key") keyEnd := ctx.Params("*1") if keyEnd != "" { @@ -2737,8 +2837,21 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &key, PartNumber: partNumber, + VersionId: &versionId, }) if err != nil { + if res != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-delete-marker", + Value: "true", + }, + { + Key: "Last-Modified", + Value: res.LastModified.Format(timefmt), + }, + }) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -2826,6 +2939,12 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: contentType, }) + if getstring(res.VersionId) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-version-id", + Value: getstring(res.VersionId), + }) + } utils.SetResponseHeaders(ctx, headers) return SendResponse(ctx, nil, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 0f1066b..96ff2dd 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -385,8 +385,8 @@ func TestS3ApiController_ListActions(t *testing.T) { GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) { return &s3.GetBucketVersioningOutput{}, nil }, - ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { - return &s3.ListObjectVersionsOutput{}, nil + ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return s3response.ListVersionsResult{}, nil }, GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { return []byte{}, nil @@ -677,7 +677,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) { PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { return nil }, - PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error { + PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { return nil }, PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error { @@ -968,8 +968,8 @@ func TestS3ApiController_PutActions(t *testing.T) { CopyObjectResult: &types.CopyObjectResult{}, }, nil }, - PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) { - return "ETag", nil + PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { + return s3response.PutObjectOutput{}, nil }, UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) { return "hello", nil @@ -1383,8 +1383,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) { GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error { - return nil + DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return &s3.DeleteObjectOutput{}, nil }, AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error { return nil @@ -1414,8 +1414,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) { GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error { - return s3err.GetAPIError(7) + DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) }, GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) diff --git a/s3err/s3err.go b/s3err/s3err.go index 2767d0c..5395007 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -132,6 +132,7 @@ const ( ErrMissingSecurityHeader ErrInvalidMetadataDirective ErrKeyTooLong + ErrInvalidVersionId // Non-AWS errors ErrExistingObjectIsDirectory @@ -517,8 +518,12 @@ var errorCodeResponse = map[ErrorCode]APIError{ HTTPStatusCode: http.StatusNotFound, }, ErrInvalidMetadataDirective: { + Code: "InvalidArgument", + Description: "Unknown metadata directive.", + }, + ErrInvalidVersionId: { Code: "InvalidArgument", - Description: "Unknown metadata directive.", + Description: "Invalid version id specified", HTTPStatusCode: http.StatusBadRequest, }, ErrKeyTooLong: { diff --git a/s3response/s3response.go b/s3response/s3response.go index ed54ad8..865e800 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -23,6 +23,11 @@ import ( const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z" +type PutObjectOutput struct { + ETag string + VersionID string +} + // Part describes part metadata. type Part struct { PartNumber int @@ -360,3 +365,21 @@ type InitiateMultipartUploadResult struct { Key string UploadId string } + +type ListVersionsResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"` + CommonPrefixes []types.CommonPrefix + DeleteMarkers []types.DeleteMarkerEntry `xml:"DeleteMarker"` + Delimiter *string + EncodingType types.EncodingType + IsTruncated *bool + KeyMarker *string + MaxKeys *int32 + Name *string + NextKeyMarker *string + NextVersionIdMarker *string + Prefix *string + RequestCharged types.RequestCharged + VersionIdMarker *string + Versions []types.ObjectVersion `xml:"Version"` +}