mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-18 05:06:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: basic logic implementation of bucket object versioning in posix backend

Browse Source 
New posix backend option --versioning-dir will enable storing object versions
in specified directory.
This commit is contained in:
jonaustin09
2024-09-18 13:04:34 -07:00
committed by Ben McClelland
parent cf067b5d00
commit 8252ecd452
14 changed files with 1264 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
auth/bucket_policy_actions.go
+3
View File
@@ -36,6 +36,7 @@ const (
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
GetObjectVersionAction Action = "s3:GetObjectVersion"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
@@ -75,6 +76,7 @@ var supportedActionList = map[Action]struct{}{
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
@@ -103,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
backend/azure/azure.go
+13 -11
View File
@@ -284,10 +284,10 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin
return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership))
}
func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
tags, err := parseTags(po.Tagging)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}
opts := &blockblob.UploadStreamOptions{
@@ -312,14 +312,14 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,
uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts)
if err != nil {
return "", azureErrToS3Err(err)
return s3response.PutObjectOutput{}, azureErrToS3Err(err)
}
// Set object legal hold
if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}
}
@@ -331,15 +331,17 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,
}
retParsed, err := json.Marshal(retention)
if err != nil {
return "", fmt.Errorf("parse object lock retention: %w", err)
return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err)
}
err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}
}
return string(*uploadResp.ETag), nil
return s3response.PutObjectOutput{
ETag: string(*uploadResp.ETag),
}, nil
}
func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
@@ -675,22 +677,22 @@ Pager:
}, nil
}
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
azerr, ok := err.(*azcore.ResponseError)
if ok && azerr.StatusCode == 404 {
// if the object does not exist, S3 returns success
return nil
return &s3.DeleteObjectOutput{}, nil
}
}
return azureErrToS3Err(err)
return &s3.DeleteObjectOutput{}, azureErrToS3Err(err)
}
func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
delResult, errs := []types.DeletedObject{}, []types.Error{}
for _, obj := range input.Delete.Objects {
err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
_, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: input.Bucket,
Key: obj.Key,
})
backend/backend.go
+11 -11
View File
@@ -38,7 +38,7 @@ type Backend interface {
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
@@ -57,7 +57,7 @@ type Backend interface {
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (string, error)
PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
@@ -65,10 +65,10 @@ type Backend interface {
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -126,7 +126,7 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
@@ -173,8 +173,8 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -197,8 +197,8 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -225,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
}
}
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
backend/posix/posix.go
+777 -69
View File
File diff suppressed because it is too large Load Diff
backend/s3proxy/s3.go
+56 -6
View File
@@ -161,6 +161,48 @@ func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket stri
return handleError(err)
}
func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
_, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucket,
VersioningConfiguration: &types.VersioningConfiguration{
Status: status,
},
})
return handleError(err)
}
func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
Bucket: &bucket,
})
return out, handleError(err)
}
func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
out, err := s.client.ListObjectVersions(ctx, input)
if err != nil {
return s3response.ListVersionsResult{}, handleError(err)
}
return s3response.ListVersionsResult{
CommonPrefixes: out.CommonPrefixes,
DeleteMarkers: out.DeleteMarkers,
Delimiter: out.Delimiter,
EncodingType: out.EncodingType,
IsTruncated: out.IsTruncated,
KeyMarker: out.KeyMarker,
MaxKeys: out.MaxKeys,
Name: out.Name,
NextKeyMarker: out.NextKeyMarker,
NextVersionIdMarker: out.NextVersionIdMarker,
Prefix: out.Prefix,
VersionIdMarker: input.VersionIdMarker,
Versions: out.Versions,
}, nil
}
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
out, err := s.client.CreateMultipartUpload(ctx, input)
if err != nil {
@@ -304,17 +346,25 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
}, nil
}
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
if err != nil {
return "", handleError(err)
return s3response.PutObjectOutput{}, handleError(err)
}
return *output.ETag, nil
var versionID string
if output.VersionId != nil {
versionID = *output.VersionId
}
return s3response.PutObjectOutput{
ETag: *output.ETag,
VersionID: versionID,
}, nil
}
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
@@ -416,9 +466,9 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
}, nil
}
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
_, err := s.client.DeleteObject(ctx, input)
return handleError(err)
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
res, err := s.client.DeleteObject(ctx, input)
return res, handleError(err)
}
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
backend/walk.go
+186
View File
@@ -246,3 +246,189 @@ func contains(a string, strs []string) bool {
}
return false
}
type WalkVersioningResults struct {
CommonPrefixes []types.CommonPrefix
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
Truncated bool
NextMarker string
NextVersionIdMarker string
}
type ObjVersionFuncResult struct {
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
NextVersionIdMarker string
Truncated bool
}
type GetVersionsFunc func(path, versionIdMarker string, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
// WalkVersions walks the supplied fs.FS and returns results compatible with
// ListObjectVersions action response
func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
cpmap := make(map[string]struct{})
var objects []types.ObjectVersion
var delMarkers []types.DeleteMarkerEntry
var pastMarker bool
if keyMarker == "" {
pastMarker = true
}
var nextMarker string
var nextVersionIdMarker string
var truncated bool
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
// Ignore the root directory
if path == "." {
return nil
}
if contains(d.Name(), skipdirs) {
return fs.SkipDir
}
if d.IsDir() {
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
// match this prefix. Make sure to append the / at the end of
// directories since this is implied as a directory path name.
// If path is a prefix of prefix, then path could still be
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
return fs.SkipDir
}
// skip directory objects, as they can't have versions
return nil
}
if !pastMarker {
if path == keyMarker {
pastMarker = true
return nil
}
if path < keyMarker {
return nil
}
}
// If object doesn't have prefix, don't include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return nil
}
if delimiter == "" {
// If no delimiter specified, then all files with matching
// prefix are included in results
res, err := getObj(path, versionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, res.ObjectVersions...)
delMarkers = append(delMarkers, res.DelMarkers...)
if res.Truncated {
truncated = true
nextMarker = path
nextVersionIdMarker = res.NextVersionIdMarker
return fs.SkipAll
}
return nil
}
// Since delimiter is specified, we only want results that
// do not contain the delimiter beyond the prefix. If the
// delimiter exists past the prefix, then the substring
// between the prefix and delimiter is part of common prefixes.
//
// For example:
// prefix = A/
// delimiter = /
// and objects:
// A/file
// A/B/file
// B/C
// would return:
// objects: A/file
// common prefix: A/B/
//
// Note: No objects are included past the common prefix since
// these are all rolled up into the common prefix.
// Note: The delimiter can be anything, so we have to operate on
// the full path without any assumptions on posix directory hierarchy
// here. Usually the delimiter will be "/", but thats not required.
suffix := strings.TrimPrefix(path, prefix)
before, _, found := strings.Cut(suffix, delimiter)
if !found {
res, err := getObj(path, versionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, res.ObjectVersions...)
delMarkers = append(delMarkers, res.DelMarkers...)
if res.Truncated {
truncated = true
nextMarker = path
nextVersionIdMarker = res.NextVersionIdMarker
return fs.SkipAll
}
return nil
}
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimiter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == int(max) {
nextMarker = path
truncated = true
return fs.SkipAll
}
return nil
})
if err != nil {
return WalkVersioningResults{}, err
}
var commonPrefixStrings []string
for k := range cpmap {
commonPrefixStrings = append(commonPrefixStrings, k)
}
sort.Strings(commonPrefixStrings)
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
for _, cp := range commonPrefixStrings {
pfx := cp
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
Prefix: &pfx,
})
}
return WalkVersioningResults{
CommonPrefixes: commonPrefixes,
ObjectVersions: objects,
DelMarkers: delMarkers,
Truncated: truncated,
NextMarker: nextMarker,
NextVersionIdMarker: nextVersionIdMarker,
}, nil
}
cmd/versitygw/posix.go
+11 -3
View File
@@ -25,6 +25,7 @@ import (
var (
chownuid, chowngid bool
bucketlinks bool
versioningDir string
)
func posixCommand() *cli.Command {
@@ -61,6 +62,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
},
&cli.StringFlag{
Name: "versioning-dir",
Usage: "the directory path to enable bucket versioning",
EnvVars: []string{"VGW_VERSIONING_DIR"},
Destination: &versioningDir,
},
},
}
}
@@ -77,9 +84,10 @@ func runPosix(ctx *cli.Context) error {
}
be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
VersioningDir: versioningDir,
})
if err != nil {
return fmt.Errorf("init posix: %v", err)
go.mod
+1
View File
@@ -45,6 +45,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/oklog/ulid/v2 v2.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
go.sum
+3
View File
@@ -129,6 +129,9 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
s3api/controllers/backend_moq_test.go
+29 -23
View File
@@ -53,7 +53,7 @@ var _ backend.Backend = &BackendMock{}
// DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
// panic("mock out the DeleteBucketTagging method")
// },
// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
// panic("mock out the DeleteObject method")
// },
// DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
@@ -113,7 +113,7 @@ var _ backend.Backend = &BackendMock{}
// ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
// panic("mock out the ListMultipartUploads method")
// },
// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
// panic("mock out the ListObjectVersions method")
// },
// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
@@ -137,10 +137,10 @@ var _ backend.Backend = &BackendMock{}
// PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
// panic("mock out the PutBucketTagging method")
// },
// PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
// PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
// panic("mock out the PutBucketVersioning method")
// },
// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
// panic("mock out the PutObject method")
// },
// PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
@@ -214,7 +214,7 @@ type BackendMock struct {
DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
// DeleteObjectFunc mocks the DeleteObject method.
DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
// DeleteObjectTaggingFunc mocks the DeleteObjectTagging method.
DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
@@ -274,7 +274,7 @@ type BackendMock struct {
ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
// ListObjectVersionsFunc mocks the ListObjectVersions method.
ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
@@ -298,10 +298,10 @@ type BackendMock struct {
PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
// PutBucketVersioningFunc mocks the PutBucketVersioning method.
PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error
PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error
// PutObjectFunc mocks the PutObject method.
PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error)
// PutObjectAclFunc mocks the PutObjectAcl method.
PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
@@ -632,8 +632,10 @@ type BackendMock struct {
PutBucketVersioning []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// PutBucketVersioningInput is the putBucketVersioningInput argument value.
PutBucketVersioningInput *s3.PutBucketVersioningInput
// Bucket is the bucket argument value.
Bucket string
// Status is the status argument value.
Status types.BucketVersioningStatus
}
// PutObject holds details about calls to the PutObject method.
PutObject []struct {
@@ -1154,7 +1156,7 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
}
// DeleteObject calls DeleteObjectFunc.
func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
if mock.DeleteObjectFunc == nil {
panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called")
}
@@ -1898,7 +1900,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
}
// ListObjectVersions calls ListObjectVersionsFunc.
func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
if mock.ListObjectVersionsFunc == nil {
panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called")
}
@@ -2202,21 +2204,23 @@ func (mock *BackendMock) PutBucketTaggingCalls() []struct {
}
// PutBucketVersioning calls PutBucketVersioningFunc.
func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
if mock.PutBucketVersioningFunc == nil {
panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called")
}
callInfo := struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
ContextMoqParam context.Context
Bucket string
Status types.BucketVersioningStatus
}{
ContextMoqParam: contextMoqParam,
PutBucketVersioningInput: putBucketVersioningInput,
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Status: status,
}
mock.lockPutBucketVersioning.Lock()
mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo)
mock.lockPutBucketVersioning.Unlock()
return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput)
return mock.PutBucketVersioningFunc(contextMoqParam, bucket, status)
}
// PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning.
@@ -2224,12 +2228,14 @@ func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, pu
//
// len(mockedBackend.PutBucketVersioningCalls())
func (mock *BackendMock) PutBucketVersioningCalls() []struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
ContextMoqParam context.Context
Bucket string
Status types.BucketVersioningStatus
} {
var calls []struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
ContextMoqParam context.Context
Bucket string
Status types.BucketVersioningStatus
}
mock.lockPutBucketVersioning.RLock()
calls = mock.calls.PutBucketVersioning
@@ -2238,7 +2244,7 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct {
}
// PutObject calls PutObjectFunc.
func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
if mock.PutObjectFunc == nil {
panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
}
s3api/controllers/base.go
+136 -17
View File
@@ -382,6 +382,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
})
}
action := auth.GetObjectAction
if versionId != "" {
action = auth.GetObjectVersionAction
}
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
@@ -390,7 +395,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectAction,
Action: action,
})
if err != nil {
return SendResponse(ctx, err,
@@ -410,11 +415,23 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
VersionId: &versionId,
})
if err != nil {
if res != nil {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-delete-marker",
Value: "true",
},
{
Key: "Last-Modified",
Value: res.LastModified.Format(timefmt),
},
})
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
Action: metrics.ActionHeadObject,
BucketOwner: parsedAcl.Owner,
})
}
@@ -981,8 +998,8 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
objectOwnership := types.ObjectOwnership(
ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)),
)
mfa := ctx.Get("X-Amz-Mfa")
contentMD5 := ctx.Get("Content-MD5")
// mfa := ctx.Get("X-Amz-Mfa")
// contentMD5 := ctx.Get("Content-MD5")
acct := ctx.Locals("account").(auth.Account)
isRoot := ctx.Locals("isRoot").(bool)
@@ -1136,13 +1153,21 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
})
}
err = c.be.PutBucketVersioning(ctx.Context(),
&s3.PutBucketVersioningInput{
Bucket: &bucket,
MFA: &mfa,
VersioningConfiguration: &versioningConf,
ContentMD5: &contentMD5,
})
if versioningConf.Status != types.BucketVersioningStatusEnabled &&
versioningConf.Status != types.BucketVersioningStatusSuspended {
if c.debug {
log.Printf("invalid versioning configuration status: %v\n", versioningConf.Status)
}
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionPutBucketVersioning,
BucketOwner: parsedAcl.Owner,
})
}
err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status)
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
@@ -2141,6 +2166,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
StorageClass: types.StorageClass(storageClass),
})
if err == nil {
hdrs := []utils.CustomHeader{}
if getstring(res.VersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
if getstring(res.CopySourceVersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-copy-source-version-id",
Value: getstring(res.CopySourceVersionId),
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendXMLResponse(ctx, res.CopyObjectResult, err,
&MetaOpts{
Logger: c.logger,
@@ -2232,7 +2272,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
ctx.Locals("logReqBody", false)
etag, err := c.be.PutObject(ctx.Context(),
res, err := c.be.PutObject(ctx.Context(),
&s3.PutObjectInput{
Bucket: &bucket,
Key: &keyStart,
@@ -2246,8 +2286,36 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ObjectLockMode: objLock.ObjectLockMode,
ObjectLockLegalHoldStatus: objLock.LegalHoldStatus,
})
ctx.Response().Header.Set("ETag", etag)
return SendResponse(ctx, err,
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
ContentLength: contentLength,
EvSender: c.evSender,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
ObjectSize: contentLength,
EventName: s3event.EventObjectCreatedPut,
})
}
hdrs := []utils.CustomHeader{
{
Key: "ETag",
Value: res.ETag,
},
}
if res.VersionID != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: res.VersionID,
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -2255,7 +2323,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
EvSender: c.evSender,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
ObjectETag: &etag,
ObjectETag: &res.ETag,
ObjectSize: contentLength,
EventName: s3event.EventObjectCreatedPut,
})
@@ -2569,6 +2637,8 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
})
}
//TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object
err := auth.VerifyAccess(ctx.Context(), c.be,
auth.AccessOptions{
Readonly: c.readonly,
@@ -2604,13 +2674,42 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
})
}
err = c.be.DeleteObject(ctx.Context(),
res, err := c.be.DeleteObject(ctx.Context(),
&s3.DeleteObjectInput{
Bucket: &bucket,
Key: &key,
VersionId: &versionId,
})
return SendResponse(ctx, err,
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
EvSender: c.evSender,
Action: metrics.ActionDeleteObject,
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectRemovedDelete,
Status: http.StatusNoContent,
})
}
hdrs := []utils.CustomHeader{}
if res.VersionId != nil && *res.VersionId != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: *res.VersionId,
})
}
if res.DeleteMarker != nil && *res.DeleteMarker {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-delete-marker",
Value: "true",
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -2683,6 +2782,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
isRoot := ctx.Locals("isRoot").(bool)
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
partNumberQuery := int32(ctx.QueryInt("partNumber", -1))
versionId := ctx.Query("versionId")
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
if keyEnd != "" {
@@ -2737,8 +2837,21 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Bucket: &bucket,
Key: &key,
PartNumber: partNumber,
VersionId: &versionId,
})
if err != nil {
if res != nil {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-delete-marker",
Value: "true",
},
{
Key: "Last-Modified",
Value: res.LastModified.Format(timefmt),
},
})
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
@@ -2826,6 +2939,12 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: contentType,
})
if getstring(res.VersionId) != "" {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
utils.SetResponseHeaders(ctx, headers)
return SendResponse(ctx, nil,
s3api/controllers/base_test.go
+9 -9
View File
@@ -385,8 +385,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return &s3.GetBucketVersioningOutput{}, nil
},
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return &s3.ListObjectVersionsOutput{}, nil
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, nil
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return []byte{}, nil
@@ -677,7 +677,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
return nil
},
PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
return nil
},
PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
@@ -968,8 +968,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
CopyObjectResult: &types.CopyObjectResult{},
}, nil
},
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
return "ETag", nil
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, nil
},
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
return "hello", nil
@@ -1383,8 +1383,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return nil
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return &s3.DeleteObjectOutput{}, nil
},
AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
return nil
@@ -1414,8 +1414,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(7)
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
s3err/s3err.go
+6 -1
View File
@@ -132,6 +132,7 @@ const (
ErrMissingSecurityHeader
ErrInvalidMetadataDirective
ErrKeyTooLong
ErrInvalidVersionId
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -517,8 +518,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
HTTPStatusCode: http.StatusNotFound,
},
ErrInvalidMetadataDirective: {
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
},
ErrInvalidVersionId: {
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
Description: "Invalid version id specified",
HTTPStatusCode: http.StatusBadRequest,
},
ErrKeyTooLong: {
s3response/s3response.go
+23
View File
@@ -23,6 +23,11 @@ import (
const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
type PutObjectOutput struct {
ETag string
VersionID string
}
// Part describes part metadata.
type Part struct {
PartNumber int
@@ -360,3 +365,21 @@ type InitiateMultipartUploadResult struct {
Key string
UploadId string
}
type ListVersionsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"`
CommonPrefixes []types.CommonPrefix
DeleteMarkers []types.DeleteMarkerEntry `xml:"DeleteMarker"`
Delimiter *string
EncodingType types.EncodingType
IsTruncated *bool
KeyMarker *string
MaxKeys *int32
Name *string
NextKeyMarker *string
NextVersionIdMarker *string
Prefix *string
RequestCharged types.RequestCharged
VersionIdMarker *string
Versions []types.ObjectVersion `xml:"Version"`
}