From 684ab2371bfae2c87386f37910dfe24926f340b2 Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Mon, 26 Aug 2024 16:20:44 -0400 Subject: [PATCH] fix: Changed ListObjects and ListObjectsV2 actions return types Changed ListObjectsV2 and ListObjects actions return types from *s3.ListObjects(V2)Output to s3response.ListObjects(V2)Result. Changed the listing objects timestamp to RFC3339 to match AWS S3 objects timestamp. Fixes #752 --- backend/azure/azure.go | 24 +++++------ backend/backend.go | 12 +++--- backend/posix/posix.go | 52 ++++++++++++----------- backend/s3proxy/s3.go | 60 +++++++++++++++++++++++++-- backend/scoutfs/scoutfs.go | 55 ++++++++++++------------ backend/walk.go | 7 ++-- backend/walk_test.go | 29 ++++++------- s3api/controllers/backend_moq_test.go | 12 +++--- s3api/controllers/base.go | 5 +-- s3api/controllers/base_test.go | 12 +++--- s3response/s3response.go | 40 ++++++++++++++++++ 11 files changed, 201 insertions(+), 107 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 2ca3abf5..625b9347 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -525,14 +525,14 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt }, nil } -func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{ Marker: input.Marker, MaxResults: input.MaxKeys, Prefix: input.Prefix, }) - var objects []types.Object + var objects []s3response.Object var nextMarker *string var isTruncated bool var maxKeys int32 = math.MaxInt32 @@ -545,7 +545,7 @@ Pager: for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { - return nil, azureErrToS3Err(err) + return s3response.ListObjectsResult{}, azureErrToS3Err(err) } for _, v := range resp.Segment.BlobItems { @@ -556,10 +556,10 @@ Pager: if len(objects) >= int(maxKeys) { break Pager } - objects = append(objects, types.Object{ + objects = append(objects, s3response.Object{ ETag: (*string)(v.Properties.ETag), Key: v.Name, - LastModified: v.Properties.LastModified, + LastModified: backend.GetStringPtr(v.Properties.LastModified.UTC().Format(backend.RFC3339TimeFormat)), Size: v.Properties.ContentLength, StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier), }) @@ -568,7 +568,7 @@ Pager: // TODO: generate common prefixes when appropriate - return &s3.ListObjectsOutput{ + return s3response.ListObjectsResult{ Contents: objects, Marker: input.Marker, MaxKeys: input.MaxKeys, @@ -580,7 +580,7 @@ Pager: }, nil } -func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { marker := "" if *input.ContinuationToken > *input.StartAfter { marker = *input.ContinuationToken @@ -593,7 +593,7 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input Prefix: input.Prefix, }) - var objects []types.Object + var objects []s3response.Object var nextMarker *string var isTruncated bool var maxKeys int32 = math.MaxInt32 @@ -606,7 +606,7 @@ Pager: for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { - return nil, azureErrToS3Err(err) + return s3response.ListObjectsV2Result{}, azureErrToS3Err(err) } for _, v := range resp.Segment.BlobItems { if nextMarker == nil && *resp.NextMarker != "" { @@ -617,10 +617,10 @@ Pager: break Pager } nextMarker = resp.NextMarker - objects = append(objects, types.Object{ + objects = append(objects, s3response.Object{ ETag: (*string)(v.Properties.ETag), Key: v.Name, - LastModified: v.Properties.LastModified, + LastModified: backend.GetStringPtr(v.Properties.LastModified.UTC().Format(backend.RFC3339TimeFormat)), Size: v.Properties.ContentLength, StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier), }) @@ -629,7 +629,7 @@ Pager: // TODO: generate common prefixes when appropriate - return &s3.ListObjectsV2Output{ + return s3response.ListObjectsV2Result{ Contents: objects, ContinuationToken: input.ContinuationToken, MaxKeys: input.MaxKeys, diff --git a/backend/backend.go b/backend/backend.go index 00c6b824..3976be21 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -63,8 +63,8 @@ type Backend interface { GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) - ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) - ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) + ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) DeleteObject(context.Context, *s3.DeleteObjectInput) error DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error @@ -191,11 +191,11 @@ func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttr func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index eab959fa..e9839f65 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -2082,9 +2082,9 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. }, nil } -func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { if input.Bucket == nil { - return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } bucket := *input.Bucket prefix := "" @@ -2106,20 +2106,20 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err) } fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, p.fileToObj(bucket), []string{metaTmpDir}) if err != nil { - return nil, fmt.Errorf("walk %v: %w", bucket, err) + return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } - return &s3.ListObjectsOutput{ + return s3response.ListObjectsResult{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, @@ -2133,33 +2133,34 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s } func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { - return func(path string, d fs.DirEntry) (types.Object, error) { + return func(path string, d fs.DirEntry) (s3response.Object, error) { if d.IsDir() { // directory object only happens if directory empty // check to see if this is a directory object by checking etag etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get etag: %w", err) + return s3response.Object{}, fmt.Errorf("get etag: %w", err) } etag := string(etagBytes) fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } size := int64(0) + mDate := fi.ModTime().UTC().Format(backend.RFC3339TimeFormat) - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: &mDate, Size: &size, }, nil } @@ -2167,10 +2168,10 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { // file object, get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return types.Object{}, fmt.Errorf("get etag: %w", err) + return s3response.Object{}, fmt.Errorf("get etag: %w", err) } // note: meta.ErrNoSuchKey will return etagBytes = []byte{} // so this will just set etag to "" if its not already set @@ -2179,26 +2180,27 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } size := fi.Size() + mDate := fi.ModTime().UTC().Format(backend.RFC3339TimeFormat) - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: &mDate, Size: &size, }, nil } } -func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { if input.Bucket == nil { - return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } bucket := *input.Bucket prefix := "" @@ -2228,22 +2230,22 @@ func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err) } fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, p.fileToObj(bucket), []string{metaTmpDir}) if err != nil { - return nil, fmt.Errorf("walk %v: %w", bucket, err) + return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } count := int32(len(results.Objects)) - return &s3.ListObjectsV2Output{ + return s3response.ListObjectsV2Result{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 6960e948..55cb406a 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -373,14 +373,47 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s return out, handleError(err) } -func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { out, err := s.client.ListObjects(ctx, input) - return out, handleError(err) + if err != nil { + return s3response.ListObjectsResult{}, handleError(err) + } + + contents := convertObjects(out.Contents) + + return s3response.ListObjectsResult{ + CommonPrefixes: out.CommonPrefixes, + Contents: contents, + Delimiter: out.Delimiter, + IsTruncated: out.IsTruncated, + Marker: out.Marker, + MaxKeys: out.MaxKeys, + Name: out.Name, + NextMarker: out.NextMarker, + Prefix: out.Prefix, + }, nil } -func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { out, err := s.client.ListObjectsV2(ctx, input) - return out, handleError(err) + if err != nil { + return s3response.ListObjectsV2Result{}, handleError(err) + } + + contents := convertObjects(out.Contents) + + return s3response.ListObjectsV2Result{ + CommonPrefixes: out.CommonPrefixes, + Contents: contents, + Delimiter: out.Delimiter, + IsTruncated: out.IsTruncated, + ContinuationToken: out.ContinuationToken, + MaxKeys: out.MaxKeys, + Name: out.Name, + NextContinuationToken: out.NextContinuationToken, + Prefix: out.Prefix, + KeyCount: out.KeyCount, + }, nil } func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { @@ -738,3 +771,22 @@ func base64Decode(encoded string) ([]byte, error) { } return decoded, nil } + +func convertObjects(objs []types.Object) []s3response.Object { + result := make([]s3response.Object, len(objs)) + + for _, obj := range objs { + lastModified := obj.LastModified.UTC().Format(backend.RFC3339TimeFormat) + result = append(result, s3response.Object{ + ETag: obj.ETag, + Key: obj.Key, + LastModified: &lastModified, + Owner: obj.Owner, + Size: obj.Size, + RestoreStatus: obj.RestoreStatus, + StorageClass: obj.StorageClass, + }) + } + + return result +} diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index fb48b0cb..30961bf4 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -36,6 +36,7 @@ import ( "github.com/versity/versitygw/backend/meta" "github.com/versity/versitygw/backend/posix" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" ) type ScoutfsOpts struct { @@ -733,9 +734,9 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) return tags, nil } -func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { if input.Bucket == nil { - return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } bucket := *input.Bucket prefix := "" @@ -757,20 +758,20 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) ( _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err) } fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, s.fileToObj(bucket), []string{metaTmpDir}) if err != nil { - return nil, fmt.Errorf("walk %v: %w", bucket, err) + return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } - return &s3.ListObjectsOutput{ + return s3response.ListObjectsResult{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, @@ -783,9 +784,9 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) ( }, nil } -func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { if input.Bucket == nil { - return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } bucket := *input.Bucket prefix := "" @@ -807,20 +808,20 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err) } fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys), s.fileToObj(bucket), []string{metaTmpDir}) if err != nil { - return nil, fmt.Errorf("walk %v: %w", bucket, err) + return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } - return &s3.ListObjectsV2Output{ + return s3response.ListObjectsV2Result{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, @@ -834,34 +835,34 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu } func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { - return func(path string, d fs.DirEntry) (types.Object, error) { + return func(path string, d fs.DirEntry) (s3response.Object, error) { objPath := filepath.Join(bucket, path) if d.IsDir() { // directory object only happens if directory empty // check to see if this is a directory object by checking etag etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey) if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get etag: %w", err) + return s3response.Object{}, fmt.Errorf("get etag: %w", err) } etag := string(etagBytes) fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } key := path + "/" - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &key, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: backend.GetStringPtr(fi.ModTime().UTC().Format(backend.RFC3339TimeFormat)), StorageClass: types.ObjectStorageClassStandard, }, nil } @@ -869,10 +870,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { // file object, get object info and fill out object data b, err := s.meta.RetrieveAttribute(bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return types.Object{}, fmt.Errorf("get etag: %w", err) + return s3response.Object{}, fmt.Errorf("get etag: %w", err) } // note: meta.ErrNoSuchKey will return etagBytes = []byte{} // so this will just set etag to "" if its not already set @@ -881,10 +882,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } sc := types.ObjectStorageClassStandard @@ -893,10 +894,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { // If so, we will return the InvalidObjectState error. st, err := statMore(objPath) if errors.Is(err, fs.ErrNotExist) { - return types.Object{}, backend.ErrSkipObj + return s3response.Object{}, backend.ErrSkipObj } if err != nil { - return types.Object{}, fmt.Errorf("stat more: %w", err) + return s3response.Object{}, fmt.Errorf("stat more: %w", err) } if st.Offline_blocks != 0 { sc = types.ObjectStorageClassGlacier @@ -905,10 +906,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { size := fi.Size() - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: backend.GetStringPtr(fi.ModTime().UTC().Format(backend.RFC3339TimeFormat)), Size: &size, StorageClass: sc, }, nil diff --git a/backend/walk.go b/backend/walk.go index 7ffef172..d2f1cea0 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -24,16 +24,17 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3response" ) type WalkResults struct { CommonPrefixes []types.CommonPrefix - Objects []types.Object + Objects []s3response.Object Truncated bool NextMarker string } -type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error) +type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error) var ErrSkipObj = errors.New("skip this object") @@ -41,7 +42,7 @@ var ErrSkipObj = errors.New("skip this object") // objects responses func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) { cpmap := make(map[string]struct{}) - var objects []types.Object + var objects []s3response.Object var pastMarker bool if marker == "" { diff --git a/backend/walk_test.go b/backend/walk_test.go index 19f03156..9adbc4c0 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -27,6 +27,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3response" ) type walkTest struct { @@ -35,19 +36,19 @@ type walkTest struct { getobj backend.GetObjFunc } -func getObj(path string, d fs.DirEntry) (types.Object, error) { +func getObj(path string, d fs.DirEntry) (s3response.Object, error) { if d.IsDir() { etag := getMD5(path) fi, err := d.Info() if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: backend.GetStringPtr(fi.ModTime().UTC().Format(backend.RFC3339TimeFormat)), }, nil } @@ -55,15 +56,15 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) { fi, err := d.Info() if err != nil { - return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) } size := fi.Size() - return types.Object{ + return s3response.Object{ ETag: &etag, Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), + LastModified: backend.GetStringPtr(fi.ModTime().UTC().Format(backend.RFC3339TimeFormat)), Size: &size, }, nil } @@ -89,7 +90,7 @@ func TestWalk(t *testing.T) { CommonPrefixes: []types.CommonPrefix{{ Prefix: backend.GetStringPtr("photos/"), }}, - Objects: []types.Object{{ + Objects: []s3response.Object{{ Key: backend.GetStringPtr("sample.jpg"), }}, }, @@ -104,7 +105,7 @@ func TestWalk(t *testing.T) { CommonPrefixes: []types.CommonPrefix{{ Prefix: backend.GetStringPtr("test/"), }}, - Objects: []types.Object{}, + Objects: []s3response.Object{}, }, getobj: getObj, }, @@ -171,7 +172,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string { return res + "]" } -func compareObjects(a, b []types.Object) bool { +func compareObjects(a, b []s3response.Object) bool { if len(a) == 0 && len(b) == 0 { return true } @@ -187,7 +188,7 @@ func compareObjects(a, b []types.Object) bool { return false } -func containsObject(c types.Object, list []types.Object) bool { +func containsObject(c s3response.Object, list []s3response.Object) bool { for _, cp := range list { if *c.Key == *cp.Key { return true @@ -196,7 +197,7 @@ func containsObject(c types.Object, list []types.Object) bool { return false } -func printObjects(list []types.Object) string { +func printObjects(list []s3response.Object) string { res := "[" for _, cp := range list { if res == "[" { @@ -239,8 +240,8 @@ func TestWalkStop(t *testing.T) { go func() { defer wg.Done() _, err = backend.Walk(ctx, s, "", "/", "", 1000, - func(path string, d fs.DirEntry) (types.Object, error) { - return types.Object{}, nil + func(path string, d fs.DirEntry) (s3response.Object, error) { + return s3response.Object{}, nil }, []string{}) }() diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 5e1af217..0feaae5d 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -116,10 +116,10 @@ var _ backend.Backend = &BackendMock{} // ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) { // panic("mock out the ListObjectVersions method") // }, -// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { // panic("mock out the ListObjects method") // }, -// ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +// ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { // panic("mock out the ListObjectsV2 method") // }, // ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) { @@ -277,10 +277,10 @@ type BackendMock struct { ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) // ListObjectsFunc mocks the ListObjects method. - ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) + ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) // ListObjectsV2Func mocks the ListObjectsV2 method. - ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) // ListPartsFunc mocks the ListParts method. ListPartsFunc func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) @@ -1934,7 +1934,7 @@ func (mock *BackendMock) ListObjectVersionsCalls() []struct { } // ListObjects calls ListObjectsFunc. -func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { +func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { if mock.ListObjectsFunc == nil { panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called") } @@ -1970,7 +1970,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct { } // ListObjectsV2 calls ListObjectsV2Func. -func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { +func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { if mock.ListObjectsV2Func == nil { panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 78ff020d..59be03f8 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -952,10 +952,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Delimiter: &delimiter, MaxKeys: &maxkeys, }) - return SendXMLResponse(ctx, struct { - *s3.ListObjectsOutput - XMLName struct{} `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` - }{ListObjectsOutput: res}, err, + return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 5c633ca5..0f1066b7 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -373,11 +373,11 @@ func TestS3ApiController_ListActions(t *testing.T) { ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { return s3response.ListMultipartUploadsResult{}, nil }, - ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { - return &s3.ListObjectsV2Output{}, nil + ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + return s3response.ListObjectsV2Result{}, nil }, - ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { - return &s3.ListObjectsOutput{}, nil + ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return s3response.ListObjectsResult{}, nil }, GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { return map[string]string{}, nil @@ -416,8 +416,8 @@ func TestS3ApiController_ListActions(t *testing.T) { GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) + ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) }, GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) diff --git a/s3response/s3response.go b/s3response/s3response.go index e7ed5e4b..dfe8c77e 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -91,6 +91,46 @@ type ListMultipartUploadsResult struct { CommonPrefixes []CommonPrefix } +type ListObjectsResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + Name *string + Prefix *string + Marker *string + NextMarker *string + MaxKeys *int32 + Delimiter *string + IsTruncated *bool + Contents []Object + CommonPrefixes []types.CommonPrefix + EncodingType types.EncodingType +} + +type ListObjectsV2Result struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + Name *string + Prefix *string + StartAfter *string + ContinuationToken *string + NextContinuationToken *string + KeyCount *int32 + MaxKeys *int32 + Delimiter *string + IsTruncated *bool + Contents []Object + CommonPrefixes []types.CommonPrefix + EncodingType types.EncodingType +} + +type Object struct { + ETag *string + Key *string + LastModified *string + Owner *types.Owner + RestoreStatus *types.RestoreStatus + Size *int64 + StorageClass types.ObjectStorageClass +} + // Upload describes in progress multipart upload type Upload struct { Key string