From 01be7a2a6b6a1ef4208a17dd6241e61faaf3f9ca Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 23 May 2023 11:37:49 -0700 Subject: [PATCH] posix: add list objects --- backend/backend.go | 10 +- backend/backend_moq_test.go | 12 +- backend/common.go | 6 + backend/posix/posix.go | 212 +++++++++++++++++++++++++- s3api/controllers/backend_moq_test.go | 27 ++-- s3api/controllers/base_test.go | 10 +- 6 files changed, 243 insertions(+), 34 deletions(-) diff --git a/backend/backend.go b/backend/backend.go index a6715f9c..81e52578 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -10,7 +10,7 @@ import ( ) //go:generate moq -out backend_moq_test.go . Backend -//go:generate moq -out ../s3api/controllers/backend_moq_test.go . Backend +//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend type Backend interface { fmt.Stringer GetIAMConfig() ([]byte, error) @@ -36,8 +36,8 @@ type Backend interface { GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) - ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) - ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) + ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) + ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) DeleteObject(bucket, object string) error DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error PutBucketAcl(*s3.PutBucketAclInput) error @@ -144,10 +144,10 @@ func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } diff --git a/backend/backend_moq_test.go b/backend/backend_moq_test.go index 1743a64f..4a91c279 100644 --- a/backend/backend_moq_test.go +++ b/backend/backend_moq_test.go @@ -80,10 +80,10 @@ var _ Backend = &BackendMock{} // ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { // panic("mock out the ListObjectParts method") // }, -// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { // panic("mock out the ListObjects method") // }, -// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { // panic("mock out the ListObjectsV2 method") // }, // PutBucketFunc: func(bucket string) error { @@ -190,10 +190,10 @@ type BackendMock struct { ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) // ListObjectsFunc mocks the ListObjects method. - ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) + ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) // ListObjectsV2Func mocks the ListObjectsV2 method. - ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) + ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) // PutBucketFunc mocks the PutBucket method. PutBucketFunc func(bucket string) error @@ -1270,7 +1270,7 @@ func (mock *BackendMock) ListObjectPartsCalls() []struct { } // ListObjects calls ListObjectsFunc. -func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { if mock.ListObjectsFunc == nil { panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called") } @@ -1318,7 +1318,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct { } // ListObjectsV2 calls ListObjectsV2Func. -func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { if mock.ListObjectsV2Func == nil { panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called") } diff --git a/backend/common.go b/backend/common.go index 080ee865..2afc2b6b 100644 --- a/backend/common.go +++ b/backend/common.go @@ -14,6 +14,12 @@ func (d ByBucketName) Len() int { return len(d) } func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] } func (d ByBucketName) Less(i, j int) bool { return *d[i].Name < *d[j].Name } +type ByObjectName []types.Object + +func (d ByObjectName) Len() int { return len(d) } +func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] } +func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key } + func GetStringPtr(s string) *string { return &s } diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 13e12df1..b5bd9a74 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -954,9 +955,212 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (* }, nil } -func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { + _, err := os.Stat(bucket) + if err != nil && os.IsNotExist(err) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + results, err := walk(bucket, prefix, delim, marker, maxkeys) + if err != nil { + return nil, fmt.Errorf("walk %v: %w", bucket, err) + } + + return &s3.ListObjectsOutput{ + CommonPrefixes: results.commonPrefixes, + Contents: results.objects, + Delimiter: &delim, + IsTruncated: results.truncated, + Marker: &marker, + MaxKeys: int32(maxkeys), + Name: &bucket, + NextMarker: &results.nextMarker, + Prefix: &prefix, + }, nil } -func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { + _, err := os.Stat(bucket) + if err != nil && os.IsNotExist(err) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + results, err := walk(bucket, prefix, delim, marker, maxkeys) + if err != nil { + return nil, fmt.Errorf("walk %v: %w", bucket, err) + } + + return &s3.ListObjectsV2Output{ + CommonPrefixes: results.commonPrefixes, + Contents: results.objects, + Delimiter: &delim, + IsTruncated: results.truncated, + ContinuationToken: &marker, + MaxKeys: int32(maxkeys), + Name: &bucket, + NextContinuationToken: &results.nextMarker, + Prefix: &prefix, + }, nil +} + +type walkResults struct { + commonPrefixes []types.CommonPrefix + objects []types.Object + truncated bool + nextMarker string +} + +func walk(root, prefix, delimiter, marker string, max int) (walkResults, error) { + fileSystem := os.DirFS(root) + cpmap := make(map[string]struct{}) + var objects []types.Object + + var pastMarker bool + if marker == "" { + pastMarker = true + } + + var pastMax bool + var newMarker string + var truncated bool + + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if pastMax { + newMarker = path + truncated = true + return fs.SkipAll + } + + if d.IsDir() { + // Ignore the root directory + if path == "." { + return nil + } + + // If prefix is defined and the directory does not match prefix, + // do not descend into the directory because nothing will + // match this prefix. Make sure to append the / at the end of + // directories since this is implied as a directory path name. + if prefix != "" && !strings.HasPrefix(path+string(os.PathSeparator), prefix) { + return fs.SkipDir + } + + // TODO: special case handling if directory is empty + // and was "PUT" explicitly + return nil + } + + if !pastMarker { + if path != marker { + return nil + } + pastMarker = true + } + + // If object doesnt have prefix, dont include in results. + if prefix != "" && !strings.HasPrefix(path, prefix) { + return nil + } + + if delimiter == "" { + // If no delimeter specified, then all files with matching + // prefix are included in results + fi, err := d.Info() + if err != nil { + return fmt.Errorf("get info for %v: %w", path, err) + } + + objects = append(objects, types.Object{ + ETag: new(string), + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: fi.Size(), + }) + if (len(objects) + len(cpmap)) == max { + pastMax = true + } + return nil + } + + // Since delimiter is specified, we only want results that + // do not contain the delimiter beyond the prefix. If the + // delimiter exists past the prefix, then the substring + // between the prefix and delimiter is part of common prefixes. + // + // For example: + // prefix = A/ + // delimeter = / + // and objects: + // A/file + // A/B/file + // B/C + // would return: + // objects: A/file + // common prefix: A/B/ + // + // Note: No obects are included past the common prefix since + // these are all rolled up into the common prefix. + // Note: The delimeter can be anything, so we have to operate on + // the full path without any assumptions on posix directory heirarchy + // here. Usually the delimeter with be "/", but thats not required. + suffix := strings.TrimPrefix(path, prefix) + before, _, found := strings.Cut(suffix, delimiter) + if !found { + fi, err := d.Info() + if err != nil { + return fmt.Errorf("get info for %v: %w", path, err) + } + objects = append(objects, types.Object{ + ETag: new(string), + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: fi.Size(), + }) + if (len(objects) + len(cpmap)) == max { + pastMax = true + } + return nil + } + + // Common prefixes are a set, so should not have duplicates. + // These are abstractly a "directory", so need to include the + // delimeter at the end. + cpmap[prefix+before+delimiter] = struct{}{} + if (len(objects) + len(cpmap)) == max { + pastMax = true + } + + return nil + }) + if err != nil { + return walkResults{}, err + } + + commonPrefixStrings := make([]string, 0, len(cpmap)) + for k := range cpmap { + commonPrefixStrings = append(commonPrefixStrings, k) + } + sort.Strings(commonPrefixStrings) + commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings)) + for _, cp := range commonPrefixStrings { + commonPrefixes = append(commonPrefixes, types.CommonPrefix{ + Prefix: &cp, + }) + } + + return walkResults{ + commonPrefixes: commonPrefixes, + objects: objects, + truncated: truncated, + nextMarker: newMarker, + }, nil } diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index c2f48752..aff5385e 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -4,23 +4,22 @@ package controllers import ( - "io" - "sync" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/scoutgw/backend" + "io" + "sync" ) -// Ensure, that BackendMock does implement Backend. +// Ensure, that BackendMock does implement backend.Backend. // If this is not the case, regenerate this file with moq. var _ backend.Backend = &BackendMock{} -// BackendMock is a mock implementation of Backend. +// BackendMock is a mock implementation of backend.Backend. // // func TestSomethingThatUsesBackend(t *testing.T) { // -// // make and configure a mocked Backend +// // make and configure a mocked backend.Backend // mockedBackend := &BackendMock{ // AbortMultipartUploadFunc: func(abortMultipartUploadInput *s3.AbortMultipartUploadInput) error { // panic("mock out the AbortMultipartUpload method") @@ -82,10 +81,10 @@ var _ backend.Backend = &BackendMock{} // ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { // panic("mock out the ListObjectParts method") // }, -// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { // panic("mock out the ListObjects method") // }, -// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { // panic("mock out the ListObjectsV2 method") // }, // PutBucketFunc: func(bucket string) error { @@ -100,7 +99,7 @@ var _ backend.Backend = &BackendMock{} // PutObjectAclFunc: func(putObjectAclInput *s3.PutObjectAclInput) error { // panic("mock out the PutObjectAcl method") // }, -// PutObjectPartFunc: func(bucket string, object string, uploadID string, part int, r io.Reader) (string, error) { +// PutObjectPartFunc: func(bucket string, object string, uploadID string, part int, length int64, r io.Reader) (string, error) { // panic("mock out the PutObjectPart method") // }, // RemoveTagsFunc: func(bucket string, object string) error { @@ -126,7 +125,7 @@ var _ backend.Backend = &BackendMock{} // }, // } // -// // use mockedBackend in code that requires Backend +// // use mockedBackend in code that requires backend.Backend // // and then make assertions. // // } @@ -192,10 +191,10 @@ type BackendMock struct { ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) // ListObjectsFunc mocks the ListObjects method. - ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) + ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) // ListObjectsV2Func mocks the ListObjectsV2 method. - ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) + ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) // PutBucketFunc mocks the PutBucket method. PutBucketFunc func(bucket string) error @@ -1272,7 +1271,7 @@ func (mock *BackendMock) ListObjectPartsCalls() []struct { } // ListObjects calls ListObjectsFunc. -func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { if mock.ListObjectsFunc == nil { panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called") } @@ -1320,7 +1319,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct { } // ListObjectsV2 calls ListObjectsV2Func. -func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { +func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { if mock.ListObjectsV2Func == nil { panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called") } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 0a260a7b..0b02a44d 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -228,18 +228,18 @@ func TestS3ApiController_ListActions(t *testing.T) { ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { return &s3.ListMultipartUploadsOutput{}, nil }, - ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { - return &s3.ListBucketsOutput{}, nil + ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { + return &s3.ListObjectsV2Output{}, nil }, - ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { - return &s3.ListBucketsOutput{}, nil + ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { + return &s3.ListObjectsOutput{}, nil }, }} app.Get("/:bucket", s3ApiController.ListActions) //Error case s3ApiControllerError := S3ApiController{be: &BackendMock{ - ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) { + ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) }, }}