mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-04-28 16:26:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

posix: add list objects

Browse Source
This commit is contained in:
Ben McClelland
2023-05-23 11:37:49 -07:00
parent 16df0311e9
commit 01be7a2a6b
6 changed files with 243 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/backend.go
View File

@@ -10,7 +10,7 @@ import (
)
//go:generate moq -out backend_moq_test.go . Backend
//go:generate moq -out ../s3api/controllers/backend_moq_test.go . Backend
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
type Backend interface {
fmt.Stringer
GetIAMConfig() ([]byte, error)
@@ -36,8 +36,8 @@ type Backend interface {
GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
DeleteObject(bucket, object string) error
DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
PutBucketAcl(*s3.PutBucketAclInput) error
@@ -144,10 +144,10 @@ func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes
func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}

12
backend/backend_moq_test.go
View File

@@ -80,10 +80,10 @@ var _ Backend = &BackendMock{}
// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
// panic("mock out the ListObjectParts method")
// },
// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
// panic("mock out the ListObjects method")
// },
// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
// panic("mock out the ListObjectsV2 method")
// },
// PutBucketFunc: func(bucket string) error {
@@ -190,10 +190,10 @@ type BackendMock struct {
ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
// ListObjectsV2Func mocks the ListObjectsV2 method.
ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
// PutBucketFunc mocks the PutBucket method.
PutBucketFunc func(bucket string) error
@@ -1270,7 +1270,7 @@ func (mock *BackendMock) ListObjectPartsCalls() []struct {
}
// ListObjects calls ListObjectsFunc.
func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
if mock.ListObjectsFunc == nil {
panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called")
}
@@ -1318,7 +1318,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct {
}
// ListObjectsV2 calls ListObjectsV2Func.
func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
if mock.ListObjectsV2Func == nil {
panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called")
}

6
backend/common.go
View File

@@ -14,6 +14,12 @@ func (d ByBucketName) Len() int { return len(d) }
func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByBucketName) Less(i, j int) bool { return *d[i].Name < *d[j].Name }
type ByObjectName []types.Object
func (d ByObjectName) Len() int { return len(d) }
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
func GetStringPtr(s string) *string {
return &s
}

212
backend/posix/posix.go
View File

@@ -6,6 +6,7 @@ import (
"encoding/hex"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
@@ -954,9 +955,212 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
}, nil
}
func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
results, err := walk(bucket, prefix, delim, marker, maxkeys)
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return &s3.ListObjectsOutput{
CommonPrefixes: results.commonPrefixes,
Contents: results.objects,
Delimiter: &delim,
IsTruncated: results.truncated,
Marker: &marker,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextMarker: &results.nextMarker,
Prefix: &prefix,
}, nil
}
func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
results, err := walk(bucket, prefix, delim, marker, maxkeys)
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return &s3.ListObjectsV2Output{
CommonPrefixes: results.commonPrefixes,
Contents: results.objects,
Delimiter: &delim,
IsTruncated: results.truncated,
ContinuationToken: &marker,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.nextMarker,
Prefix: &prefix,
}, nil
}
type walkResults struct {
commonPrefixes []types.CommonPrefix
objects []types.Object
truncated bool
nextMarker string
}
func walk(root, prefix, delimiter, marker string, max int) (walkResults, error) {
fileSystem := os.DirFS(root)
cpmap := make(map[string]struct{})
var objects []types.Object
var pastMarker bool
if marker == "" {
pastMarker = true
}
var pastMax bool
var newMarker string
var truncated bool
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if pastMax {
newMarker = path
truncated = true
return fs.SkipAll
}
if d.IsDir() {
// Ignore the root directory
if path == "." {
return nil
}
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
// match this prefix. Make sure to append the / at the end of
// directories since this is implied as a directory path name.
if prefix != "" && !strings.HasPrefix(path+string(os.PathSeparator), prefix) {
return fs.SkipDir
}
// TODO: special case handling if directory is empty
// and was "PUT" explicitly
return nil
}
if !pastMarker {
if path != marker {
return nil
}
pastMarker = true
}
// If object doesnt have prefix, dont include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return nil
}
if delimiter == "" {
// If no delimeter specified, then all files with matching
// prefix are included in results
fi, err := d.Info()
if err != nil {
return fmt.Errorf("get info for %v: %w", path, err)
}
objects = append(objects, types.Object{
ETag: new(string),
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: fi.Size(),
})
if (len(objects) + len(cpmap)) == max {
pastMax = true
}
return nil
}
// Since delimiter is specified, we only want results that
// do not contain the delimiter beyond the prefix. If the
// delimiter exists past the prefix, then the substring
// between the prefix and delimiter is part of common prefixes.
//
// For example:
// prefix = A/
// delimeter = /
// and objects:
// A/file
// A/B/file
// B/C
// would return:
// objects: A/file
// common prefix: A/B/
//
// Note: No obects are included past the common prefix since
// these are all rolled up into the common prefix.
// Note: The delimeter can be anything, so we have to operate on
// the full path without any assumptions on posix directory heirarchy
// here. Usually the delimeter with be "/", but thats not required.
suffix := strings.TrimPrefix(path, prefix)
before, _, found := strings.Cut(suffix, delimiter)
if !found {
fi, err := d.Info()
if err != nil {
return fmt.Errorf("get info for %v: %w", path, err)
}
objects = append(objects, types.Object{
ETag: new(string),
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: fi.Size(),
})
if (len(objects) + len(cpmap)) == max {
pastMax = true
}
return nil
}
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimeter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == max {
pastMax = true
}
return nil
})
if err != nil {
return walkResults{}, err
}
commonPrefixStrings := make([]string, 0, len(cpmap))
for k := range cpmap {
commonPrefixStrings = append(commonPrefixStrings, k)
}
sort.Strings(commonPrefixStrings)
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
for _, cp := range commonPrefixStrings {
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
Prefix: &cp,
})
}
return walkResults{
commonPrefixes: commonPrefixes,
objects: objects,
truncated: truncated,
nextMarker: newMarker,
}, nil
}

27
s3api/controllers/backend_moq_test.go
View File

@@ -4,23 +4,22 @@
package controllers
import (
"io"
"sync"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/scoutgw/backend"
"io"
"sync"
)
// Ensure, that BackendMock does implement Backend.
// Ensure, that BackendMock does implement backend.Backend.
// If this is not the case, regenerate this file with moq.
var _ backend.Backend = &BackendMock{}
// BackendMock is a mock implementation of Backend.
// BackendMock is a mock implementation of backend.Backend.
//
// func TestSomethingThatUsesBackend(t *testing.T) {
//
// // make and configure a mocked Backend
// // make and configure a mocked backend.Backend
// mockedBackend := &BackendMock{
// AbortMultipartUploadFunc: func(abortMultipartUploadInput *s3.AbortMultipartUploadInput) error {
// panic("mock out the AbortMultipartUpload method")
@@ -82,10 +81,10 @@ var _ backend.Backend = &BackendMock{}
// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
// panic("mock out the ListObjectParts method")
// },
// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
// panic("mock out the ListObjects method")
// },
// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
// panic("mock out the ListObjectsV2 method")
// },
// PutBucketFunc: func(bucket string) error {
@@ -100,7 +99,7 @@ var _ backend.Backend = &BackendMock{}
// PutObjectAclFunc: func(putObjectAclInput *s3.PutObjectAclInput) error {
// panic("mock out the PutObjectAcl method")
// },
// PutObjectPartFunc: func(bucket string, object string, uploadID string, part int, r io.Reader) (string, error) {
// PutObjectPartFunc: func(bucket string, object string, uploadID string, part int, length int64, r io.Reader) (string, error) {
// panic("mock out the PutObjectPart method")
// },
// RemoveTagsFunc: func(bucket string, object string) error {
@@ -126,7 +125,7 @@ var _ backend.Backend = &BackendMock{}
// },
// }
//
// // use mockedBackend in code that requires Backend
// // use mockedBackend in code that requires backend.Backend
// // and then make assertions.
//
// }
@@ -192,10 +191,10 @@ type BackendMock struct {
ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
// ListObjectsV2Func mocks the ListObjectsV2 method.
ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
// PutBucketFunc mocks the PutBucket method.
PutBucketFunc func(bucket string) error
@@ -1272,7 +1271,7 @@ func (mock *BackendMock) ListObjectPartsCalls() []struct {
}
// ListObjects calls ListObjectsFunc.
func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
if mock.ListObjectsFunc == nil {
panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called")
}
@@ -1320,7 +1319,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct {
}
// ListObjectsV2 calls ListObjectsV2Func.
func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
if mock.ListObjectsV2Func == nil {
panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called")
}

10
s3api/controllers/base_test.go
View File

@@ -228,18 +228,18 @@ func TestS3ApiController_ListActions(t *testing.T) {
ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
return &s3.ListMultipartUploadsOutput{}, nil
},
ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
return &s3.ListBucketsOutput{}, nil
ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
},
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
return &s3.ListBucketsOutput{}, nil
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
}}
app.Get("/:bucket", s3ApiController.ListActions)
//Error case
s3ApiControllerError := S3ApiController{be: &BackendMock{
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
}}