Compare commits

...

21 Commits

Author SHA1 Message Date
jonaustin09
1649c5cafd fix: Added KeyCount property in ListObjectsV2 action result, added a test case for one 2023-07-25 20:44:57 +04:00
Ben McClelland
7c08ea44a6 Merge pull request #149 from versity/ben/backend_interface
fix: standardize Backend interface args for s3 types
2023-07-24 08:26:56 -07:00
Ben McClelland
e73d661de1 Merge pull request #148 from versity/ben/admin_cleanup
fix: cleanup unused adminRegion
2023-07-24 08:26:45 -07:00
Ben McClelland
2291c22eaa fix: standardize Backend interface args for s3 types 2023-07-22 22:45:24 -07:00
Ben McClelland
51e818b3e3 fix: cleanup unused adminRegion 2023-07-22 18:53:58 -07:00
Ben McClelland
daa4aa1510 Merge pull request #135 from versity/ben/cleanup
fix: signal.go spelling
2023-07-20 14:03:21 -07:00
Ben McClelland
8765a6c67f fix: signal.go spelling 2023-07-20 13:59:51 -07:00
Ben McClelland
c5a7b5aae1 Merge pull request #134 from versity/event-notif-nats
feat: cleanup nats for kafka similarity
2023-07-20 13:58:10 -07:00
Ben McClelland
2ae39c3ee8 feat: cleanup nats for kafka similarity 2023-07-20 13:54:55 -07:00
Ben McClelland
d0b3139640 Merge pull request #133 from versity/event-notif-nats
Bucket event notifications(nats)
2023-07-20 13:50:53 -07:00
jonaustin09
7bceaaca39 feat: Set up bucket event notifications with nats 2023-07-20 13:36:16 -07:00
Ben McClelland
6f0f527e5f Merge pull request #132 from versity/event-notifications
Bucket event notifications(kafka)
2023-07-20 13:27:40 -07:00
jonaustin09
fe547a19e9 feat: bucket event notifications
Set up Bucket event notifications interface to send aws compatible format event messages to a configured event service.
First integrated service is kafka message broker as an option for bucket event notifications.
2023-07-20 11:37:14 -07:00
Ben McClelland
df7f01f7e2 Merge pull request #129 from versity/audit-logging-setup
feat: Set up audit logging basic structure, set up webhook logger, bu…
2023-07-14 12:50:32 -07:00
Ben McClelland
5aeb96f138 Merge pull request #131 from versity/fix-posix-delete-object
Fix Posix Delete Objects
2023-07-14 12:46:57 -07:00
jonaustin09
ef1de682a4 fix: Error handling for posix DeleteObject function to return an error when the object doesn't exist 2023-07-14 23:41:52 +04:00
jonaustin09
87d61a1eb3 feat: Setup audit loggin with webhook url and root level access.log file. CLI enables either webhook or server access logs by providing the flags 2023-07-14 23:40:05 +04:00
Ben McClelland
18899f8029 Merge pull request #128 from versity/ben/update
update package deps
2023-07-06 20:59:52 -07:00
Ben McClelland
ca28792458 update package deps 2023-07-06 21:56:59 -06:00
Ben McClelland
8c469cbd69 Merge pull request #127 from versity/ben/issue_templates
feat: add issue templates
2023-07-06 20:43:40 -07:00
Ben McClelland
ff4bf23b6b feat: add issue templates 2023-07-06 21:40:57 -06:00
28 changed files with 1969 additions and 831 deletions

27
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,27 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior.
**Expected behavior**
A clear and concise description of what you expected to happen.
**Server Version**
output of
```
./versitygw -version
uname -a
```
**Additional context**
Describe s3 client and version if applicable.

View File

@@ -0,0 +1,14 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: ''
---
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Additional context**
Add any other context or screenshots about the feature request here.

View File

@@ -19,7 +19,6 @@ import (
"io"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
@@ -29,33 +28,33 @@ type Backend interface {
fmt.Stringer
Shutdown()
ListBuckets() (s3response.ListAllMyBucketsResult, error)
HeadBucket(bucket string) (*s3.HeadBucketOutput, error)
GetBucketAcl(bucket string) ([]byte, error)
PutBucket(bucket, owner string) error
ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error)
CreateBucket(*s3.CreateBucketInput) error
PutBucketAcl(bucket string, data []byte) error
DeleteBucket(bucket string) error
DeleteBucket(*s3.DeleteBucketInput) error
CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error)
UploadPart(*s3.UploadPartInput) (etag string, err error)
UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
PutObject(*s3.PutObjectInput) (string, error)
HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error)
ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
DeleteObject(bucket, object string) error
DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(*s3.DeleteObjectInput) error
DeleteObjects(*s3.DeleteObjectsInput) error
PutObjectAcl(*s3.PutObjectAclInput) error
RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error
RestoreObject(*s3.RestoreObjectInput) error
GetTags(bucket, object string) (map[string]string, error)
SetTags(bucket, object string, tags map[string]string) error
@@ -73,7 +72,7 @@ func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets() (s3response.ListAllMyBucketsResult, error) {
func (BackendUnsupported) ListBuckets(string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error {
@@ -82,72 +81,72 @@ func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error {
func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
func (BackendUnsupported) RestoreObject(*s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) {
func (BackendUnsupported) GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
func (BackendUnsupported) HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucket(bucket, owner string) error {
func (BackendUnsupported) CreateBucket(*s3.CreateBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(bucket string) error {
func (BackendUnsupported) DeleteBucket(*s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
func (BackendUnsupported) CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
func (BackendUnsupported) CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error {
func (BackendUnsupported) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
func (BackendUnsupported) ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
func (BackendUnsupported) ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) {
func (BackendUnsupported) UploadPart(*s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(bucket, object string) error {
func (BackendUnsupported) DeleteObject(*s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
func (BackendUnsupported) DeleteObjects(*s3.DeleteObjectsInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
func (BackendUnsupported) GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
func (BackendUnsupported) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
func (BackendUnsupported) GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
func (BackendUnsupported) GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
func (BackendUnsupported) CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
func (BackendUnsupported) ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
func (BackendUnsupported) ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}

View File

@@ -99,7 +99,7 @@ func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
return startOffset, endOffset - startOffset + 1, nil
}
func GetMultipartMD5(parts []types.Part) string {
func GetMultipartMD5(parts []types.CompletedPart) string {
var partsEtagBytes []byte
for _, part := range parts {
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)

View File

@@ -96,7 +96,7 @@ func (p *Posix) String() string {
return "Posix Gateway"
}
func (p *Posix) ListBuckets() (s3response.ListAllMyBucketsResult, error) {
func (p *Posix) ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
entries, err := os.ReadDir(".")
if err != nil {
return s3response.ListAllMyBucketsResult{},
@@ -131,8 +131,8 @@ func (p *Posix) ListBuckets() (s3response.ListAllMyBucketsResult, error) {
}, nil
}
func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
_, err := os.Lstat(bucket)
func (p *Posix) HeadBucket(input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
_, err := os.Lstat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -143,7 +143,10 @@ func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
return &s3.HeadBucketOutput{}, nil
}
func (p *Posix) PutBucket(bucket string, owner string) error {
func (p *Posix) CreateBucket(input *s3.CreateBucketInput) error {
bucket := *input.Bucket
owner := string(input.ObjectOwnership)
err := os.Mkdir(bucket, 0777)
if err != nil && os.IsExist(err) {
return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
@@ -165,8 +168,8 @@ func (p *Posix) PutBucket(bucket string, owner string) error {
return nil
}
func (p *Posix) DeleteBucket(bucket string) error {
names, err := os.ReadDir(bucket)
func (p *Posix) DeleteBucket(input *s3.DeleteBucketInput) error {
names, err := os.ReadDir(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -177,13 +180,13 @@ func (p *Posix) DeleteBucket(bucket string) error {
if len(names) == 1 && names[0].Name() == metaTmpDir {
// if .sgwtmp is only item in directory
// then clean this up before trying to remove the bucket
err = os.RemoveAll(filepath.Join(bucket, metaTmpDir))
err = os.RemoveAll(filepath.Join(*input.Bucket, metaTmpDir))
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove temp dir: %w", err)
}
}
err = os.Remove(bucket)
err = os.Remove(*input.Bucket)
if err != nil && err.(*os.PathError).Err == syscall.ENOTEMPTY {
return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
}
@@ -245,7 +248,12 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
}, nil
}
func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
func (p *Posix) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
parts := input.MultipartUpload.Parts
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -611,8 +619,24 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo
}, nil
}
func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
func (p *Posix) ListParts(input *s3.ListPartsInput) (s3response.ListPartsResponse, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
stringMarker := *input.PartNumberMarker
maxParts := int(input.MaxParts)
var lpr s3response.ListPartsResponse
var partNumberMarker int
if stringMarker != "" {
var err error
partNumberMarker, err = strconv.Atoi(stringMarker)
if err != nil {
return lpr, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)
}
}
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return lpr, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -693,7 +717,14 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
}, nil
}
func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) {
func (p *Posix) UploadPart(input *s3.UploadPartInput) (string, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
part := input.PartNumber
length := input.ContentLength
r := input.Body
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -912,7 +943,10 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
return etag, nil
}
func (p *Posix) DeleteObject(bucket, object string) error {
func (p *Posix) DeleteObject(input *s3.DeleteObjectInput) error {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -921,7 +955,7 @@ func (p *Posix) DeleteObject(bucket, object string) error {
return fmt.Errorf("stat bucket: %w", err)
}
os.Remove(filepath.Join(bucket, object))
err = os.Remove(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -966,10 +1000,13 @@ func (p *Posix) removeParents(bucket, object string) error {
return nil
}
func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
func (p *Posix) DeleteObjects(input *s3.DeleteObjectsInput) error {
// delete object already checks bucket
for _, obj := range objects.Delete.Objects {
err := p.DeleteObject(bucket, *obj.Key)
for _, obj := range input.Delete.Objects {
err := p.DeleteObject(&s3.DeleteObjectInput{
Bucket: input.Bucket,
Key: obj.Key,
})
if err != nil {
return err
}
@@ -978,7 +1015,11 @@ func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) err
return nil
}
func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
func (p *Posix) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1051,7 +1092,10 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer)
}, nil
}
func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
func (p *Posix) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1088,7 +1132,14 @@ func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
}, nil
}
func (p *Posix) CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
func (p *Posix) CopyObject(input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
if !ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
dstBucket := *input.Bucket
dstObject := *input.Key
_, err := os.Stat(srcBucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1138,7 +1189,13 @@ func (p *Posix) CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*
}, nil
}
func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
func (p *Posix) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1160,7 +1217,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (
Delimiter: &delim,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: int32(maxkeys),
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
@@ -1228,7 +1285,13 @@ func fileToObj(bucket string) backend.GetObjFunc {
}
}
func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
func (p *Posix) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1238,7 +1301,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int)
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
@@ -1254,6 +1317,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int)
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
KeyCount: int32(len(results.Objects)),
}, nil
}
@@ -1273,8 +1337,8 @@ func (p *Posix) PutBucketAcl(bucket string, data []byte) error {
return nil
}
func (p *Posix) GetBucketAcl(bucket string) ([]byte, error) {
_, err := os.Stat(bucket)
func (p *Posix) GetBucketAcl(input *s3.GetBucketAclInput) ([]byte, error) {
_, err := os.Stat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -1282,7 +1346,7 @@ func (p *Posix) GetBucketAcl(bucket string) ([]byte, error) {
return nil, fmt.Errorf("stat bucket: %w", err)
}
b, err := xattr.Get(bucket, aclkey)
b, err := xattr.Get(*input.Bucket, aclkey)
if isNoAttr(err) {
return []byte{}, nil
}

View File

@@ -114,7 +114,12 @@ func (*ScoutFS) String() string {
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
// ioctl to not have to read and copy the part data to the final object. This
// saves a read and write cycle for all mutlipart uploads.
func (s *ScoutFS) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
parts := input.MultipartUpload.Parts
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -347,7 +352,10 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
return nil
}
func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -417,7 +425,11 @@ func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error
}, nil
}
func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
func (s *ScoutFS) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -527,7 +539,13 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
return tags, nil
}
func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -549,14 +567,20 @@ func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int)
Delimiter: &delim,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: int32(maxkeys),
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
}, nil
}
func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
func (s *ScoutFS) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -566,7 +590,7 @@ func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys in
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
s.fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
@@ -663,7 +687,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
// RestoreObject will set stage request on file if offline and do nothing if
// file is online
func (s *ScoutFS) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
func (s *ScoutFS) RestoreObject(input *s3.RestoreObjectInput) error {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)

View File

@@ -38,7 +38,7 @@ var ErrSkipObj = errors.New("skip this object")
// Walk walks the supplied fs.FS and returns results compatible with list
// objects responses
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
cpmap := make(map[string]struct{})
var objects []types.Object
@@ -129,7 +129,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
}
objects = append(objects, obj)
if max > 0 && (len(objects)+len(cpmap)) == max {
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
pastMax = true
}
@@ -168,7 +168,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, obj)
if (len(objects) + len(cpmap)) == max {
if (len(objects) + len(cpmap)) == int(max) {
pastMax = true
}
return nil
@@ -178,7 +178,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
// These are abstractly a "directory", so need to include the
// delimiter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == max {
if (len(objects) + len(cpmap)) == int(max) {
pastMax = true
}

View File

@@ -30,7 +30,6 @@ import (
var (
adminAccess string
adminSecret string
adminRegion string
)
func adminCommand() *cli.Command {
@@ -97,13 +96,6 @@ func adminCommand() *cli.Command {
Aliases: []string{"s"},
Destination: &adminSecret,
},
&cli.StringFlag{
Name: "region",
Usage: "s3 region string",
Value: "us-east-1",
Destination: &adminRegion,
Aliases: []string{"r"},
},
},
}
}
@@ -129,7 +121,7 @@ func createUser(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -146,7 +138,7 @@ func createUser(ctx *cli.Context) error {
return err
}
fmt.Printf("%s", body)
fmt.Printf("%s\n", body)
return nil
}
@@ -169,7 +161,7 @@ func deleteUser(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -186,7 +178,7 @@ func deleteUser(ctx *cli.Context) error {
return err
}
fmt.Printf("%s", body)
fmt.Printf("%s\n", body)
return nil
}

View File

@@ -27,15 +27,21 @@ import (
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
var (
port string
rootUserAccess string
rootUserSecret string
region string
certFile, keyFile string
debug bool
port string
rootUserAccess string
rootUserSecret string
region string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
logWebhookURL string
accessLog bool
debug bool
)
var (
@@ -141,6 +147,46 @@ func initFlags() []cli.Flag {
Usage: "enable debug output",
Destination: &debug,
},
&cli.BoolFlag{
Name: "access-log",
Usage: "enable server access logging in the root directory",
Destination: &accessLog,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
Destination: &logWebhookURL,
},
&cli.StringFlag{
Name: "event-kafka-url",
Usage: "kafka server url to send the bucket notifications.",
Destination: &kafkaURL,
Aliases: []string{"eku"},
},
&cli.StringFlag{
Name: "event-kafka-topic",
Usage: "kafka server pub-sub topic to send the bucket notifications to",
Destination: &kafkaTopic,
Aliases: []string{"ekt"},
},
&cli.StringFlag{
Name: "event-kafka-key",
Usage: "kafka server put-sub topic key to send the bucket notifications to",
Destination: &kafkaKey,
Aliases: []string{"ekk"},
},
&cli.StringFlag{
Name: "event-nats-url",
Usage: "nats server url to send the bucket notifications",
Destination: &natsURL,
Aliases: []string{"enu"},
},
&cli.StringFlag{
Name: "event-nats-topic",
Usage: "nats server pub-sub topic to send the bucket notifications to",
Destination: &natsTopic,
Aliases: []string{"ent"},
},
}
}
@@ -182,10 +228,29 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
return fmt.Errorf("setup internal iam service: %w", err)
}
logger, err := s3log.InitLogger(&s3log.LogConfig{
IsFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
return fmt.Errorf("setup logger: %w", err)
}
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
KafkaURL: kafkaURL,
KafkaTopic: kafkaTopic,
KafkaTopicKey: kafkaKey,
NatsURL: natsURL,
NatsTopic: natsTopic,
})
if err != nil {
return fmt.Errorf("unable to connect to the message broker: %w", err)
}
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
Access: rootUserAccess,
Secret: rootUserSecret,
}, port, region, iam, opts...)
}, port, region, iam, logger, evSender, opts...)
if err != nil {
return fmt.Errorf("init gateway: %v", err)
}

24
go.mod
View File

@@ -4,15 +4,17 @@ go 1.20
require (
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0
github.com/aws/smithy-go v1.13.5
github.com/gofiber/fiber/v2 v2.46.0
github.com/gofiber/fiber/v2 v2.47.0
github.com/google/uuid v1.3.0
github.com/nats-io/nats.go v1.28.0
github.com/pkg/xattr v0.4.9
github.com/urfave/cli/v2 v2.25.6
github.com/valyala/fasthttp v1.47.0
github.com/segmentio/kafka-go v0.4.42
github.com/urfave/cli/v2 v2.25.7
github.com/valyala/fasthttp v1.48.0
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
golang.org/x/sys v0.9.0
golang.org/x/sys v0.10.0
)
require (
@@ -21,7 +23,15 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/nats-io/nats-server/v2 v2.9.20 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/crypto v0.11.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)
require (
@@ -29,7 +39,7 @@ require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.27
github.com/aws/aws-sdk-go-v2/credentials v1.13.26
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
@@ -38,7 +48,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect

85
go.sum
View File

@@ -10,8 +10,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RK
github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 h1:4bh28MeeXoBFTjb0JjQ5sVatzlf5xA1DziV8mZed9v4=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70/go.mod h1:9yI5NXzqy2yOiMytv6QLZHvlyHLwYxO9iIq+bZIbrFg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
@@ -28,10 +28,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a92
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 h1:rYYwwsGqbwvGgQHjBkqgDt8MynXk+I8xgS0IEj5gOT0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o=
github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=
@@ -43,17 +41,25 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.47.0 h1:EN5lHVCc+Pyqh5OEsk8fzRiifgwpbrP0rulQ4iNf3fs=
github.com/gofiber/fiber/v2 v2.47.0/go.mod h1:mbFMVN1lQuzziTkkakgtKKdjfsXSw9BKR5lmcNksUoU=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -61,11 +67,25 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
github.com/nats-io/nats-server/v2 v2.9.20/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
@@ -77,20 +97,34 @@ github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU=
github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -99,6 +133,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -108,6 +144,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -122,16 +160,22 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
@@ -139,6 +183,15 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -426,6 +426,10 @@ func TestListObject(s *S3Conf) {
failF("object %v not found", obj2)
return
}
if out.KeyCount != 2 {
failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount)
return
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{

File diff suppressed because it is too large Load Diff

View File

@@ -17,13 +17,13 @@ package controllers
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -33,25 +33,33 @@ import (
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
"github.com/versity/versitygw/s3response"
)
type S3ApiController struct {
be backend.Backend
iam auth.IAMService
be backend.Backend
iam auth.IAMService
logger s3log.AuditLogger
evSender s3event.S3EventSender
}
func New(be backend.Backend, iam auth.IAMService) S3ApiController {
return S3ApiController{be: be, iam: iam}
const (
iso8601Format = "20060102T150405Z"
)
func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) S3ApiController {
return S3ApiController{be: be, iam: iam, logger: logger, evSender: evs}
}
func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
if err := auth.IsAdmin(access, isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
}
res, err := c.be.ListBuckets()
return SendXMLResponse(ctx, res, err)
res, err := c.be.ListBuckets(access, isRoot)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
}
func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
@@ -60,7 +68,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
keyEnd := ctx.Params("*1")
uploadId := ctx.Query("uploadId")
maxParts := ctx.QueryInt("max-parts", 0)
partNumberMarker := ctx.QueryInt("part-number-marker", 0)
partNumberMarker := ctx.Query("part-number-marker")
acceptRange := ctx.Get("Range")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
@@ -68,24 +76,24 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
}
tags, err := c.be.GetTags(bucket, key)
if err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
}
resp := s3response.Tagging{TagSet: s3response.TagSet{Tags: []s3response.Tag{}}}
@@ -93,52 +101,76 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
resp.TagSet.Tags = append(resp.TagSet.Tags, s3response.Tag{Key: key, Value: val})
}
return SendXMLResponse(ctx, resp, nil)
return SendXMLResponse(ctx, resp, nil, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
}
if uploadId != "" {
if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker))
if partNumberMarker != "" {
n, err := strconv.Atoi(partNumberMarker)
if err != nil || n < 0 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts)
return SendXMLResponse(ctx, res, err)
res, err := c.be.ListParts(&s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
PartNumberMarker: &partNumberMarker,
MaxParts: int32(maxParts),
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.GetObjectAcl(bucket, key)
return SendXMLResponse(ctx, res, err)
res, err := c.be.GetObjectAcl(&s3.GetObjectAclInput{
Bucket: &bucket,
Key: &key,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner})
}
if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ","))
return SendXMLResponse(ctx, res, err)
var oattrs []types.ObjectAttributes
for _, a := range strings.Split(attrs, ",") {
oattrs = append(oattrs, types.ObjectAttributes(a))
}
res, err := c.be.GetObjectAttributes(&s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &key,
ObjectAttributes: oattrs,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
ctx.Locals("logResBody", false)
res, err := c.be.GetObject(bucket, key, acceptRange, ctx.Response().BodyWriter())
res, err := c.be.GetObject(&s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
}, ctx.Response().BodyWriter())
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
if res == nil {
return SendResponse(ctx, fmt.Errorf("get object nil response"))
return SendResponse(ctx, fmt.Errorf("get object nil response"), &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -172,7 +204,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
},
})
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
func getstring(s *string) string {
@@ -191,47 +223,59 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
}
res, err := auth.ParseACLOutput(data)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
}
if ctx.Request().URI().QueryArgs().Has("uploads") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
}
if ctx.QueryInt("list-type") == 2 {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys)
return SendXMLResponse(ctx, res, err)
res, err := c.be.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: &bucket,
Prefix: &prefix,
ContinuationToken: &marker,
Delimiter: &delimiter,
MaxKeys: int32(maxkeys),
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
return SendXMLResponse(ctx, res, err)
res, err := c.be.ListObjects(&s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
Marker: &marker,
Delimiter: &delimiter,
MaxKeys: int32(maxkeys),
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
@@ -252,15 +296,29 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
if ctx.Request().URI().QueryArgs().Has("acl") {
var input *s3.PutBucketAclInput
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
if len(ctx.Body()) > 0 {
if grants+acl != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
var accessControlPolicy auth.AccessControlPolicy
err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
input = &s3.PutBucketAclInput{
@@ -271,10 +329,10 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
}
if acl != "" {
if acl != "private" && acl != "public-read" && acl != "public-read-write" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
if len(ctx.Body()) > 0 || grants != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
input = &s3.PutBucketAclInput{
@@ -296,31 +354,20 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
}
}
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
return SendResponse(ctx, err)
}
updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
err = c.be.PutBucketAcl(bucket, updAcl)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
err := c.be.PutBucket(bucket, access)
return SendResponse(ctx, err)
err := c.be.CreateBucket(&s3.CreateBucketInput{
Bucket: &bucket,
ObjectOwnership: types.ObjectOwnership(access),
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)})
}
func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
@@ -361,30 +408,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
keyStart = keyStart + "/"
}
var contentLength int64
if contentLengthStr != "" {
var err error
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
}
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
var objTagging s3response.Tagging
err := xml.Unmarshal(ctx.Body(), &objTagging)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
}
tags := make(map[string]string, len(objTagging.TagSet.Tags))
@@ -394,18 +432,23 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
}
err = c.be.SetTags(bucket, keyStart, tags)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "PutObjectTagging",
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectTaggingPut,
})
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" {
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
}
resp, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{
@@ -417,25 +460,36 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ExpectedBucketOwner: &bucketOwner,
CopySourceRange: &copySrcRange,
})
return SendXMLResponse(ctx, resp, err)
return SendXMLResponse(ctx, resp, err, &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") {
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
ctx.Locals("logReqBody", false)
etag, err := c.be.PutObjectPart(bucket, keyStart, uploadId,
partNumber, contentLength, body)
etag, err := c.be.UploadPart(&s3.UploadPartInput{
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: int32(partNumber),
ContentLength: contentLength,
Body: body,
})
ctx.Response().Header.Set("Etag", etag)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
@@ -443,13 +497,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
if len(ctx.Body()) > 0 {
if grants+acl != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner})
}
var accessControlPolicy auth.AccessControlPolicy
err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner})
}
input = &s3.PutObjectAclInput{
@@ -461,10 +515,10 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if acl != "" {
if acl != "private" && acl != "public-read" && acl != "public-read-write" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner})
}
if len(ctx.Body()) > 0 || grants != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner})
}
input = &s3.PutObjectAclInput{
@@ -489,27 +543,71 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
err = c.be.PutObjectAcl(input)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "PutObjectAcl",
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectAclPut,
})
}
if copySource != "" {
_, _, _, _ = copySrcIfMatch, copySrcIfNoneMatch,
copySrcModifSince, copySrcUnmodifSince
copySourceSplit := strings.Split(copySource, "/")
srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:]
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.CopyObject(srcBucket, strings.Join(srcObject, "/"), bucket, keyStart)
return SendXMLResponse(ctx, res, err)
var mtime time.Time
if copySrcModifSince != "" {
mtime, err = time.Parse(iso8601Format, copySrcModifSince)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
}
var umtime time.Time
if copySrcModifSince != "" {
mtime, err = time.Parse(iso8601Format, copySrcUnmodifSince)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
}
res, err := c.be.CopyObject(&s3.CopyObjectInput{
Bucket: &bucket,
Key: &keyStart,
CopySource: &copySource,
CopySourceIfMatch: &copySrcIfMatch,
CopySourceIfNoneMatch: &copySrcIfNoneMatch,
CopySourceIfModifiedSince: &mtime,
CopySourceIfUnmodifiedSince: &umtime,
})
if err == nil {
return SendXMLResponse(ctx, res, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "CopyObject",
BucketOwner: parsedAcl.Owner,
ObjectETag: res.CopyObjectResult.ETag,
VersionId: res.VersionId,
EventName: s3event.EventObjectCopy,
})
} else {
return SendXMLResponse(ctx, res, err, &MetaOpts{
Logger: c.logger,
Action: "CopyObject",
BucketOwner: parsedAcl.Owner,
})
}
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
}
ctx.Locals("logReqBody", false)
@@ -521,54 +619,67 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Body: bytes.NewReader(ctx.Request().Body()),
})
ctx.Response().Header.Set("ETag", etag)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "PutObject",
BucketOwner: parsedAcl.Owner,
ObjectETag: &etag,
ObjectSize: contentLength,
EventName: s3event.EventObjectPut,
})
}
func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteBucket(bucket)
return SendResponse(ctx, err)
err = c.be.DeleteBucket(&s3.DeleteBucketInput{
Bucket: &bucket,
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
var dObj types.Delete
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
}
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj})
return SendResponse(ctx, err)
err = c.be.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: &bucket,
Delete: &dObj,
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
@@ -583,30 +694,36 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner})
}
err = c.be.RemoveTags(bucket, key)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "RemoveObjectTagging",
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectTaggingDelete,
})
}
if uploadId != "" {
expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
}
err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
@@ -616,37 +733,48 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
ExpectedBucketOwner: &expectedBucketOwner,
RequestPayer: types.RequestPayer(requestPayer),
})
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObject", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteObject(bucket, key)
return SendResponse(ctx, err)
err = c.be.DeleteObject(&s3.DeleteObjectInput{
Bucket: &bucket,
Key: &key,
})
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "DeleteObject",
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectDelete,
})
}
func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner})
}
_, err = c.be.HeadBucket(bucket)
_, err = c.be.HeadBucket(&s3.HeadBucketInput{
Bucket: &bucket,
})
// TODO: set bucket response headers
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner})
}
const (
@@ -661,26 +789,31 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{
Bucket: &bucket,
})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.HeadObject(bucket, key)
res, err := c.be.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
}
if res == nil {
return SendResponse(ctx, fmt.Errorf("head object nil response"))
return SendResponse(ctx, fmt.Errorf("head object nil response"), &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
}
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -719,7 +852,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
},
})
return SendResponse(ctx, nil)
return SendResponse(ctx, nil, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
@@ -734,57 +867,107 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(bucket)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
var restoreRequest s3.RestoreObjectInput
if ctx.Request().URI().QueryArgs().Has("restore") {
xmlErr := xml.Unmarshal(ctx.Body(), &restoreRequest)
if xmlErr != nil {
return errors.New("wrong api call")
err := xml.Unmarshal(ctx.Body(), &restoreRequest)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
}
err := c.be.RestoreObject(bucket, key, &restoreRequest)
return SendResponse(ctx, err)
restoreRequest.Bucket = &bucket
restoreRequest.Key = &key
err = c.be.RestoreObject(&restoreRequest)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "RestoreObject",
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectRestoreCompleted,
})
}
if uploadId != "" {
data := struct {
Parts []types.Part `xml:"Part"`
Parts []types.CompletedPart `xml:"Part"`
}{}
if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
return errors.New("wrong api call")
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts)
return SendXMLResponse(ctx, res, err)
res, err := c.be.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: data.Parts,
},
})
if err == nil {
return SendXMLResponse(ctx, res, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
Action: "CompleteMultipartUpload",
BucketOwner: parsedAcl.Owner,
ObjectETag: res.ETag,
EventName: s3event.EventCompleteMultipartUpload,
VersionId: res.VersionId,
})
} else {
return SendXMLResponse(ctx, res, err, &MetaOpts{
Logger: c.logger,
Action: "CompleteMultipartUpload",
BucketOwner: parsedAcl.Owner,
})
}
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
}
func SendResponse(ctx *fiber.Ctx, err error) error {
type MetaOpts struct {
Logger s3log.AuditLogger
EvSender s3event.S3EventSender
Action string
BucketOwner string
ObjectSize int64
EventName s3event.EventType
ObjectETag *string
VersionId *string
}
func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
if l.Logger != nil {
l.Logger.Log(ctx, err, nil, s3log.LogMeta{
Action: l.Action,
BucketOwner: l.BucketOwner,
ObjectSize: l.ObjectSize,
})
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
@@ -797,6 +980,15 @@ func SendResponse(ctx *fiber.Ctx, err error) error {
return ctx.Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
}
if l.EvSender != nil {
l.EvSender.SendEvent(ctx, s3event.EventMeta{
ObjectSize: l.ObjectSize,
ObjectETag: l.ObjectETag,
EventName: l.EventName,
BucketOwner: l.BucketOwner,
VersionId: l.VersionId,
})
}
utils.LogCtxDetails(ctx, []byte{})
@@ -806,9 +998,15 @@ func SendResponse(ctx *fiber.Ctx, err error) error {
return nil
}
func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error {
func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error {
if err != nil {
fmt.Println(err)
if l.Logger != nil {
l.Logger.Log(ctx, err, nil, s3log.LogMeta{
Action: l.Action,
BucketOwner: l.BucketOwner,
ObjectSize: l.ObjectSize,
})
}
serr, ok := err.(s3err.APIError)
if ok {
ctx.Status(serr.HTTPStatusCode)
@@ -835,6 +1033,23 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error {
}
utils.LogCtxDetails(ctx, b)
if l.Logger != nil {
l.Logger.Log(ctx, nil, b, s3log.LogMeta{
Action: l.Action,
BucketOwner: l.BucketOwner,
ObjectSize: l.ObjectSize,
})
}
if l.EvSender != nil {
l.EvSender.SendEvent(ctx, s3event.EventMeta{
BucketOwner: l.BucketOwner,
ObjectSize: l.ObjectSize,
ObjectETag: l.ObjectETag,
VersionId: l.VersionId,
EventName: l.EventName,
})
}
return ctx.Send(b)
}

View File

@@ -74,7 +74,7 @@ func TestNew(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := New(tt.args.be, tt.args.iam); !reflect.DeepEqual(got, tt.want) {
if got := New(tt.args.be, tt.args.iam, nil, nil); !reflect.DeepEqual(got, tt.want) {
t.Errorf("New() = %v, want %v", got, tt.want)
}
})
@@ -89,10 +89,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, nil
},
},
@@ -110,10 +110,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
},
},
@@ -187,32 +187,33 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
}
}
func getPtr(val string) *string {
return &val
}
func TestS3ApiController_GetActions(t *testing.T) {
type args struct {
req *http.Request
}
getPtr := func(val string) *string {
return &val
}
now := time.Now()
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
ListPartsFunc: func(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, nil
},
GetObjectAclFunc: func(bucket, object string) (*s3.GetObjectAclOutput, error) {
GetObjectAclFunc: func(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
GetObjectAttributesFunc: func(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return &s3.GetObjectAttributesOutput{}, nil
},
GetObjectFunc: func(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
GetObjectFunc: func(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Metadata: map[string]string{"hello": "world"},
ContentType: getPtr("application/xml"),
@@ -352,16 +353,16 @@ func TestS3ApiController_ListActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
return s3response.ListMultipartUploadsResponse{}, nil
},
ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
ListObjectsV2Func: func(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
},
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
},
@@ -379,10 +380,10 @@ func TestS3ApiController_ListActions(t *testing.T) {
//Error case
s3ApiControllerError := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
},
@@ -497,13 +498,13 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
PutBucketAclFunc: func(string, []byte) error {
return nil
},
PutBucketFunc: func(bucket, owner string) error {
CreateBucketFunc: func(*s3.CreateBucketInput) error {
return nil
},
},
@@ -649,19 +650,21 @@ func TestS3ApiController_PutActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
PutObjectAclFunc: func(*s3.PutObjectAclInput) error {
return nil
},
CopyObjectFunc: func(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
return &s3.CopyObjectOutput{}, nil
CopyObjectFunc: func(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return &s3.CopyObjectOutput{
CopyObjectResult: &types.CopyObjectResult{},
}, nil
},
PutObjectFunc: func(*s3.PutObjectInput) (string, error) {
return "Hey", nil
return "ETag", nil
},
PutObjectPartFunc: func(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) {
UploadPartFunc: func(*s3.UploadPartInput) (string, error) {
return "hello", nil
},
SetTagsFunc: func(bucket, object string, tags map[string]string) error {
@@ -861,10 +864,10 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteBucketFunc: func(bucket string) error {
DeleteBucketFunc: func(*s3.DeleteBucketInput) error {
return nil
},
},
@@ -917,10 +920,10 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error {
DeleteObjectsFunc: func(*s3.DeleteObjectsInput) error {
return nil
},
},
@@ -987,10 +990,10 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(bucket, object string) error {
DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
return nil
},
AbortMultipartUploadFunc: func(*s3.AbortMultipartUploadInput) error {
@@ -1014,10 +1017,10 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(bucket, object string) error {
DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
return s3err.GetAPIError(7)
},
}}
@@ -1095,10 +1098,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return &s3.HeadBucketOutput{}, nil
},
},
@@ -1117,10 +1120,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(3)
},
},
@@ -1189,10 +1192,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) {
HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return &s3.HeadObjectOutput{
ContentEncoding: &contentEncoding,
ContentLength: 64,
@@ -1217,10 +1220,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) {
HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(42)
},
},
@@ -1280,13 +1283,13 @@ func TestS3ApiController_CreateActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(bucket string) ([]byte, error) {
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
RestoreObjectFunc: func(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
RestoreObjectFunc: func(restoreRequest *s3.RestoreObjectInput) error {
return nil
},
CompleteMultipartUploadFunc: func(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
CompleteMultipartUploadFunc: func(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return &s3.CompleteMultipartUploadOutput{}, nil
},
CreateMultipartUploadFunc: func(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
@@ -1435,7 +1438,7 @@ func Test_XMLresponse(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err); (err != nil) != tt.wantErr {
if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr {
t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
}
@@ -1515,7 +1518,7 @@ func Test_response(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := SendResponse(tt.args.ctx, tt.args.err); (err != nil) != tt.wantErr {
if err := SendResponse(tt.args.ctx, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr {
t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
}

View File

@@ -29,6 +29,7 @@ import (
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
const (
@@ -40,13 +41,15 @@ type RootUserConfig struct {
Secret string
}
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler {
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
}
// Check the signature version
@@ -56,48 +59,52 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
}
if len(authParts) != 3 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
startParts := strings.Split(authParts[0], " ")
if startParts[0] != "AWS4-HMAC-SHA256" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
}
credKv := strings.Split(startParts[1], "=")
if len(credKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
creds := strings.Split(credKv[1], "/")
if len(creds) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("access", creds[0])
ctx.Locals("isRoot", creds[0] == root.Access)
signHdrKv := strings.Split(authParts[1], "=")
if len(signHdrKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
signedHdrs := strings.Split(signHdrKv[1], ";")
account, err := acct.getAccount(creds[0])
if err == auth.ErrNoSuchUser {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
}
if err != nil {
return controllers.SendResponse(ctx, err)
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("role", account.Role)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
}
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
@@ -110,14 +117,14 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
// Compare the calculated hash with the hash provided
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
}
}
// Create a new http request instance from fasthttp request
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
signer := v4.NewSigner()
@@ -132,24 +139,20 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
}
})
if signErr != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("role", account.Role)
ctx.Locals("access", creds[0])
ctx.Locals("isRoot", creds[0] == root.Access)
return ctx.Next()
}
}

View File

@@ -21,9 +21,10 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyMD5Body() fiber.Handler {
func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
incomingSum := ctx.Get("Content-Md5")
if incomingSum == "" {
@@ -34,10 +35,9 @@ func VerifyMD5Body() fiber.Handler {
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
}
}

View File

@@ -19,12 +19,14 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiRouter struct{}
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
s3ApiController := controllers.New(be, iam)
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
s3ApiController := controllers.New(be, iam, logger, evs)
adminController := controllers.AdminController{IAMService: iam}
app.Patch("/create-user", adminController.CreateUser)

View File

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam)
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
})
}
}

View File

@@ -22,6 +22,8 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiServer struct {
@@ -33,7 +35,7 @@ type S3ApiServer struct {
debug bool
}
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) {
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
server := &S3ApiServer{
app: app,
backend: be,
@@ -50,10 +52,10 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, region, server.debug))
app.Use(middlewares.VerifyMD5Body())
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyMD5Body(l))
server.router.Init(app, be, iam)
server.router.Init(app, be, iam, l, evs)
return server, nil
}

View File

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
tt.args.port, "us-east-1", &auth.IAMServiceInternal{})
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return

130
s3event/event.go Normal file
View File

@@ -0,0 +1,130 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"fmt"
"github.com/gofiber/fiber/v2"
)
type S3EventSender interface {
SendEvent(ctx *fiber.Ctx, meta EventMeta)
}
type EventMeta struct {
BucketOwner string
EventName EventType
ObjectSize int64
ObjectETag *string
VersionId *string
}
type EventFields struct {
Records []EventSchema
}
type EventType string
const (
EventObjectPut EventType = "s3:ObjectCreated:Put"
EventObjectCopy EventType = "s3:ObjectCreated:Copy"
EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
EventObjectDelete EventType = "s3:ObjectRemoved:Delete"
EventObjectRestoreCompleted EventType = "s3:ObjectRestore:Completed"
EventObjectTaggingPut EventType = "s3:ObjectTagging:Put"
EventObjectTaggingDelete EventType = "s3:ObjectTagging:Delete"
EventObjectAclPut EventType = "s3:ObjectAcl:Put"
// Not supported
// EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
// EventObjectRestoreDelete EventType = "s3:ObjectRestore:Delete"
)
type EventSchema struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
EventTime string `json:"eventTime"`
EventName EventType `json:"eventName"`
UserIdentity EventUserIdentity `json:"userIdentity"`
RequestParameters EventRequestParams `json:"requestParameters"`
ResponseElements EventResponseElements `json:"responseElements"`
S3 EventS3Data `json:"s3"`
GlacierEventData EventGlacierData `json:"glacierEventData"`
}
type EventUserIdentity struct {
PrincipalId string `json:"PrincipalId"`
}
type EventRequestParams struct {
SourceIPAddress string `json:"sourceIPAddress"`
}
type EventResponseElements struct {
RequestId string `json:"x-amz-request-id"`
HostId string `json:"x-amz-id-2"`
}
type EventS3Data struct {
S3SchemaVersion string `json:"s3SchemaVersion"`
ConfigurationId string `json:"configurationId"`
Bucket EventS3BucketData `json:"bucket"`
Object EventObjectData `json:"object"`
}
type EventGlacierData struct {
RestoreEventData EventRestoreData `json:"restoreEventData"`
}
type EventRestoreData struct {
LifecycleRestorationExpiryTime string `json:"lifecycleRestorationExpiryTime"`
LifecycleRestoreStorageClass string `json:"lifecycleRestoreStorageClass"`
}
type EventS3BucketData struct {
Name string `json:"name"`
OwnerIdentity EventUserIdentity `json:"ownerIdentity"`
Arn string `json:"arn"`
}
type EventObjectData struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag *string `json:"eTag"`
VersionId *string `json:"versionId"`
Sequencer string `json:"sequencer"`
}
type EventConfig struct {
KafkaURL string
KafkaTopic string
KafkaTopicKey string
NatsURL string
NatsTopic string
}
func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
if cfg.KafkaURL != "" && cfg.NatsURL != "" {
return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
}
if cfg.NatsURL != "" {
return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
}
if cfg.KafkaURL != "" {
return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
}
return nil, nil
}

153
s3event/kafka.go Normal file
View File

@@ -0,0 +1,153 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/segmentio/kafka-go"
)
var sequencer = 0
type Kafka struct {
key string
writer *kafka.Writer
mu sync.Mutex
}
func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
if topic == "" {
return nil, fmt.Errorf("kafka message topic should be specified")
}
w := kafka.NewWriter(kafka.WriterConfig{
Brokers: []string{url},
Topic: topic,
Balancer: &kafka.LeastBytes{},
BatchTimeout: 5 * time.Millisecond,
})
msg := map[string]string{
"Service": "S3",
"Event": "s3:TestEvent",
"Time": time.Now().Format(time.RFC3339),
"Bucket": "Test-Bucket",
}
msgJSON, err := json.Marshal(msg)
if err != nil {
return nil, err
}
message := kafka.Message{
Key: []byte(key),
Value: msgJSON,
}
ctx := context.Background()
err = w.WriteMessages(ctx, message)
if err != nil {
return nil, err
}
return &Kafka{
key: key,
writer: w,
}, nil
}
func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
ks.mu.Lock()
defer ks.mu.Unlock()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
schema := EventSchema{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amx-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
// This field will come up after implementing per bucket notifications
ConfigurationId: "kafka-global",
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
},
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
},
}
ks.send([]EventSchema{schema})
}
func (ks *Kafka) send(evnt []EventSchema) {
msg, err := json.Marshal(evnt)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
return
}
message := kafka.Message{
Key: []byte(ks.key),
Value: msg,
}
ctx := context.Background()
err = ks.writer.WriteMessages(ctx, message)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
}
}
func genSequencer() string {
sequencer = sequencer + 1
return fmt.Sprintf("%X", sequencer)
}

112
s3event/nats.go Normal file
View File

@@ -0,0 +1,112 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/nats-io/nats.go"
)
type NatsEventSender struct {
topic string
client *nats.Conn
mu sync.Mutex
}
func InitNatsEventService(url, topic string) (S3EventSender, error) {
if topic == "" {
return nil, fmt.Errorf("nats message topic should be specified")
}
client, err := nats.Connect(url)
if err != nil {
return nil, err
}
return &NatsEventSender{
topic: topic,
client: client,
}, nil
}
func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
ns.mu.Lock()
defer ns.mu.Unlock()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
schema := EventSchema{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amx-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
// This field will come up after implementing per bucket notifications
ConfigurationId: "nats-global",
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
},
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
},
}
ns.send([]EventSchema{schema})
}
func (ns *NatsEventSender) send(evnt []EventSchema) {
msg, err := json.Marshal(evnt)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
}
err = ns.client.Publish(ns.topic, msg)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
}
}

110
s3log/audit-logger.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"encoding/hex"
"fmt"
"math/rand"
"strings"
"time"
"github.com/gofiber/fiber/v2"
)
type AuditLogger interface {
Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
}
type LogMeta struct {
BucketOwner string
ObjectSize int64
Action string
}
type LogConfig struct {
IsFile bool
WebhookURL string
}
type LogFields struct {
BucketOwner string
Bucket string
Time time.Time
RemoteIP string
Requester string
RequestID string
Operation string
Key string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
ObjectSize int64
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
VersionID string
HostID string
SignatureVersion string
CipherSuite string
AuthenticationType string
HostHeader string
TLSVersion string
AccessPointARN string
AclRequired string
}
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
if cfg.WebhookURL != "" && cfg.IsFile {
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
}
if cfg.WebhookURL != "" {
return InitWebhookLogger(cfg.WebhookURL)
}
if cfg.IsFile {
return InitFileLogger()
}
return nil, nil
}
func genID() string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, 8)
if _, err := src.Read(b); err != nil {
panic(err)
}
return strings.ToUpper(hex.EncodeToString(b))
}
func getTLSVersionName(version uint16) string {
switch version {
case tls.VersionTLS10:
return "TLSv1.0"
case tls.VersionTLS11:
return "TLSv1.1"
case tls.VersionTLS12:
return "TLSv1.2"
case tls.VersionTLS13:
return "TLSv1.3"
default:
return ""
}
}

204
s3log/file.go Normal file
View File

@@ -0,0 +1,204 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"errors"
"fmt"
"io/fs"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
const (
logFile = "access.log"
logFileMode = 0600
timeFormat = "02/January/2006:15:04:05 -0700"
)
type FileLogger struct {
LogFields
mu sync.Mutex
}
var _ AuditLogger = &FileLogger{}
func InitFileLogger() (AuditLogger, error) {
_, err := os.ReadFile(logFile)
if err != nil && errors.Is(err, fs.ErrNotExist) {
err := os.WriteFile(logFile, []byte{}, logFileMode)
if err != nil {
return nil, err
} else {
return nil, err
}
}
return &FileLogger{}, nil
}
func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
f.mu.Lock()
defer f.mu.Unlock()
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
f.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
f.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
f.BucketOwner = meta.BucketOwner
f.Bucket = bucket
f.Time = time.Now()
f.RemoteIP = ctx.IP()
f.Requester = access
f.RequestID = genID()
f.Operation = meta.Action
f.Key = object
f.RequestURI = reqURI
f.HttpStatus = httpStatus
f.ErrorCode = errorCode
f.BytesSent = len(body)
f.ObjectSize = meta.ObjectSize
f.TotalTime = time.Since(startTime).Milliseconds()
f.TurnAroundTime = time.Since(startTime).Milliseconds()
f.Referer = ctx.Get("Referer")
f.UserAgent = ctx.Get("User-Agent")
f.VersionID = ctx.Query("versionId")
f.HostID = ctx.Get("X-Amz-Id-2")
f.SignatureVersion = "SigV4"
f.AuthenticationType = "AuthHeader"
f.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
f.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
f.AclRequired = "Yes"
f.writeLog()
}
func (fl *FileLogger) writeLog() {
if fl.BucketOwner == "" {
fl.BucketOwner = "-"
}
if fl.Bucket == "" {
fl.Bucket = "-"
}
if fl.RemoteIP == "" {
fl.RemoteIP = "-"
}
if fl.Requester == "" {
fl.Requester = "-"
}
if fl.Operation == "" {
fl.Operation = "-"
}
if fl.Key == "" {
fl.Key = "-"
}
if fl.RequestURI == "" {
fl.RequestURI = "-"
}
if fl.ErrorCode == "" {
fl.ErrorCode = "-"
}
if fl.Referer == "" {
fl.Referer = "-"
}
if fl.UserAgent == "" {
fl.UserAgent = "-"
}
if fl.VersionID == "" {
fl.VersionID = "-"
}
if fl.HostID == "" {
fl.HostID = "-"
}
if fl.CipherSuite == "" {
fl.CipherSuite = "-"
}
if fl.HostHeader == "" {
fl.HostHeader = "-"
}
if fl.TLSVersion == "" {
fl.TLSVersion = "-"
}
log := fmt.Sprintf("\n%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v",
fl.BucketOwner,
fl.Bucket,
fmt.Sprintf("[%v]", fl.Time.Format(timeFormat)),
fl.RemoteIP,
fl.Requester,
fl.RequestID,
fl.Operation,
fl.Key,
fl.RequestURI,
fl.HttpStatus,
fl.ErrorCode,
fl.BytesSent,
fl.ObjectSize,
fl.TotalTime,
fl.TurnAroundTime,
fl.Referer,
fl.UserAgent,
fl.VersionID,
fl.HostID,
fl.SignatureVersion,
fl.CipherSuite,
fl.AuthenticationType,
fl.HostHeader,
fl.TLSVersion,
fl.AccessPointARN,
fl.AclRequired,
)
file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, logFileMode)
if err != nil {
fmt.Printf("error opening the log file: %v", err.Error())
}
defer file.Close()
_, err = file.WriteString(log)
if err != nil {
fmt.Printf("error writing in log file: %v", err.Error())
}
}

141
s3log/webhook.go Normal file
View File

@@ -0,0 +1,141 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
type WebhookLogger struct {
LogFields
mu sync.Mutex
url string
}
var _ AuditLogger = &WebhookLogger{}
func InitWebhookLogger(url string) (AuditLogger, error) {
client := &http.Client{
Timeout: 3 * time.Second,
}
_, err := client.Post(url, "application/json", nil)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
return nil, fmt.Errorf("unreachable webhook url")
}
}
return &WebhookLogger{
url: url,
}, nil
}
func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
wl.mu.Lock()
defer wl.mu.Unlock()
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
wl.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
wl.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
wl.BucketOwner = meta.BucketOwner
wl.Bucket = bucket
wl.Time = time.Now()
wl.RemoteIP = ctx.IP()
wl.Requester = access
wl.RequestID = genID()
wl.Operation = meta.Action
wl.Key = object
wl.RequestURI = reqURI
wl.HttpStatus = httpStatus
wl.ErrorCode = errorCode
wl.BytesSent = len(body)
wl.ObjectSize = meta.ObjectSize
wl.TotalTime = time.Since(startTime).Milliseconds()
wl.TurnAroundTime = time.Since(startTime).Milliseconds()
wl.Referer = ctx.Get("Referer")
wl.UserAgent = ctx.Get("User-Agent")
wl.VersionID = ctx.Query("versionId")
wl.HostID = ctx.Get("X-Amz-Id-2")
wl.SignatureVersion = "SigV4"
wl.AuthenticationType = "AuthHeader"
wl.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
wl.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
wl.AclRequired = "Yes"
wl.sendLog()
}
func (wl *WebhookLogger) sendLog() {
jsonLog, err := json.Marshal(wl)
if err != nil {
fmt.Printf("\n failed to parse the log data: %v", err.Error())
}
req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
if err != nil {
fmt.Println(err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
go makeRequest(req)
}
func makeRequest(req *http.Request) {
client := &http.Client{
Timeout: 1 * time.Second,
}
_, err := client.Do(req)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
fmt.Println("error sending the log to the specified url")
}
}
}