Compare commits

..

30 Commits

Author SHA1 Message Date
Ben McClelland
34830954c3 Merge pull request #178 from versity/ben/deps
fix: upgrade module dependencies
2023-08-01 21:54:10 -07:00
Ben McClelland
77a4a9e3a5 fix: upgrade module dependencies 2023-08-01 21:50:17 -07:00
Ben McClelland
25b02dc8fa Merge pull request #177 from versity/select-object-content-fe
SelectObjectContent FE
2023-08-01 13:37:29 -07:00
jonaustin09
009ceee748 feat: Added FE support for SelectObjectContent action 2023-08-02 00:08:28 +04:00
Ben McClelland
af69adf080 Merge pull request #176 from versity/fix/s3response-cleanup
s3response cleanup
2023-07-31 21:45:18 -07:00
jonaustin09
97847735c8 fix: s3response action responses naming cleanup 2023-07-31 21:41:10 -07:00
Ben McClelland
ac9aa25ff1 Merge pull request #175 from versity/fix/issue-143
Issue 143
2023-07-31 21:38:10 -07:00
Jon Austin
091375fa00 Issue 151 (#174)
* fix: Fixes #151. Fixed DeleteObjects action bugs: Corrected request body serialization type, added return type
2023-07-31 21:36:33 -07:00
Ben McClelland
f1e22b0a4d Merge pull request #173 from versity/fix/issue-168
Issu 168
2023-07-31 21:35:00 -07:00
jonaustin09
3f8c218431 fix: Fixes #143. Fixed action name in bucket creation admin checker response handler 2023-07-31 20:54:16 +04:00
jonaustin09
70818de594 fix: Fixes #168. Changed PutObject existing object error from custom internal error to ErrExistingObjectIsDirectory 2023-07-31 18:17:29 +04:00
Ben McClelland
366ed21ede Merge pull request #172 from versity/fix/issue-152
Issue 152
2023-07-28 21:24:39 -07:00
Ben McClelland
b96da570a7 Merge pull request #171 from versity/fix/issue-153
Issue 153
2023-07-28 21:23:22 -07:00
jonaustin09
898c3efaa0 fix: Fixes #153. Fixed CompleteMultipartUpload invalid ETag error case, fixed UploadPart xattr.Set error 2023-07-28 18:20:07 +04:00
jonaustin09
838a7f9ef9 fix: Fixes #152. Changed CompleteMultiPartUpload invalid payload error to MalformedXML 2023-07-28 18:19:15 +04:00
Jon Austin
bf33b9f5a2 Issue 154 (#169)
* fix: Fixes #154, Changed GetObject range error to InvalidRange
2023-07-27 11:05:40 -07:00
Jon Austin
77080328c1 Issue 156 (#167)
* fix: Fixes #156, Added bucket name validation on bucket creation
2023-07-27 11:04:50 -07:00
Ben McClelland
b0259ae1de Merge pull request #166 from versity/ben/context 2023-07-27 06:44:43 -07:00
Ben McClelland
884fd029c3 feat: add context to backend calls
This adds a context to the backend interface calls so that the backend
can enable request cancellation. This change isn't acutally implementing
any backend handling, but just putting the pieces into place to pass the
context to the backend.
2023-07-26 21:54:12 -07:00
Ben McClelland
36eb6d795f Merge pull request #165 from versity/acl-checker-refactoring
ACL refactoring
2023-07-26 19:06:52 -07:00
Ben McClelland
7de01cc983 Merge pull request #163 from versity/ben/log_cleanup
fix: allow logging to user specified log files
2023-07-26 19:05:39 -07:00
jonaustin09
7fb2a7f9ba feat: ACL refactoring, moved ace parsing from controllers to middleware 2023-07-26 20:54:50 +04:00
Ben McClelland
5a9b744dd1 fix: allow logging to user specified log files
This also cleans up some the of the error output to send to stderr.

This adds the Shutdown() to the logging interface, so we can keep the
log file open and just append entries.

This add HangUp() to the logging interface for log rotations.
2023-07-25 23:39:45 -07:00
Ben McClelland
5b31a7bafc Merge pull request #162 from versity/fix/issue-136
Issue 136
2023-07-25 10:03:28 -07:00
Ben McClelland
ee703479d0 Merge pull request #161 from versity/fix/issue-150
Issue 150
2023-07-25 10:02:57 -07:00
Ben McClelland
bedd353d72 Merge pull request #160 from versity/fix/issue-155
Issue 155
2023-07-25 10:02:16 -07:00
Ben McClelland
84fe647b81 Merge pull request #159 from versity/fix/issue-157
Issue 157
2023-07-25 10:00:35 -07:00
jonaustin09
4c451a4822 feat: Added support to add object tags on object creation 2023-07-25 20:42:58 +04:00
jonaustin09
287db7a7b6 fix: Fixed ListObjects marker bug, now it takes the correct query param as marker 2023-07-25 20:31:40 +04:00
jonaustin09
c598ee5416 fix: Added accept-range, Content-range and x-amz-tagging-count headers in GetObject action response, added test cases for these 2023-07-25 20:28:40 +04:00
23 changed files with 1330 additions and 754 deletions

View File

@@ -261,5 +261,5 @@ func IsAdmin(access string, isRoot bool) error {
if acc.Role == "admin" {
return nil
}
return fmt.Errorf("only admin users have access to this resource")
return s3err.GetAPIError(s3err.ErrAccessDenied)
}

View File

@@ -15,6 +15,7 @@
package backend
import (
"context"
"fmt"
"io"
@@ -28,37 +29,38 @@ type Backend interface {
fmt.Stringer
Shutdown()
ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error)
CreateBucket(*s3.CreateBucketInput) error
PutBucketAcl(bucket string, data []byte) error
DeleteBucket(*s3.DeleteBucketInput) error
ListBuckets(_ context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(context.Context, *s3.CreateBucketInput) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error)
UploadPart(*s3.UploadPartInput) (etag string, err error)
UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
PutObject(*s3.PutObjectInput) (string, error)
HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(*s3.DeleteObjectInput) error
DeleteObjects(*s3.DeleteObjectsInput) error
PutObjectAcl(*s3.PutObjectAclInput) error
RestoreObject(*s3.RestoreObjectInput) error
PutObject(context.Context, *s3.PutObjectInput) (string, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
RestoreObject(context.Context, *s3.RestoreObjectInput) error
GetTags(bucket, object string) (map[string]string, error)
SetTags(bucket, object string, tags map[string]string) error
RemoveTags(bucket, object string) error
GetTags(_ context.Context, bucket, object string) (map[string]string, error)
SetTags(_ context.Context, bucket, object string, tags map[string]string) error
RemoveTags(_ context.Context, bucket, object string) error
}
type BackendUnsupported struct{}
@@ -72,90 +74,93 @@ func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets(string, bool) (s3response.ListAllMyBucketsResult, error) {
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error {
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RestoreObject(*s3.RestoreObjectInput) error {
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error) {
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateBucket(*s3.CreateBucketInput) error {
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(*s3.DeleteBucketInput) error {
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error {
func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(*s3.UploadPartInput) (etag string, err error) {
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(*s3.DeleteObjectInput) error {
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(*s3.DeleteObjectsInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
func (BackendUnsupported) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
func (BackendUnsupported) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RemoveTags(bucket, object string) error {
func (BackendUnsupported) RemoveTags(_ context.Context, bucket, object string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}

View File

@@ -56,7 +56,7 @@ func GetTimePtr(t time.Time) *time.Time {
}
var (
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRequest)
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
)
// ParseRange parses input range header and returns startoffset, length, and

View File

@@ -15,6 +15,7 @@
package posix
import (
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
@@ -96,7 +97,7 @@ func (p *Posix) String() string {
return "Posix Gateway"
}
func (p *Posix) ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
func (p *Posix) ListBuckets(_ context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
entries, err := os.ReadDir(".")
if err != nil {
return s3response.ListAllMyBucketsResult{},
@@ -131,7 +132,7 @@ func (p *Posix) ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBuck
}, nil
}
func (p *Posix) HeadBucket(input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
_, err := os.Lstat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -143,7 +144,7 @@ func (p *Posix) HeadBucket(input *s3.HeadBucketInput) (*s3.HeadBucketOutput, err
return &s3.HeadBucketOutput{}, nil
}
func (p *Posix) CreateBucket(input *s3.CreateBucketInput) error {
func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error {
bucket := *input.Bucket
owner := string(input.ObjectOwnership)
@@ -168,7 +169,7 @@ func (p *Posix) CreateBucket(input *s3.CreateBucketInput) error {
return nil
}
func (p *Posix) DeleteBucket(input *s3.DeleteBucketInput) error {
func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
names, err := os.ReadDir(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -197,7 +198,7 @@ func (p *Posix) DeleteBucket(input *s3.DeleteBucketInput) error {
return nil
}
func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
bucket := *mpu.Bucket
object := *mpu.Key
@@ -248,7 +249,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
}, nil
}
func (p *Posix) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
@@ -294,7 +295,9 @@ func (p *Posix) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput)
if err != nil {
etag = ""
}
parts[i].ETag = &etag
if etag != *parts[i].ETag {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
}
f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, totalsize)
@@ -475,7 +478,7 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
return nil
}
func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error {
func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
bucket := *mpu.Bucket
object := *mpu.Key
uploadID := *mpu.UploadId
@@ -505,7 +508,7 @@ func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error {
return nil
}
func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
bucket := *mpu.Bucket
var delimiter string
if mpu.Delimiter != nil {
@@ -516,7 +519,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo
prefix = *mpu.Prefix
}
var lmu s3response.ListMultipartUploadsResponse
var lmu s3response.ListMultipartUploadsResult
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -592,7 +595,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo
Initiated: fi.ModTime().Format(backend.RFC3339TimeFormat),
})
if len(uploads) == int(mpu.MaxUploads) {
return s3response.ListMultipartUploadsResponse{
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
IsTruncated: i != len(objs) || j != len(upids),
@@ -608,7 +611,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo
}
}
return s3response.ListMultipartUploadsResponse{
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
@@ -619,14 +622,14 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo
}, nil
}
func (p *Posix) ListParts(input *s3.ListPartsInput) (s3response.ListPartsResponse, error) {
func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
stringMarker := *input.PartNumberMarker
maxParts := int(input.MaxParts)
var lpr s3response.ListPartsResponse
var lpr s3response.ListPartsResult
var partNumberMarker int
if stringMarker != "" {
@@ -705,7 +708,7 @@ func (p *Posix) ListParts(input *s3.ListPartsInput) (s3response.ListPartsRespons
upiddir := filepath.Join(objdir, uploadID)
loadUserMetaData(upiddir, userMetaData)
return s3response.ListPartsResponse{
return s3response.ListPartsResult{
Bucket: bucket,
IsTruncated: oldLen != newLen,
Key: object,
@@ -717,7 +720,7 @@ func (p *Posix) ListParts(input *s3.ListPartsInput) (s3response.ListPartsRespons
}, nil
}
func (p *Posix) UploadPart(input *s3.UploadPartInput) (string, error) {
func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
@@ -751,7 +754,6 @@ func (p *Posix) UploadPart(input *s3.UploadPartInput) (string, error) {
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
}
defer f.cleanup()
hash := md5.New()
tr := io.TeeReader(r, hash)
@@ -765,14 +767,16 @@ func (p *Posix) UploadPart(input *s3.UploadPartInput) (string, error) {
return "", fmt.Errorf("link object in namespace: %w", err)
}
f.cleanup()
dataSum := hash.Sum(nil)
etag := hex.EncodeToString(dataSum)
xattr.Set(partPath, etagkey, []byte(etag))
xattr.Set(filepath.Join(bucket, partPath), etagkey, []byte(etag))
return etag, nil
}
func (p *Posix) UploadPartCopy(upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
_, err := os.Stat(*upi.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -877,7 +881,9 @@ func (p *Posix) UploadPartCopy(upi *s3.UploadPartCopyInput) (s3response.CopyObje
}, nil
}
func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
tagsStr := getString(po.Tagging)
tags := make(map[string]string)
_, err := os.Stat(*po.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -886,6 +892,17 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
return "", fmt.Errorf("stat bucket: %w", err)
}
if tagsStr != "" {
tagParts := strings.Split(tagsStr, "&")
for _, prt := range tagParts {
p := strings.Split(prt, "=")
if len(p) != 2 {
return "", s3err.GetAPIError(s3err.ErrInvalidTag)
}
tags[p[0]] = p[1]
}
}
name := filepath.Join(*po.Bucket, *po.Key)
if strings.HasSuffix(*po.Key, "/") {
@@ -929,13 +946,20 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
err = f.link()
if err != nil {
return "", fmt.Errorf("link object in namespace: %w", err)
return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
}
for k, v := range po.Metadata {
xattr.Set(name, "user."+k, []byte(v))
}
if tagsStr != "" {
err := p.SetTags(ctx, *po.Bucket, *po.Key, tags)
if err != nil {
return "", err
}
}
dataSum := hash.Sum(nil)
etag := hex.EncodeToString(dataSum[:])
xattr.Set(name, etagkey, []byte(etag))
@@ -943,7 +967,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
return etag, nil
}
func (p *Posix) DeleteObject(input *s3.DeleteObjectInput) error {
func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
bucket := *input.Bucket
object := *input.Key
@@ -1000,25 +1024,46 @@ func (p *Posix) removeParents(bucket, object string) error {
return nil
}
func (p *Posix) DeleteObjects(input *s3.DeleteObjectsInput) error {
func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
// delete object already checks bucket
delResult, errs := []types.DeletedObject{}, []types.Error{}
for _, obj := range input.Delete.Objects {
err := p.DeleteObject(&s3.DeleteObjectInput{
//TODO: Make the delete operation concurrent
err := p.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: input.Bucket,
Key: obj.Key,
})
if err != nil {
return err
if err == nil {
delResult = append(delResult, types.DeletedObject{Key: obj.Key})
} else {
serr, ok := err.(s3err.APIError)
if ok {
errs = append(errs, types.Error{
Key: obj.Key,
Code: &serr.Code,
Message: &serr.Description,
})
} else {
errs = append(errs, types.Error{
Key: obj.Key,
Code: getStringPtr("InternalError"),
Message: getStringPtr(err.Error()),
})
}
}
}
return nil
return s3response.DeleteObjectsResult{
Deleted: delResult,
Errors: errs,
}, nil
}
func (p *Posix) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
var contentRange string
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1050,6 +1095,10 @@ func (p *Posix) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetOb
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if acceptRange != "" {
contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, fi.Size())
}
f, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
@@ -1089,10 +1138,11 @@ func (p *Posix) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetOb
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: int32(len(tags)),
ContentRange: &contentRange,
}, nil
}
func (p *Posix) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
@@ -1132,7 +1182,7 @@ func (p *Posix) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, err
}, nil
}
func (p *Posix) CopyObject(input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
if !ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
@@ -1171,7 +1221,7 @@ func (p *Posix) CopyObject(input *s3.CopyObjectInput) (*s3.CopyObjectOutput, err
return nil, fmt.Errorf("stat object: %w", err)
}
etag, err := p.PutObject(&s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size()})
etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size()})
if err != nil {
return nil, err
}
@@ -1189,7 +1239,7 @@ func (p *Posix) CopyObject(input *s3.CopyObjectInput) (*s3.CopyObjectOutput, err
}, nil
}
func (p *Posix) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.Marker
@@ -1285,7 +1335,7 @@ func fileToObj(bucket string) backend.GetObjFunc {
}
}
func (p *Posix) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.ContinuationToken
@@ -1321,7 +1371,7 @@ func (p *Posix) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Ou
}, nil
}
func (p *Posix) PutBucketAcl(bucket string, data []byte) error {
func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1337,7 +1387,7 @@ func (p *Posix) PutBucketAcl(bucket string, data []byte) error {
return nil
}
func (p *Posix) GetBucketAcl(input *s3.GetBucketAclInput) ([]byte, error) {
func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
_, err := os.Stat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1356,7 +1406,7 @@ func (p *Posix) GetBucketAcl(input *s3.GetBucketAclInput) ([]byte, error) {
return b, nil
}
func (p *Posix) GetTags(bucket, object string) (map[string]string, error) {
func (p *Posix) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1389,7 +1439,7 @@ func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
return tags, nil
}
func (p *Posix) SetTags(bucket, object string, tags map[string]string) error {
func (p *Posix) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1425,8 +1475,8 @@ func (p *Posix) SetTags(bucket, object string, tags map[string]string) error {
return nil
}
func (p *Posix) RemoveTags(bucket, object string) error {
return p.SetTags(bucket, object, nil)
func (p *Posix) RemoveTags(ctx context.Context, bucket, object string) error {
return p.SetTags(ctx, bucket, object, nil)
}
const (
@@ -1634,3 +1684,14 @@ func isNoAttr(err error) bool {
}
return false
}
func getString(str *string) string {
if str == nil {
return ""
}
return *str
}
func getStringPtr(str string) *string {
return &str
}

View File

@@ -15,6 +15,7 @@
package scoutfs
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
@@ -114,7 +115,7 @@ func (*ScoutFS) String() string {
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
// ioctl to not have to read and copy the part data to the final object. This
// saves a read and write cycle for all mutlipart uploads.
func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
@@ -165,7 +166,9 @@ func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput
if err != nil {
etag = ""
}
parts[i].ETag = &etag
if etag != *parts[i].ETag {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
}
// use totalsize=0 because we wont be writing to the file, only moving
@@ -352,7 +355,7 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
return nil
}
func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
@@ -425,7 +428,7 @@ func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, e
}, nil
}
func (s *ScoutFS) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
@@ -539,7 +542,7 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
return tags, nil
}
func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.Marker
@@ -574,7 +577,7 @@ func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput
}, nil
}
func (s *ScoutFS) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.ContinuationToken
@@ -687,7 +690,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
// RestoreObject will set stage request on file if offline and do nothing if
// file is online
func (s *ScoutFS) RestoreObject(input *s3.RestoreObjectInput) error {
func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
bucket := *input.Bucket
object := *input.Key

View File

@@ -40,7 +40,7 @@ var (
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
logWebhookURL string
accessLog bool
accessLog string
debug bool
)
@@ -147,14 +147,16 @@ func initFlags() []cli.Flag {
Usage: "enable debug output",
Destination: &debug,
},
&cli.BoolFlag{
&cli.StringFlag{
Name: "access-log",
Usage: "enable server access logging in the root directory",
Usage: "enable server access logging to specified file",
EnvVars: []string{"LOGFILE"},
Destination: &accessLog,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
EnvVars: []string{"WEBHOOK"},
Destination: &logWebhookURL,
},
&cli.StringFlag{
@@ -229,7 +231,7 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
}
logger, err := s3log.InitLogger(&s3log.LogConfig{
IsFile: accessLog,
LogFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
@@ -258,12 +260,32 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
c := make(chan error, 1)
go func() { c <- srv.Serve() }()
select {
case <-ctx.Done():
be.Shutdown()
return ctx.Err()
case err := <-c:
be.Shutdown()
return err
// for/select blocks until shutdown
Loop:
for {
select {
case <-ctx.Done():
err = ctx.Err()
break Loop
case err = <-c:
break Loop
case <-sigHup:
if logger != nil {
err = logger.HangUp()
if err != nil {
err = fmt.Errorf("HUP logger: %w", err)
break Loop
}
}
}
}
be.Shutdown()
if logger != nil {
lerr := logger.Shutdown()
if lerr != nil {
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", lerr)
}
}
return err
}

View File

@@ -23,6 +23,7 @@ import (
var (
sigDone = make(chan bool, 1)
sigHup = make(chan bool, 1)
)
func setupSignalHandler() {
@@ -36,6 +37,7 @@ func setupSignalHandler() {
case syscall.SIGINT, syscall.SIGTERM:
sigDone <- true
case syscall.SIGHUP:
sigHup <- true
}
}
}()

48
go.mod
View File

@@ -3,10 +3,10 @@ module github.com/versity/versitygw
go 1.20
require (
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0
github.com/aws/smithy-go v1.13.5
github.com/gofiber/fiber/v2 v2.47.0
github.com/aws/aws-sdk-go-v2 v1.20.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1
github.com/aws/smithy-go v1.14.0
github.com/gofiber/fiber/v2 v2.48.0
github.com/google/uuid v1.3.0
github.com/nats-io/nats.go v1.28.0
github.com/pkg/xattr v0.4.9
@@ -18,17 +18,17 @@ require (
)
require (
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/nats-io/nats-server/v2 v2.9.20 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/crypto v0.11.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
@@ -36,28 +36,24 @@ require (
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.27
github.com/aws/aws-sdk-go-v2/credentials v1.13.26
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.32
github.com/aws/aws-sdk-go-v2/credentials v1.13.31
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
github.com/tinylib/msgp v1.1.8 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

118
go.sum
View File

@@ -1,50 +1,50 @@
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w=
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8=
github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU=
github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI=
github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc=
github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk=
github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 h1:DJ1kHj0GI9BbX+XhF0kHxlzOVjcncmDUXmCvXdbfdAE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76/go.mod h1:/AZCdswMSgwpB2yMSFfY5H4pVeBLnCuPehdmO/r3xSM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0=
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY=
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg=
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs=
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI=
github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io=
github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.47.0 h1:EN5lHVCc+Pyqh5OEsk8fzRiifgwpbrP0rulQ4iNf3fs=
github.com/gofiber/fiber/v2 v2.47.0/go.mod h1:mbFMVN1lQuzziTkkakgtKKdjfsXSw9BKR5lmcNksUoU=
github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0=
github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -65,8 +65,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
@@ -77,12 +77,9 @@ github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -92,11 +89,6 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -106,9 +98,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -127,64 +116,45 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=

View File

@@ -27,10 +27,17 @@ func TestMakeBucket(s *S3Conf) {
s3client := s3.NewFromConfig(s.Config())
invBucket := "aa"
err := setup(s, invBucket)
if err == nil {
failF("%v: expected bucket name validation error", testname)
return
}
bucket := "testbucket"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket})
_, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket})
cancel()
if err == nil {
failF("%v: expected error, instead got success response", testname)
@@ -216,8 +223,9 @@ func TestPutGetObject(s *S3Conf) {
}
}
if objCount != 2 {
if objCount != 0 {
failF("%v: expected object count %v instead got %v", testname, 2, objCount)
return
}
err = teardown(s, bucket)
@@ -770,8 +778,45 @@ func TestIncorrectMultiParts(s *S3Conf) {
return
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mpu.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: []types.CompletedPart{
{
ETag: mp2.ETag,
PartNumber: 96,
},
{
ETag: mp1.ETag,
PartNumber: 99,
},
},
},
})
cancel()
if err == nil {
failF("%v: complete multipart expected err", testname)
return
}
badEtag := "bogusEtagValue"
// Empty multipart upload
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mpu.UploadId,
})
cancel()
if err == nil {
failF("%v: complete multipart expected err", testname)
return
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -785,7 +830,7 @@ func TestIncorrectMultiParts(s *S3Conf) {
},
{
ETag: &badEtag,
PartNumber: 99,
PartNumber: 42,
},
},
},
@@ -1024,6 +1069,20 @@ func TestRangeGet(s *S3Conf) {
return
}
// Invalid range
invRange := "bytes=100-asd"
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &name,
Range: &invRange,
})
defer cancel()
if err == nil {
failF("%v: expected range error", testname)
return
}
rangeString := "bytes=100-200"
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -1039,6 +1098,14 @@ func TestRangeGet(s *S3Conf) {
}
defer out.Body.Close()
if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", datalen) {
failF("%v: expected content range: %v, instead got: %v", testname, fmt.Sprintf("bytes 100-200/%v", datalen), getString(out.ContentRange))
return
}
if getString(out.AcceptRanges) != rangeString {
failF("%v: expected accept range: %v, instead got: %v", testname, rangeString, getString(out.AcceptRanges))
}
b, err := io.ReadAll(out.Body)
if err != nil {
failF("%v: read body %v", testname, err)

View File

@@ -154,3 +154,10 @@ func execCommand(args ...string) ([]byte, error) {
return cmd.CombinedOutput()
}
func getString(str *string) string {
if str == nil {
return ""
}
return *str
}

File diff suppressed because it is too large Load Diff

View File

@@ -55,10 +55,7 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
if err := auth.IsAdmin(access, isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
}
res, err := c.be.ListBuckets(access, isRoot)
res, err := c.be.ListBuckets(ctx.Context(), access, isRoot)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
}
@@ -72,26 +69,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
acceptRange := ctx.Get("Range")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
}
tags, err := c.be.GetTags(bucket, key)
tags, err := c.be.GetTags(ctx.Context(), bucket, key)
if err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
}
@@ -119,7 +107,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListParts(&s3.ListPartsInput{
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
@@ -133,7 +121,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.GetObjectAcl(&s3.GetObjectAclInput{
res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{
Bucket: &bucket,
Key: &key,
})
@@ -148,7 +136,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
for _, a := range strings.Split(attrs, ",") {
oattrs = append(oattrs, types.ObjectAttributes(a))
}
res, err := c.be.GetObjectAttributes(&s3.GetObjectAttributesInput{
res, err := c.be.GetObjectAttributes(ctx.Context(), &s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &key,
ObjectAttributes: oattrs,
@@ -161,7 +149,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
}
ctx.Locals("logResBody", false)
res, err := c.be.GetObject(&s3.GetObjectInput{
res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
@@ -178,6 +166,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
if res.LastModified != nil {
lastmod = res.LastModified.Format(timefmt)
}
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "Content-Length",
@@ -203,6 +192,18 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
},
{
Key: "Content-Range",
Value: getstring(res.ContentRange),
},
{
Key: "accept-ranges",
Value: getstring(res.AcceptRanges),
},
{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(res.TagCount),
},
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
@@ -217,27 +218,24 @@ func getstring(s *string) string {
func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
bucket := ctx.Params("bucket")
prefix := ctx.Query("prefix")
marker := ctx.Query("continuation-token")
cToken := ctx.Query("continuation-token")
marker := ctx.Query("marker")
delimiter := ctx.Query("delimiter")
maxkeys := ctx.QueryInt("max-keys")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
if ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
}
data, err := c.be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
res, err := auth.ParseACLOutput(data)
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
}
@@ -246,7 +244,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
}
@@ -254,10 +252,10 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListObjectsV2(&s3.ListObjectsV2Input{
res, err := c.be.ListObjectsV2(ctx.Context(), &s3.ListObjectsV2Input{
Bucket: &bucket,
Prefix: &prefix,
ContinuationToken: &marker,
ContinuationToken: &cToken,
Delimiter: &delimiter,
MaxKeys: int32(maxkeys),
})
@@ -268,7 +266,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.ListObjects(&s3.ListObjectsInput{
res, err := c.be.ListObjects(ctx.Context(), &s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
Marker: &marker,
@@ -294,18 +292,9 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
if ctx.Request().URI().QueryArgs().Has("acl") {
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
var input *s3.PutBucketAclInput
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
@@ -359,15 +348,19 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
err = c.be.PutBucketAcl(bucket, updAcl)
err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl)
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
}
err := c.be.CreateBucket(&s3.CreateBucketInput{
if ok := utils.IsValidBucketName(bucket); !ok {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), &MetaOpts{Logger: c.logger, Action: "CreateBucket"})
}
err := c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{
Bucket: &bucket,
ObjectOwnership: types.ObjectOwnership(access),
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "CreateBucket", BucketOwner: access})
}
func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
@@ -377,6 +370,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
uploadId := ctx.Query("uploadId")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
tagging := ctx.Get("x-amz-tagging")
// Copy source headers
copySource := ctx.Get("X-Amz-Copy-Source")
@@ -408,16 +403,6 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
keyStart = keyStart + "/"
}
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
var objTagging s3response.Tagging
err := xml.Unmarshal(ctx.Body(), &objTagging)
@@ -435,7 +420,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
}
err = c.be.SetTags(bucket, keyStart, tags)
err = c.be.SetTags(ctx.Context(), bucket, keyStart, tags)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
@@ -451,7 +436,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
}
resp, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{
resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &keyStart,
CopySource: &copySource,
@@ -480,7 +465,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
ctx.Locals("logReqBody", false)
etag, err := c.be.UploadPart(&s3.UploadPartInput{
etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
@@ -542,7 +527,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
}
err = c.be.PutObjectAcl(input)
err := c.be.PutObjectAcl(ctx.Context(), input)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
@@ -558,6 +543,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
var mtime time.Time
var err error
if copySrcModifSince != "" {
mtime, err = time.Parse(iso8601Format, copySrcModifSince)
if err != nil {
@@ -571,7 +557,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
}
res, err := c.be.CopyObject(&s3.CopyObjectInput{
res, err := c.be.CopyObject(ctx.Context(), &s3.CopyObjectInput{
Bucket: &bucket,
Key: &keyStart,
CopySource: &copySource,
@@ -611,12 +597,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
ctx.Locals("logReqBody", false)
etag, err := c.be.PutObject(&s3.PutObjectInput{
etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &keyStart,
ContentLength: contentLength,
Metadata: metadata,
Body: bytes.NewReader(ctx.Request().Body()),
Tagging: &tagging,
})
ctx.Response().Header.Set("ETag", etag)
return SendResponse(ctx, err, &MetaOpts{
@@ -631,41 +618,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
}
bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteBucket(&s3.DeleteBucketInput{
err := c.be.DeleteBucket(ctx.Context(), &s3.DeleteBucketInput{
Bucket: &bucket,
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
var dObj types.Delete
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
}
bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
var dObj s3response.DeleteObjects
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
@@ -675,11 +642,13 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteObjects(&s3.DeleteObjectsInput{
res, err := c.be.DeleteObjects(ctx.Context(), &s3.DeleteObjectsInput{
Bucket: &bucket,
Delete: &dObj,
Delete: &types.Delete{
Objects: dObj.Objects,
},
})
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
}
func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
@@ -689,27 +658,18 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
uploadId := ctx.Query("uploadId")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner})
}
err = c.be.RemoveTags(bucket, key)
err := c.be.RemoveTags(ctx.Context(), bucket, key)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
@@ -726,7 +686,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
}
err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
err := c.be.AbortMultipartUpload(ctx.Context(), &s3.AbortMultipartUploadInput{
UploadId: &uploadId,
Bucket: &bucket,
Key: &key,
@@ -740,7 +700,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObject", BucketOwner: parsedAcl.Owner})
}
err = c.be.DeleteObject(&s3.DeleteObjectInput{
err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{
Bucket: &bucket,
Key: &key,
})
@@ -754,23 +714,13 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
}
func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
}
bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner})
}
_, err = c.be.HeadBucket(&s3.HeadBucketInput{
_, err := c.be.HeadBucket(ctx.Context(), &s3.HeadBucketInput{
Bucket: &bucket,
})
// TODO: set bucket response headers
@@ -782,30 +732,18 @@ const (
)
func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{
Bucket: &bucket,
})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.HeadObject(&s3.HeadObjectInput{
res, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
})
@@ -862,21 +800,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
uploadId := ctx.Query("uploadId")
access := ctx.Locals("access").(string)
isRoot := ctx.Locals("isRoot").(bool)
parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
}
var restoreRequest s3.RestoreObjectInput
if ctx.Request().URI().QueryArgs().Has("restore") {
err := xml.Unmarshal(ctx.Body(), &restoreRequest)
@@ -891,7 +820,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
restoreRequest.Bucket = &bucket
restoreRequest.Key = &key
err = c.be.RestoreObject(&restoreRequest)
err = c.be.RestoreObject(ctx.Context(), &restoreRequest)
return SendResponse(ctx, err, &MetaOpts{
Logger: c.logger,
EvSender: c.evSender,
@@ -901,20 +830,52 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
})
}
if ctx.Request().URI().QueryArgs().Has("select") && ctx.Query("select-type") == "2" {
var payload s3response.SelectObjectContentPayload
if err := xml.Unmarshal(ctx.Body(), &payload); err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrMalformedXML), &MetaOpts{
Logger: c.logger,
Action: "SelectObjectContent",
BucketOwner: parsedAcl.Owner,
})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
Bucket: &bucket,
Key: &key,
Expression: payload.Expression,
ExpressionType: payload.ExpressionType,
InputSerialization: payload.InputSerialization,
OutputSerialization: payload.OutputSerialization,
RequestProgress: payload.RequestProgress,
ScanRange: payload.ScanRange,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
}
if uploadId != "" {
data := struct {
Parts []types.CompletedPart `xml:"Part"`
}{}
if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrMalformedXML), &MetaOpts{
Logger: c.logger,
Action: "CompleteMultipartUpload",
BucketOwner: parsedAcl.Owner,
})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
res, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
@@ -945,7 +906,8 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
}
res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
res, err := c.be.CreateMultipartUpload(ctx.Context(),
&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
}

View File

@@ -15,6 +15,7 @@
package controllers
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -89,10 +90,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, nil
},
},
@@ -110,10 +111,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
},
},
@@ -127,16 +128,6 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
})
appErr.Get("/", s3ApiControllerErr.ListBuckets)
//Admin error case
admErr := fiber.New()
admErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", false)
ctx.Locals("isDebug", false)
return ctx.Next()
})
admErr.Get("/", s3ApiController.ListBuckets)
tests := []struct {
name string
args args
@@ -162,15 +153,6 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "admin-error-case",
args: args{
req: httptest.NewRequest(http.MethodGet, "/", nil),
},
app: admErr,
wantErr: false,
statusCode: 500,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -201,19 +183,19 @@ func TestS3ApiController_GetActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListPartsFunc: func(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, nil
ListPartsFunc: func(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, nil
},
GetObjectAclFunc: func(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return &s3.GetObjectAttributesOutput{}, nil
},
GetObjectFunc: func(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Metadata: map[string]string{"hello": "world"},
ContentType: getPtr("application/xml"),
@@ -224,7 +206,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
StorageClass: "storage class",
}, nil
},
GetTagsFunc: func(bucket, object string) (map[string]string, error) {
GetTagsFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
return map[string]string{"hello": "world"}, nil
},
},
@@ -233,6 +215,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Get("/:bucket/:key/*", s3ApiController.GetActions)
@@ -353,16 +336,16 @@ func TestS3ApiController_ListActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
return s3response.ListMultipartUploadsResponse{}, nil
ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, nil
},
ListObjectsV2Func: func(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
},
ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
},
@@ -372,6 +355,7 @@ func TestS3ApiController_ListActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
@@ -380,10 +364,10 @@ func TestS3ApiController_ListActions(t *testing.T) {
//Error case
s3ApiControllerError := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
},
@@ -393,6 +377,7 @@ func TestS3ApiController_ListActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
appError.Get("/:bucket", s3ApiControllerError.ListActions)
@@ -498,13 +483,13 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
PutBucketAclFunc: func(string, []byte) error {
PutBucketAclFunc: func(context.Context, string, []byte) error {
return nil
},
CreateBucketFunc: func(*s3.CreateBucketInput) error {
CreateBucketFunc: func(context.Context, *s3.CreateBucketInput) error {
return nil
},
},
@@ -514,6 +499,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
return ctx.Next()
})
app.Put("/:bucket", s3ApiController.PutBucketActions)
@@ -593,6 +579,15 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Put-bucket-invalid-bucket-name",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/aa", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-bucket-success",
app: app,
@@ -650,27 +645,27 @@ func TestS3ApiController_PutActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
PutObjectAclFunc: func(*s3.PutObjectAclInput) error {
PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
return nil
},
CopyObjectFunc: func(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return &s3.CopyObjectOutput{
CopyObjectResult: &types.CopyObjectResult{},
}, nil
},
PutObjectFunc: func(*s3.PutObjectInput) (string, error) {
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
return "ETag", nil
},
UploadPartFunc: func(*s3.UploadPartInput) (string, error) {
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
return "hello", nil
},
SetTagsFunc: func(bucket, object string, tags map[string]string) error {
SetTagsFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
return nil
},
UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
},
},
@@ -679,6 +674,7 @@ func TestS3ApiController_PutActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Put("/:bucket/:key/*", s3ApiController.PutActions)
@@ -864,10 +860,10 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteBucketFunc: func(*s3.DeleteBucketInput) error {
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
return nil
},
},
@@ -877,6 +873,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
@@ -920,11 +917,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectsFunc: func(*s3.DeleteObjectsInput) error {
return nil
DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
return s3response.DeleteObjectsResult{}, nil
},
},
}
@@ -933,6 +930,7 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Post("/:bucket", s3ApiController.DeleteObjects)
@@ -990,16 +988,16 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return nil
},
AbortMultipartUploadFunc: func(*s3.AbortMultipartUploadInput) error {
AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
return nil
},
RemoveTagsFunc: func(bucket, object string) error {
RemoveTagsFunc: func(_ context.Context, bucket, object string) error {
return nil
},
},
@@ -1009,6 +1007,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
@@ -1017,10 +1016,10 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(7)
},
}}
@@ -1029,6 +1028,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
@@ -1098,10 +1098,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return &s3.HeadBucketOutput{}, nil
},
},
@@ -1111,6 +1111,7 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
@@ -1120,10 +1121,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(3)
},
},
@@ -1133,6 +1134,7 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
@@ -1192,10 +1194,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return &s3.HeadObjectOutput{
ContentEncoding: &contentEncoding,
ContentLength: 64,
@@ -1211,6 +1213,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
@@ -1220,10 +1223,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(42)
},
},
@@ -1233,6 +1236,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
@@ -1283,25 +1287,36 @@ func TestS3ApiController_CreateActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
RestoreObjectFunc: func(restoreRequest *s3.RestoreObjectInput) error {
RestoreObjectFunc: func(context.Context, *s3.RestoreObjectInput) error {
return nil
},
CompleteMultipartUploadFunc: func(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return &s3.CompleteMultipartUploadOutput{}, nil
},
CreateMultipartUploadFunc: func(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return &s3.CreateMultipartUploadOutput{}, nil
},
SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, nil
},
},
}
bdy := `
<SelectObjectContentRequest>
<Expression>string</Expression>
<ExpressionType>string</ExpressionType>
</SelectObjectContentRequest>
`
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("access", "valid access")
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
return ctx.Next()
})
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
@@ -1331,6 +1346,24 @@ func TestS3ApiController_CreateActions(t *testing.T) {
wantErr: false,
statusCode: 500,
},
{
name: "Select-object-content-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Select-object-content-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", strings.NewReader(bdy)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Complete-multipart-upload-error",
app: app,
@@ -1338,7 +1371,7 @@ func TestS3ApiController_CreateActions(t *testing.T) {
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", nil),
},
wantErr: false,
statusCode: 500,
statusCode: 400,
},
{
name: "Complete-multipart-upload-success",

View File

@@ -0,0 +1,61 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3log"
)
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, access := ctx.Locals("isRoot").(bool), ctx.Locals("access").(string)
path := ctx.Path()
pathParts := strings.Split(path, "/")
bucket := pathParts[1]
if path == "/" && ctx.Method() == http.MethodGet {
return ctx.Next()
}
if ctx.Method() == http.MethodPatch {
return ctx.Next()
}
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.IsAdmin(access, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
return ctx.Next()
}
//TODO: provide correct action names for the logger, after implementing DetectAction middleware
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("parsedAcl", parsedAcl)
return ctx.Next()
}
}

View File

@@ -54,6 +54,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.AclParser(be, l))
server.router.Init(app, be, iam, l, evs)

View File

@@ -19,12 +19,18 @@ import (
"errors"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
)
var (
bucketNameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
)
func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
metadata = make(map[string]string)
headers.VisitAll(func(key, value []byte) {
@@ -83,6 +89,22 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
}
}
func IsValidBucketName(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
}
// Checks to contain only digits, lowercase letters, dot, hyphen.
// Checks to start and end with only digits and lowercase letters.
if !bucketNameRegexp.MatchString(bucket) {
return false
}
// Checks not to be a valid IP address
if bucketNameIpRegexp.MatchString(bucket) {
return false
}
return true
}
func includeHeader(hdr string, signedHdrs []string) bool {
for _, shdr := range signedHdrs {
if strings.EqualFold(hdr, shdr) {

View File

@@ -153,3 +153,71 @@ func Test_includeHeader(t *testing.T) {
})
}
}
func TestIsValidBucketName(t *testing.T) {
type args struct {
bucket string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "IsValidBucketName-short-name",
args: args{
bucket: "a",
},
want: false,
},
{
name: "IsValidBucketName-start-with-hyphen",
args: args{
bucket: "-bucket",
},
want: false,
},
{
name: "IsValidBucketName-start-with-dot",
args: args{
bucket: ".bucket",
},
want: false,
},
{
name: "IsValidBucketName-contain-invalid-character",
args: args{
bucket: "my@bucket",
},
want: false,
},
{
name: "IsValidBucketName-end-with-hyphen",
args: args{
bucket: "bucket-",
},
want: false,
},
{
name: "IsValidBucketName-end-with-dot",
args: args{
bucket: "bucket.",
},
want: false,
},
{
name: "IsValidBucketName-valid-bucket-name",
args: args{
bucket: "my-bucket",
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidBucketName(tt.args.bucket); got != tt.want {
t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -106,7 +106,9 @@ const (
ErrNotImplemented
ErrPreconditionFailed
ErrInvalidObjectState
ErrInvalidRange
// Non-AWS errors
ErrExistingObjectIsDirectory
ErrObjectParentIsFile
)
@@ -374,6 +376,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The operation is not valid for the current state of the object",
HTTPStatusCode: http.StatusForbidden,
},
ErrInvalidRange: {
Code: "InvalidRange",
Description: "The requested range is not valid for the request. Try another range.",
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
},
ErrExistingObjectIsDirectory: {
Code: "ExistingObjectIsDirectory",
Description: "Existing Object is a directory.",

View File

@@ -27,6 +27,8 @@ import (
type AuditLogger interface {
Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
HangUp() error
Shutdown() error
}
type LogMeta struct {
@@ -36,7 +38,7 @@ type LogMeta struct {
}
type LogConfig struct {
IsFile bool
LogFile string
WebhookURL string
}
@@ -70,14 +72,14 @@ type LogFields struct {
}
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
if cfg.WebhookURL != "" && cfg.IsFile {
if cfg.WebhookURL != "" && cfg.LogFile != "" {
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
}
if cfg.WebhookURL != "" {
return InitWebhookLogger(cfg.WebhookURL)
}
if cfg.IsFile {
return InitFileLogger()
if cfg.LogFile != "" {
return InitFileLogger(cfg.LogFile)
}
return nil, nil

View File

@@ -16,9 +16,7 @@ package s3log
import (
"crypto/tls"
"errors"
"fmt"
"io/fs"
"os"
"strings"
"sync"
@@ -29,36 +27,43 @@ import (
)
const (
logFile = "access.log"
logFileMode = 0600
timeFormat = "02/January/2006:15:04:05 -0700"
)
// FileLogger is a local file audit log
type FileLogger struct {
LogFields
mu sync.Mutex
logfile string
f *os.File
gotErr bool
mu sync.Mutex
}
var _ AuditLogger = &FileLogger{}
func InitFileLogger() (AuditLogger, error) {
_, err := os.ReadFile(logFile)
if err != nil && errors.Is(err, fs.ErrNotExist) {
err := os.WriteFile(logFile, []byte{}, logFileMode)
if err != nil {
return nil, err
} else {
return nil, err
}
// InitFileLogger initializes audit logs to local file
func InitFileLogger(logname string) (AuditLogger, error) {
f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, fmt.Errorf("open log: %w", err)
}
return &FileLogger{}, nil
f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
return &FileLogger{logfile: logname, f: f}, nil
}
// Log sends log message to file logger
func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
f.mu.Lock()
defer f.mu.Unlock()
if f.gotErr {
return
}
lf := LogFields{}
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
@@ -68,8 +73,8 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
f.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
f.TLSVersion = getTLSVersionName(tlsConnState.Version)
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
@@ -88,117 +93,138 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
access = ctx.Locals("access").(string)
}
f.BucketOwner = meta.BucketOwner
f.Bucket = bucket
f.Time = time.Now()
f.RemoteIP = ctx.IP()
f.Requester = access
f.RequestID = genID()
f.Operation = meta.Action
f.Key = object
f.RequestURI = reqURI
f.HttpStatus = httpStatus
f.ErrorCode = errorCode
f.BytesSent = len(body)
f.ObjectSize = meta.ObjectSize
f.TotalTime = time.Since(startTime).Milliseconds()
f.TurnAroundTime = time.Since(startTime).Milliseconds()
f.Referer = ctx.Get("Referer")
f.UserAgent = ctx.Get("User-Agent")
f.VersionID = ctx.Query("versionId")
f.HostID = ctx.Get("X-Amz-Id-2")
f.SignatureVersion = "SigV4"
f.AuthenticationType = "AuthHeader"
f.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
f.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
f.AclRequired = "Yes"
lf.BucketOwner = meta.BucketOwner
lf.Bucket = bucket
lf.Time = time.Now()
lf.RemoteIP = ctx.IP()
lf.Requester = access
lf.RequestID = genID()
lf.Operation = meta.Action
lf.Key = object
lf.RequestURI = reqURI
lf.HttpStatus = httpStatus
lf.ErrorCode = errorCode
lf.BytesSent = len(body)
lf.ObjectSize = meta.ObjectSize
lf.TotalTime = time.Since(startTime).Milliseconds()
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
lf.Referer = ctx.Get("Referer")
lf.UserAgent = ctx.Get("User-Agent")
lf.VersionID = ctx.Query("versionId")
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"
f.writeLog()
f.writeLog(lf)
}
func (fl *FileLogger) writeLog() {
if fl.BucketOwner == "" {
fl.BucketOwner = "-"
func (f *FileLogger) writeLog(lf LogFields) {
if lf.BucketOwner == "" {
lf.BucketOwner = "-"
}
if fl.Bucket == "" {
fl.Bucket = "-"
if lf.Bucket == "" {
lf.Bucket = "-"
}
if fl.RemoteIP == "" {
fl.RemoteIP = "-"
if lf.RemoteIP == "" {
lf.RemoteIP = "-"
}
if fl.Requester == "" {
fl.Requester = "-"
if lf.Requester == "" {
lf.Requester = "-"
}
if fl.Operation == "" {
fl.Operation = "-"
if lf.Operation == "" {
lf.Operation = "-"
}
if fl.Key == "" {
fl.Key = "-"
if lf.Key == "" {
lf.Key = "-"
}
if fl.RequestURI == "" {
fl.RequestURI = "-"
if lf.RequestURI == "" {
lf.RequestURI = "-"
}
if fl.ErrorCode == "" {
fl.ErrorCode = "-"
if lf.ErrorCode == "" {
lf.ErrorCode = "-"
}
if fl.Referer == "" {
fl.Referer = "-"
if lf.Referer == "" {
lf.Referer = "-"
}
if fl.UserAgent == "" {
fl.UserAgent = "-"
if lf.UserAgent == "" {
lf.UserAgent = "-"
}
if fl.VersionID == "" {
fl.VersionID = "-"
if lf.VersionID == "" {
lf.VersionID = "-"
}
if fl.HostID == "" {
fl.HostID = "-"
if lf.HostID == "" {
lf.HostID = "-"
}
if fl.CipherSuite == "" {
fl.CipherSuite = "-"
if lf.CipherSuite == "" {
lf.CipherSuite = "-"
}
if fl.HostHeader == "" {
fl.HostHeader = "-"
if lf.HostHeader == "" {
lf.HostHeader = "-"
}
if fl.TLSVersion == "" {
fl.TLSVersion = "-"
if lf.TLSVersion == "" {
lf.TLSVersion = "-"
}
log := fmt.Sprintf("\n%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v",
fl.BucketOwner,
fl.Bucket,
fmt.Sprintf("[%v]", fl.Time.Format(timeFormat)),
fl.RemoteIP,
fl.Requester,
fl.RequestID,
fl.Operation,
fl.Key,
fl.RequestURI,
fl.HttpStatus,
fl.ErrorCode,
fl.BytesSent,
fl.ObjectSize,
fl.TotalTime,
fl.TurnAroundTime,
fl.Referer,
fl.UserAgent,
fl.VersionID,
fl.HostID,
fl.SignatureVersion,
fl.CipherSuite,
fl.AuthenticationType,
fl.HostHeader,
fl.TLSVersion,
fl.AccessPointARN,
fl.AclRequired,
log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
lf.BucketOwner,
lf.Bucket,
fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
lf.RemoteIP,
lf.Requester,
lf.RequestID,
lf.Operation,
lf.Key,
lf.RequestURI,
lf.HttpStatus,
lf.ErrorCode,
lf.BytesSent,
lf.ObjectSize,
lf.TotalTime,
lf.TurnAroundTime,
lf.Referer,
lf.UserAgent,
lf.VersionID,
lf.HostID,
lf.SignatureVersion,
lf.CipherSuite,
lf.AuthenticationType,
lf.HostHeader,
lf.TLSVersion,
lf.AccessPointARN,
lf.AclRequired,
)
file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, logFileMode)
_, err := f.f.WriteString(log)
if err != nil {
fmt.Printf("error opening the log file: %v", err.Error())
}
defer file.Close()
_, err = file.WriteString(log)
if err != nil {
fmt.Printf("error writing in log file: %v", err.Error())
fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
// TODO: do we need to terminate on log error?
// set err for now so that we don't spew errors
f.gotErr = true
}
}
// HangUp closes current logfile handle and opens a new one
// typically needed for log rotations
func (f *FileLogger) HangUp() error {
err := f.f.Close()
if err != nil {
return fmt.Errorf("close log: %w", err)
}
f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("open log: %w", err)
}
f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
return nil
}
// Shutdown closes logfile handle
func (f *FileLogger) Shutdown() error {
return f.f.Close()
}

View File

@@ -21,6 +21,7 @@ import (
"fmt"
"net"
"net/http"
"os"
"strings"
"sync"
"time"
@@ -29,14 +30,15 @@ import (
"github.com/versity/versitygw/s3err"
)
// WebhookLogger is a webhook URL audit log
type WebhookLogger struct {
LogFields
mu sync.Mutex
url string
}
var _ AuditLogger = &WebhookLogger{}
// InitWebhookLogger initializes audit logs to webhook URL
func InitWebhookLogger(url string) (AuditLogger, error) {
client := &http.Client{
Timeout: 3 * time.Second,
@@ -44,7 +46,7 @@ func InitWebhookLogger(url string) (AuditLogger, error) {
_, err := client.Post(url, "application/json", nil)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
return nil, fmt.Errorf("unreachable webhook url")
return nil, fmt.Errorf("unreachable webhook url: %w", err)
}
}
return &WebhookLogger{
@@ -52,10 +54,13 @@ func InitWebhookLogger(url string) (AuditLogger, error) {
}, nil
}
// Log sends log message to webhook
func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
wl.mu.Lock()
defer wl.mu.Unlock()
lf := LogFields{}
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
@@ -65,8 +70,8 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
wl.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
wl.TLSVersion = getTLSVersionName(tlsConnState.Version)
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
@@ -85,43 +90,43 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
access = ctx.Locals("access").(string)
}
wl.BucketOwner = meta.BucketOwner
wl.Bucket = bucket
wl.Time = time.Now()
wl.RemoteIP = ctx.IP()
wl.Requester = access
wl.RequestID = genID()
wl.Operation = meta.Action
wl.Key = object
wl.RequestURI = reqURI
wl.HttpStatus = httpStatus
wl.ErrorCode = errorCode
wl.BytesSent = len(body)
wl.ObjectSize = meta.ObjectSize
wl.TotalTime = time.Since(startTime).Milliseconds()
wl.TurnAroundTime = time.Since(startTime).Milliseconds()
wl.Referer = ctx.Get("Referer")
wl.UserAgent = ctx.Get("User-Agent")
wl.VersionID = ctx.Query("versionId")
wl.HostID = ctx.Get("X-Amz-Id-2")
wl.SignatureVersion = "SigV4"
wl.AuthenticationType = "AuthHeader"
wl.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
wl.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
wl.AclRequired = "Yes"
lf.BucketOwner = meta.BucketOwner
lf.Bucket = bucket
lf.Time = time.Now()
lf.RemoteIP = ctx.IP()
lf.Requester = access
lf.RequestID = genID()
lf.Operation = meta.Action
lf.Key = object
lf.RequestURI = reqURI
lf.HttpStatus = httpStatus
lf.ErrorCode = errorCode
lf.BytesSent = len(body)
lf.ObjectSize = meta.ObjectSize
lf.TotalTime = time.Since(startTime).Milliseconds()
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
lf.Referer = ctx.Get("Referer")
lf.UserAgent = ctx.Get("User-Agent")
lf.VersionID = ctx.Query("versionId")
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"
wl.sendLog()
wl.sendLog(lf)
}
func (wl *WebhookLogger) sendLog() {
jsonLog, err := json.Marshal(wl)
func (wl *WebhookLogger) sendLog(lf LogFields) {
jsonLog, err := json.Marshal(lf)
if err != nil {
fmt.Printf("\n failed to parse the log data: %v", err.Error())
fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
}
req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
if err != nil {
fmt.Println(err)
fmt.Fprintln(os.Stderr, err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
@@ -135,7 +140,17 @@ func makeRequest(req *http.Request) {
_, err := client.Do(req)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
fmt.Println("error sending the log to the specified url")
fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
}
}
}
// HangUp does nothing for webhooks
func (wl *WebhookLogger) HangUp() error {
return nil
}
// Shutdown does nothing for webhooks
func (wl *WebhookLogger) Shutdown() error {
return nil
}

View File

@@ -16,6 +16,8 @@ package s3response
import (
"encoding/xml"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// Part describes part metadata.
@@ -27,7 +29,7 @@ type Part struct {
}
// ListPartsResponse - s3 api list parts response.
type ListPartsResponse struct {
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
Bucket string
@@ -50,7 +52,7 @@ type ListPartsResponse struct {
}
// ListMultipartUploadsResponse - s3 api list multipart uploads response.
type ListMultipartUploadsResponse struct {
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
Bucket string
@@ -107,3 +109,28 @@ type TagSet struct {
type Tagging struct {
TagSet TagSet `xml:"TagSet"`
}
type DeleteObjects struct {
Objects []types.ObjectIdentifier `xml:"Object"`
}
type DeleteObjectsResult struct {
Deleted []types.DeletedObject
Errors []types.Error
}
type SelectObjectContentPayload struct {
Expression *string
ExpressionType types.ExpressionType
RequestProgress *types.RequestProgress
InputSerialization *types.InputSerialization
OutputSerialization *types.OutputSerialization
ScanRange *types.ScanRange
}
type SelectObjectContentResult struct {
Records *types.RecordsEvent
Stats *types.StatsEvent
Progress *types.ProgressEvent
Cont *string
End *string
}