diff --git a/.github/workflows/betteralign.yml b/.github/workflows/betteralign.yml new file mode 100644 index 0000000..29c35ff --- /dev/null +++ b/.github/workflows/betteralign.yml @@ -0,0 +1,23 @@ +name: betteralign +on: pull_request +jobs: + build: + name: Check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "stable" + id: go + + - name: Install betteralign + run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest + + - name: Run betteralign + run: betteralign -test_files ./... diff --git a/auth/acl.go b/auth/acl.go index 5a618c5..50bb40b 100644 --- a/auth/acl.go +++ b/auth/acl.go @@ -45,18 +45,18 @@ type GetBucketAclOutput struct { type PutBucketAclInput struct { Bucket *string - ACL types.BucketCannedACL AccessControlPolicy *AccessControlPolicy GrantFullControl *string GrantRead *string GrantReadACP *string GrantWrite *string GrantWriteACP *string + ACL types.BucketCannedACL } type AccessControlPolicy struct { - AccessControlList AccessControlList `xml:"AccessControlList"` Owner *types.Owner + AccessControlList AccessControlList `xml:"AccessControlList"` } type AccessControlList struct { @@ -352,13 +352,13 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error { } type AccessOptions struct { - Acl ACL AclPermission types.Permission - IsRoot bool - Acc Account Bucket string Object string Action Action + Acl ACL + Acc Account + IsRoot bool Readonly bool } diff --git a/auth/bucket_policy.go b/auth/bucket_policy.go index 0765d27..6710f8d 100644 --- a/auth/bucket_policy.go +++ b/auth/bucket_policy.go @@ -63,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri } type BucketPolicyItem struct { - Effect BucketPolicyAccessType `json:"Effect"` Principals Principals `json:"Principal"` Actions Actions `json:"Action"` Resources Resources `json:"Resource"` + Effect BucketPolicyAccessType `json:"Effect"` } func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error { diff --git a/auth/iam.go b/auth/iam.go index 0a5b950..456f4bc 100644 --- a/auth/iam.go +++ b/auth/iam.go @@ -93,7 +93,6 @@ var ( ) type Opts struct { - RootAccount Account Dir string LDAPServerURL string LDAPBindDN string @@ -119,11 +118,12 @@ type Opts struct { S3Region string S3Bucket string S3Endpoint string + RootAccount Account + CacheTTL int + CachePrune int S3DisableSSlVerfiy bool S3Debug bool CacheDisable bool - CacheTTL int - CachePrune int } func New(o *Opts) (IAMService, error) { diff --git a/auth/iam_cache.go b/auth/iam_cache.go index 2eea1ba..9bb95bb 100644 --- a/auth/iam_cache.go +++ b/auth/iam_cache.go @@ -36,14 +36,14 @@ type IAMCache struct { var _ IAMService = &IAMCache{} type item struct { - value Account exp time.Time + value Account } type icache struct { - sync.RWMutex - expire time.Duration items map[string]item + expire time.Duration + sync.RWMutex } func (i *icache) set(k string, v Account) { diff --git a/auth/iam_internal.go b/auth/iam_internal.go index a42c5bd..9ea6f4f 100644 --- a/auth/iam_internal.go +++ b/auth/iam_internal.go @@ -33,6 +33,8 @@ const ( // IAMServiceInternal manages the internal IAM service type IAMServiceInternal struct { + dir string + rootAcc Account // This mutex will help with racing updates to the IAM data // from multiple requests to this gateway instance, but // will not help with racing updates to multiple load balanced @@ -40,8 +42,6 @@ type IAMServiceInternal struct { // IAM service. All account updates should be sent to a single // gateway instance if possible. sync.RWMutex - dir string - rootAcc Account } // UpdateAcctFunc accepts the current data and returns the new data to be stored diff --git a/auth/iam_s3_object.go b/auth/iam_s3_object.go index 2cee171..6104218 100644 --- a/auth/iam_s3_object.go +++ b/auth/iam_s3_object.go @@ -42,6 +42,14 @@ import ( // coming from iAMConfig and iamFile in iam_internal. type IAMServiceS3 struct { + client *s3.Client + + access string + secret string + region string + bucket string + endpoint string + rootAcc Account // This mutex will help with racing updates to the IAM data // from multiple requests to this gateway instance, but // will not help with racing updates to multiple load balanced @@ -50,15 +58,8 @@ type IAMServiceS3 struct { // gateway instance if possible. sync.RWMutex - access string - secret string - region string - bucket string - endpoint string sslSkipVerify bool debug bool - rootAcc Account - client *s3.Client } var _ IAMService = &IAMServiceS3{} diff --git a/auth/object_lock.go b/auth/object_lock.go index 8abfa8d..486bc82 100644 --- a/auth/object_lock.go +++ b/auth/object_lock.go @@ -29,9 +29,9 @@ import ( ) type BucketLockConfig struct { - Enabled bool DefaultRetention *types.DefaultRetention CreatedAt *time.Time + Enabled bool } func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) { diff --git a/aws/signer/v4/v4.go b/aws/signer/v4/v4.go index 02f43b9..72a7463 100644 --- a/aws/signer/v4/v4.go +++ b/aws/signer/v4/v4.go @@ -85,6 +85,10 @@ type keyDerivator interface { // SignerOptions is the SigV4 Signer options. type SignerOptions struct { + + // The logger to send log messages to. + Logger logging.Logger + // Disables the Signer's moving HTTP header key/value pairs from the HTTP // request header to the request's query string. This is most commonly used // with pre-signed requests preventing headers from being added to the @@ -100,9 +104,6 @@ type SignerOptions struct { // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html DisableURIPathEscaping bool - // The logger to send log messages to. - Logger logging.Logger - // Enable logging of signed requests. // This will enable logging of the canonical request, the string to sign, and for presigning the subsequent // presigned URL. @@ -117,8 +118,8 @@ type SignerOptions struct { // Signer applies AWS v4 signing to given request. Use this to sign requests // that need to be signed with AWS V4 Signatures. type Signer struct { - options SignerOptions keyDerivator keyDerivator + options SignerOptions } // NewSigner returns a new SigV4 Signer @@ -133,17 +134,19 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer { } type httpSigner struct { + KeyDerivator keyDerivator Request *http.Request + Credentials aws.Credentials + Time v4Internal.SigningTime ServiceName string Region string - Time v4Internal.SigningTime - Credentials aws.Credentials - KeyDerivator keyDerivator - IsPreSign bool - SignedHdrs []string PayloadHash string + SignedHdrs []string + + IsPreSign bool + DisableHeaderHoisting bool DisableURIPathEscaping bool DisableSessionToken bool diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 87d3c7e..7fad69d 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -52,25 +52,25 @@ type Posix struct { rootfd *os.File rootdir string - // chownuid/gid enable chowning of files to the account uid/gid - // when objects are uploaded - chownuid bool - chowngid bool + // bucket versioning directory path + versioningDir string // euid/egid are the effective uid/gid of the running versitygw process // used to determine if chowning is needed euid int egid int + // newDirPerm is the permission to set on newly created directories + newDirPerm fs.FileMode + + // chownuid/gid enable chowning of files to the account uid/gid + // when objects are uploaded + chownuid bool + chowngid bool + // bucketlinks is a flag to enable symlinks to directories at the top // level gateway directory to be treated as buckets the same as directories bucketlinks bool - - // bucket versioning directory path - versioningDir string - - // newDirPerm is the permission to set on newly created directories - newDirPerm fs.FileMode } var _ backend.Backend = &Posix{} @@ -102,11 +102,11 @@ const ( ) type PosixOpts struct { + VersioningDir string + NewDirPerm fs.FileMode ChownUID bool ChownGID bool BucketLinks bool - VersioningDir string - NewDirPerm fs.FileMode } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { diff --git a/backend/posix/with_otmpfile.go b/backend/posix/with_otmpfile.go index ac0281e..3782c5a 100644 --- a/backend/posix/with_otmpfile.go +++ b/backend/posix/with_otmpfile.go @@ -38,12 +38,12 @@ type tmpfile struct { f *os.File bucket string objname string - isOTmp bool size int64 - needsChown bool uid int gid int newDirPerm fs.FileMode + isOTmp bool + needsChown bool } var ( diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 6f17c75..938d887 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -48,12 +48,21 @@ type ScoutfsOpts struct { } type ScoutFS struct { + + // bucket/object metadata storage facility + meta meta.MetadataStorer + *posix.Posix rootfd *os.File rootdir string - // bucket/object metadata storage facility - meta meta.MetadataStorer + // euid/egid are the effective uid/gid of the running versitygw process + // used to determine if chowning is needed + euid int + egid int + + // newDirPerm is the permissions to use when creating new directories + newDirPerm fs.FileMode // glaciermode enables the following behavior: // GET object: if file offline, return invalid object state @@ -70,14 +79,6 @@ type ScoutFS struct { // when objects are uploaded chownuid bool chowngid bool - - // euid/egid are the effective uid/gid of the running versitygw process - // used to determine if chowning is needed - euid int - egid int - - // newDirPerm is the permissions to use when creating new directories - newDirPerm fs.FileMode } var _ backend.Backend = &ScoutFS{} diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index 9757710..cb0a4e0 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -70,10 +70,10 @@ type tmpfile struct { bucket string objname string size int64 - needsChown bool uid int gid int newDirPerm fs.FileMode + needsChown bool } var ( diff --git a/backend/walk.go b/backend/walk.go index 6511ae6..13db459 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -27,10 +27,10 @@ import ( ) type WalkResults struct { + NextMarker string CommonPrefixes []types.CommonPrefix Objects []s3response.Object Truncated bool - NextMarker string } type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error) @@ -268,18 +268,18 @@ func contains(a string, strs []string) bool { } type WalkVersioningResults struct { + NextMarker string + NextVersionIdMarker string CommonPrefixes []types.CommonPrefix ObjectVersions []types.ObjectVersion DelMarkers []types.DeleteMarkerEntry Truncated bool - NextMarker string - NextVersionIdMarker string } type ObjVersionFuncResult struct { + NextVersionIdMarker string ObjectVersions []types.ObjectVersion DelMarkers []types.DeleteMarkerEntry - NextVersionIdMarker string Truncated bool } diff --git a/backend/walk_test.go b/backend/walk_test.go index f2f9a49..5f7785c 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -41,8 +41,8 @@ type testcase struct { prefix string delimiter string marker string - maxObjs int32 expected backend.WalkResults + maxObjs int32 } func getObj(path string, d fs.DirEntry) (s3response.Object, error) { diff --git a/metrics/metrics.go b/metrics/metrics.go index 22cad81..b1e5902 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -43,13 +43,14 @@ type Tag struct { // Manager is a manager of metrics plugins type Manager struct { - wg sync.WaitGroup ctx context.Context + addDataChan chan datapoint + config Config - publishers []publisher - addDataChan chan datapoint + publishers []publisher + wg sync.WaitGroup } type Config struct { @@ -220,6 +221,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) { type datapoint struct { key string - value int64 tags []Tag + value int64 } diff --git a/s3api/admin-server.go b/s3api/admin-server.go index ee132d7..5909a38 100644 --- a/s3api/admin-server.go +++ b/s3api/admin-server.go @@ -26,11 +26,11 @@ import ( ) type S3AdminServer struct { - app *fiber.App backend backend.Backend + app *fiber.App router *S3AdminRouter - port string cert *tls.Certificate + port string } func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer { diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go index 3382461..38f7c5c 100644 --- a/s3api/controllers/admin_test.go +++ b/s3api/controllers/admin_test.go @@ -64,11 +64,11 @@ func TestAdminController_CreateUser(t *testing.T) { ` tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Admin-create-user-malformed-body", @@ -149,11 +149,11 @@ func TestAdminController_UpdateUser(t *testing.T) { ` tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Admin-update-user-success", @@ -223,11 +223,11 @@ func TestAdminController_DeleteUser(t *testing.T) { app.Patch("/delete-user", adminController.DeleteUser) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Admin-delete-user-success", @@ -280,11 +280,11 @@ func TestAdminController_ListUsers(t *testing.T) { appSucc.Patch("/list-users", adminController.ListUsers) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Admin-list-users-iam-error", @@ -361,11 +361,11 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) { appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Change-bucket-owner-check-account-server-error", @@ -424,11 +424,11 @@ func TestAdminController_ListBuckets(t *testing.T) { app.Patch("/list-buckets", adminController.ListBuckets) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "List-buckets-success", diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 00e5ec5..d924bb5 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -3308,14 +3308,14 @@ type MetaOpts struct { Logger s3log.AuditLogger EvSender s3event.S3EventSender MetricsMng *metrics.Manager - ContentLength int64 - Action string - BucketOwner string - ObjectSize int64 - ObjectCount int64 - EventName s3event.EventType ObjectETag *string VersionId *string + Action string + BucketOwner string + EventName s3event.EventType + ContentLength int64 + ObjectSize int64 + ObjectCount int64 Status int } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 635af42..84e3fc0 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) { appErr.Get("/", s3ApiControllerErr.ListBuckets) tests := []struct { - name string args args app *fiber.App - wantErr bool + name string statusCode int + wantErr bool }{ { name: "List-bucket-method-not-allowed", @@ -233,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) { getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello") tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Get-actions-get-tags-success", @@ -435,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) { appError.Get("/:bucket", s3ApiControllerError.ListActions) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Get-bucket-tagging-non-existing-bucket", @@ -728,11 +728,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) { invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello") tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Put-bucket-tagging-invalid-body", @@ -1034,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) { invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello") tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Put-object-part-error-case", @@ -1243,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) { app.Delete("/:bucket", s3ApiController.DeleteBucket) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Delete-bucket-success", @@ -1334,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { request.Header.Set("Content-Type", "application/xml") tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Delete-Objects-success", @@ -1432,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) { appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Abort-multipart-upload-success", @@ -1541,11 +1541,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) { appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Head-bucket-success", @@ -1643,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) { appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Head-object-success", @@ -1723,11 +1723,11 @@ func TestS3ApiController_CreateActions(t *testing.T) { app.Post("/:bucket/:key/*", s3ApiController.CreateActions) tests := []struct { - name string app *fiber.App args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Restore-object-success", @@ -1808,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) { ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) tests := []struct { - name string args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Internal-server-error", @@ -1883,10 +1883,10 @@ func Test_response(t *testing.T) { ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) tests := []struct { - name string args args - wantErr bool + name string statusCode int + wantErr bool }{ { name: "Internal-server-error", diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 90ce219..78b5146 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -149,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au } type accounts struct { - root RootUserConfig iam auth.IAMService + root RootUserConfig } func (a accounts) getAccount(access string) (auth.Account, error) { diff --git a/s3api/router_test.go b/s3api/router_test.go index 58f01b3..6dd25f4 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) { iam auth.IAMService } tests := []struct { - name string - sa *S3ApiRouter args args + sa *S3ApiRouter + name string }{ { name: "Initialize S3 api router", diff --git a/s3api/server.go b/s3api/server.go index cbf960b..a0a6469 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -29,15 +29,15 @@ import ( ) type S3ApiServer struct { - app *fiber.App backend backend.Backend + app *fiber.App router *S3ApiRouter - port string cert *tls.Certificate + port string + health string quiet bool debug bool readonly bool - health string } func New( diff --git a/s3api/server_test.go b/s3api/server_test.go index 64aa4fc..3e12c8e 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -39,9 +39,9 @@ func TestNew(t *testing.T) { port := ":7070" tests := []struct { - name string - args args wantS3ApiServer *S3ApiServer + args args + name string wantErr bool }{ { @@ -78,8 +78,8 @@ func TestNew(t *testing.T) { func TestS3ApiServer_Serve(t *testing.T) { tests := []struct { - name string sa *S3ApiServer + name string wantErr bool }{ { diff --git a/s3api/utils/auth-reader.go b/s3api/utils/auth-reader.go index 1694c16..21705b2 100644 --- a/s3api/utils/auth-reader.go +++ b/s3api/utils/auth-reader.go @@ -41,10 +41,10 @@ const ( // the data is completely read. type AuthReader struct { ctx *fiber.Ctx + r *HashReader auth AuthData secret string size int - r *HashReader debug bool } diff --git a/s3api/utils/chunk-reader.go b/s3api/utils/chunk-reader.go index 918d143..01038cd 100644 --- a/s3api/utils/chunk-reader.go +++ b/s3api/utils/chunk-reader.go @@ -48,15 +48,15 @@ const ( // object data stream type ChunkReader struct { r io.Reader - signingKey []byte + chunkHash hash.Hash prevSig string parsedSig string + strToSignPrefix string + signingKey []byte + stash []byte currentChunkSize int64 chunkDataLeft int64 trailerExpected int - stash []byte - chunkHash hash.Hash - strToSignPrefix string skipcheck bool } diff --git a/s3api/utils/presign-auth-reader.go b/s3api/utils/presign-auth-reader.go index fd2938b..7a2b297 100644 --- a/s3api/utils/presign-auth-reader.go +++ b/s3api/utils/presign-auth-reader.go @@ -41,10 +41,10 @@ const ( // data requests where the data size is not known until // the data is completely read. type PresignedAuthReader struct { + r io.Reader ctx *fiber.Ctx auth AuthData secret string - r io.Reader debug bool } diff --git a/s3api/utils/presign-auth-reader_test.go b/s3api/utils/presign-auth-reader_test.go index f4767b4..4e507a4 100644 --- a/s3api/utils/presign-auth-reader_test.go +++ b/s3api/utils/presign-auth-reader_test.go @@ -23,13 +23,13 @@ import ( func Test_validateExpiration(t *testing.T) { type args struct { - str string date time.Time + str string } tests := []struct { - name string args args err error + name string }{ { name: "empty-expiration", diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 1f76759..4eaa2e8 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -49,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) { request2.Header.Add("X-Amz-Mfa", "Some valid Mfa") tests := []struct { - name string args args want *http.Request - wantErr bool + name string hdrs []string + wantErr bool }{ { name: "Success-response", @@ -101,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) { req := ctx.Request() tests := []struct { - name string args args wantMetadata map[string]string + name string }{ { name: "Success-empty-response", diff --git a/s3event/event.go b/s3event/event.go index e172836..4334208 100644 --- a/s3event/event.go +++ b/s3event/event.go @@ -30,11 +30,11 @@ type S3EventSender interface { } type EventMeta struct { + ObjectETag *string + VersionId *string BucketOwner string EventName EventType ObjectSize int64 - ObjectETag *string - VersionId *string } type EventSchema struct { @@ -42,6 +42,8 @@ type EventSchema struct { } type EventRecord struct { + ResponseElements EventResponseElements `json:"responseElements"` + GlacierEventData EventGlacierData `json:"glacierEventData"` EventVersion string `json:"eventVersion"` EventSource string `json:"eventSource"` AwsRegion string `json:"awsRegion"` @@ -49,9 +51,7 @@ type EventRecord struct { EventName EventType `json:"eventName"` UserIdentity EventUserIdentity `json:"userIdentity"` RequestParameters EventRequestParams `json:"requestParameters"` - ResponseElements EventResponseElements `json:"responseElements"` S3 EventS3Data `json:"s3"` - GlacierEventData EventGlacierData `json:"glacierEventData"` } type EventUserIdentity struct { @@ -99,11 +99,11 @@ type EventS3BucketData struct { } type EventObjectData struct { - Key string `json:"key"` - Size int64 `json:"size"` ETag *string `json:"eTag"` VersionId *string `json:"versionId"` + Key string `json:"key"` Sequencer string `json:"sequencer"` + Size int64 `json:"size"` } type EventConfig struct { diff --git a/s3event/kafka.go b/s3event/kafka.go index 257c1c4..3cab7e7 100644 --- a/s3event/kafka.go +++ b/s3event/kafka.go @@ -31,9 +31,9 @@ import ( var sequencer = 0 type Kafka struct { - key string writer *kafka.Writer filter EventFilter + key string mu sync.Mutex } diff --git a/s3event/nats.go b/s3event/nats.go index 189105a..73b5dc6 100644 --- a/s3event/nats.go +++ b/s3event/nats.go @@ -27,10 +27,10 @@ import ( ) type NatsEventSender struct { - topic string client *nats.Conn - mu sync.Mutex filter EventFilter + topic string + mu sync.Mutex } func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) { diff --git a/s3event/webhook.go b/s3event/webhook.go index bce571c..40f660c 100644 --- a/s3event/webhook.go +++ b/s3event/webhook.go @@ -30,9 +30,9 @@ import ( ) type Webhook struct { - url string client *http.Client filter EventFilter + url string mu sync.Mutex } diff --git a/s3log/audit-logger.go b/s3log/audit-logger.go index 24796a4..263b7e8 100644 --- a/s3log/audit-logger.go +++ b/s3log/audit-logger.go @@ -33,8 +33,8 @@ type AuditLogger interface { type LogMeta struct { BucketOwner string - ObjectSize int64 Action string + ObjectSize int64 HttpStatus int } @@ -45,21 +45,16 @@ type LogConfig struct { } type LogFields struct { + Time time.Time BucketOwner string Bucket string - Time time.Time RemoteIP string Requester string RequestID string Operation string Key string RequestURI string - HttpStatus int ErrorCode string - BytesSent int - ObjectSize int64 - TotalTime int64 - TurnAroundTime int64 Referer string UserAgent string VersionID string @@ -71,6 +66,11 @@ type LogFields struct { TLSVersion string AccessPointARN string AclRequired string + HttpStatus int + BytesSent int + ObjectSize int64 + TotalTime int64 + TurnAroundTime int64 } type AdminLogFields struct { @@ -80,17 +80,17 @@ type AdminLogFields struct { RequestID string Operation string RequestURI string - HttpStatus int ErrorCode string - BytesSent int - TotalTime int64 - TurnAroundTime int64 Referer string UserAgent string SignatureVersion string CipherSuite string AuthenticationType string TLSVersion string + HttpStatus int + BytesSent int + TotalTime int64 + TurnAroundTime int64 } type Loggers struct { diff --git a/s3log/file.go b/s3log/file.go index 9d36fff..1d8ea68 100644 --- a/s3log/file.go +++ b/s3log/file.go @@ -34,10 +34,10 @@ const ( // FileLogger is a local file audit log type FileLogger struct { - logfile string f *os.File - gotErr bool + logfile string mu sync.Mutex + gotErr bool } var _ AuditLogger = &FileLogger{} diff --git a/s3log/webhook.go b/s3log/webhook.go index 1fe7cb6..a15d6da 100644 --- a/s3log/webhook.go +++ b/s3log/webhook.go @@ -33,8 +33,8 @@ import ( // WebhookLogger is a webhook URL audit log type WebhookLogger struct { - mu sync.Mutex url string + mu sync.Mutex } var _ AuditLogger = &WebhookLogger{} diff --git a/s3response/s3response.go b/s3response/s3response.go index 310d3ef..31674ef 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -35,17 +35,17 @@ type PutObjectOutput struct { // Part describes part metadata. type Part struct { - PartNumber int LastModified time.Time ETag string + PartNumber int Size int64 } func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias Part aux := &struct { - LastModified string `xml:"LastModified"` *Alias + LastModified string `xml:"LastModified"` }{ Alias: (*Alias)(&p), } @@ -59,23 +59,23 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` + Initiator Initiator + Owner Owner + Bucket string Key string UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - // The class of storage used to store the object. StorageClass types.StorageClass + // List of parts. + Parts []Part `xml:"Part"` + PartNumberMarker int NextPartNumberMarker int MaxParts int IsTruncated bool - - // List of parts. - Parts []Part `xml:"Part"` } type ObjectAttributes string @@ -97,23 +97,23 @@ func (o ObjectAttributes) IsValid() bool { } type GetObjectAttributesResponse struct { - ETag *string - ObjectSize *int64 - StorageClass types.StorageClass `xml:",omitempty"` - ObjectParts *ObjectParts + ETag *string + ObjectSize *int64 + ObjectParts *ObjectParts // Not included in the response body VersionId *string LastModified *time.Time DeleteMarker *bool + StorageClass types.StorageClass `xml:",omitempty"` } type ObjectParts struct { + Parts []types.ObjectPart `xml:"Part"` PartNumberMarker int NextPartNumberMarker int MaxParts int IsTruncated bool - Parts []types.ObjectPart `xml:"Part"` } // ListMultipartUploadsResponse - s3 api list multipart uploads response. @@ -128,18 +128,17 @@ type ListMultipartUploadsResult struct { Delimiter string Prefix string EncodingType string `xml:"EncodingType,omitempty"` - MaxUploads int - IsTruncated bool // List of pending uploads. Uploads []Upload `xml:"Upload"` // Delimed common prefixes. CommonPrefixes []CommonPrefix + MaxUploads int + IsTruncated bool } type ListObjectsResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` Name *string Prefix *string Marker *string @@ -147,13 +146,13 @@ type ListObjectsResult struct { MaxKeys *int32 Delimiter *string IsTruncated *bool + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + EncodingType types.EncodingType Contents []Object CommonPrefixes []types.CommonPrefix - EncodingType types.EncodingType } type ListObjectsV2Result struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` Name *string Prefix *string StartAfter *string @@ -163,9 +162,10 @@ type ListObjectsV2Result struct { MaxKeys *int32 Delimiter *string IsTruncated *bool + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + EncodingType types.EncodingType Contents []Object CommonPrefixes []types.CommonPrefix - EncodingType types.EncodingType } type Object struct { @@ -197,19 +197,19 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error { // Upload describes in progress multipart upload type Upload struct { - Key string - UploadID string `xml:"UploadId"` + Initiated time.Time Initiator Initiator Owner Owner + Key string + UploadID string `xml:"UploadId"` StorageClass types.StorageClass - Initiated time.Time } func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias Upload aux := &struct { - Initiated string `xml:"Initiated"` *Alias + Initiated string `xml:"Initiated"` }{ Alias: (*Alias)(&u), } @@ -261,11 +261,11 @@ type DeleteResult struct { } type SelectObjectContentPayload struct { Expression *string - ExpressionType types.ExpressionType RequestProgress *types.RequestProgress InputSerialization *types.InputSerialization OutputSerialization *types.OutputSerialization ScanRange *types.ScanRange + ExpressionType types.ExpressionType } type SelectObjectContentResult struct { @@ -283,30 +283,30 @@ type Bucket struct { type ListBucketsInput struct { Owner string - IsAdmin bool ContinuationToken string Prefix string MaxBuckets int32 + IsAdmin bool } type ListAllMyBucketsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` Owner CanonicalUser - Buckets ListAllMyBucketsList ContinuationToken string `xml:"ContinuationToken,omitempty"` Prefix string `xml:"Prefix,omitempty"` + Buckets ListAllMyBucketsList } type ListAllMyBucketsEntry struct { - Name string CreationDate time.Time + Name string } func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias ListAllMyBucketsEntry aux := &struct { - CreationDate string `xml:"CreationDate"` *Alias + CreationDate string `xml:"CreationDate"` }{ Alias: (*Alias)(&r), } @@ -335,8 +335,8 @@ type CopyObjectResult struct { func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias CopyObjectResult aux := &struct { - LastModified string `xml:"LastModified"` *Alias + LastModified string `xml:"LastModified"` }{ Alias: (*Alias)(&r), } @@ -404,15 +404,15 @@ type ListVersionsResult struct { } type GetBucketVersioningOutput struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"` MFADelete *types.MFADeleteStatus Status *types.BucketVersioningStatus + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"` } type PutObjectRetentionInput struct { + RetainUntilDate AmzDate XMLName xml.Name `xml:"Retention"` Mode types.ObjectLockRetentionMode - RetainUntilDate AmzDate } type AmzDate struct { diff --git a/s3select/message-handler.go b/s3select/message-handler.go index 0000025..9ceeb9b 100644 --- a/s3select/message-handler.go +++ b/s3select/message-handler.go @@ -204,7 +204,6 @@ func genErrorMessage(errorCode, errorMessage string) []byte { type GetProgress func() (bytesScanned int64, bytesProcessed int64) type MessageHandler struct { - sync.Mutex ctx context.Context cancel context.CancelFunc writer *bufio.Writer @@ -213,6 +212,7 @@ type MessageHandler struct { stopCh chan bool resetCh chan bool bytesReturned int64 + sync.Mutex } // NewMessageHandler creates a new MessageHandler instance and starts the event streaming diff --git a/tests/integration/bench.go b/tests/integration/bench.go index 442d16f..810a1a3 100644 --- a/tests/integration/bench.go +++ b/tests/integration/bench.go @@ -26,9 +26,9 @@ import ( ) type prefResult struct { + err error elapsed time.Duration size int64 - err error } func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error { diff --git a/tests/integration/data-io.go b/tests/integration/data-io.go index 5bdd774..046537f 100644 --- a/tests/integration/data-io.go +++ b/tests/integration/data-io.go @@ -22,9 +22,9 @@ import ( ) type RReader struct { + hash hash.Hash buf []byte dataleft int - hash hash.Hash } func NewDataReader(totalsize, bufsize int) *RReader { diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index 5e2cb84..c12c394 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -35,10 +35,10 @@ type S3Conf struct { awsSecret string awsRegion string endpoint string - checksumDisable bool - pathStyle bool PartSize int64 Concurrency int + checksumDisable bool + pathStyle bool debug bool versioningEnabled bool azureTests bool diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 71c42e2..9078600 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -168,9 +168,9 @@ func teardown(s *S3Conf, bucket string) error { } type setupCfg struct { - LockEnabled bool VersioningStatus types.BucketVersioningStatus Ownership types.ObjectOwnership + LockEnabled bool } type setupOpt func(*setupCfg) @@ -214,12 +214,12 @@ func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, } type authConfig struct { + date time.Time testName string path string method string - body []byte service string - date time.Time + body []byte } func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error { @@ -434,9 +434,9 @@ func contains(s []string, e string) bool { } type putObjectOutput struct { - csum [32]byte - data []byte res *s3.PutObjectOutput + data []byte + csum [32]byte } func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {