From 87d61a1eb3fc7aef7ebcdf6083f4f14c3b1f3833 Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Fri, 14 Jul 2023 23:40:05 +0400 Subject: [PATCH] feat: Setup audit loggin with webhook url and root level access.log file. CLI enables either webhook or server access logs by providing the flags --- cmd/versitygw/main.go | 24 ++- s3api/controllers/base.go | 279 +++++++++++++++------------- s3api/controllers/base_test.go | 19 +- s3api/middlewares/authentication.go | 44 +++-- s3api/middlewares/md5.go | 6 +- s3api/router.go | 5 +- s3api/router_test.go | 2 +- s3api/server.go | 9 +- s3api/server_test.go | 2 +- s3log/audit-logger.go | 110 +++++++++++ s3log/file.go | 204 ++++++++++++++++++++ s3log/webhook.go | 141 ++++++++++++++ 12 files changed, 676 insertions(+), 169 deletions(-) create mode 100644 s3log/audit-logger.go create mode 100644 s3log/file.go create mode 100644 s3log/webhook.go diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 0f9bb06..4db32c6 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -27,6 +27,7 @@ import ( "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api" "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3log" ) var ( @@ -35,6 +36,8 @@ var ( rootUserSecret string region string certFile, keyFile string + logWebhookURL string + accessLog bool debug bool ) @@ -141,6 +144,16 @@ func initFlags() []cli.Flag { Usage: "enable debug output", Destination: &debug, }, + &cli.BoolFlag{ + Name: "access-log", + Usage: "enable server access logging in the root directory", + Destination: &accessLog, + }, + &cli.StringFlag{ + Name: "log-webhook-url", + Usage: "webhook url to send the audit logs", + Destination: &logWebhookURL, + }, } } @@ -182,10 +195,19 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error { return fmt.Errorf("setup internal iam service: %w", err) } + logger, err := s3log.InitLogger(&s3log.LogConfig{ + IsFile: accessLog, + WebhookURL: logWebhookURL, + }) + if err != nil { + return fmt.Errorf("setup logger: %w", err) + } + srv, err := s3api.New(app, be, middlewares.RootUserConfig{ Access: rootUserAccess, Secret: rootUserSecret, - }, port, region, iam, opts...) + Region: region, + }, port, region, iam, logger, opts...) if err != nil { return fmt.Errorf("init gateway: %v", err) } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 8f6e5ab..c84deac 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -17,7 +17,6 @@ package controllers import ( "bytes" "encoding/xml" - "errors" "fmt" "io" "log" @@ -33,25 +32,27 @@ import ( "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" "github.com/versity/versitygw/s3response" ) type S3ApiController struct { - be backend.Backend - iam auth.IAMService + be backend.Backend + iam auth.IAMService + logger s3log.AuditLogger } -func New(be backend.Backend, iam auth.IAMService) S3ApiController { - return S3ApiController{be: be, iam: iam} +func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) S3ApiController { + return S3ApiController{be: be, iam: iam, logger: logger} } func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error { access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) if err := auth.IsAdmin(access, isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListBucket"}}) } res, err := c.be.ListBuckets() - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListBucket"}}) } func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { @@ -70,22 +71,22 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } if ctx.Request().URI().QueryArgs().Has("tagging") { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}}) } tags, err := c.be.GetTags(bucket, key) if err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}}) } resp := s3response.Tagging{TagSet: s3response.TagSet{Tags: []s3response.Tag{}}} @@ -93,52 +94,58 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { resp.TagSet.Tags = append(resp.TagSet.Tags, s3response.Tag{Key: key, Value: val}) } - return SendXMLResponse(ctx, resp, nil) + return SendXMLResponse(ctx, resp, nil, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}}) } if uploadId != "" { if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &LogOptions{ + Logger: c.logger, + Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}, + }) } if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), &LogOptions{ + Logger: c.logger, + Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}, + }) } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}}) } if ctx.Request().URI().QueryArgs().Has("acl") { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.GetObjectAcl(bucket, key) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}}) } if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ",")) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}}) } ctx.Locals("logResBody", false) res, err := c.be.GetObject(bucket, key, acceptRange, ctx.Response().BodyWriter()) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}}) } if res == nil { - return SendResponse(ctx, fmt.Errorf("get object nil response")) + return SendResponse(ctx, fmt.Errorf("get object nil response"), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}}) } utils.SetMetaHeaders(ctx, res.Metadata) @@ -172,7 +179,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: string(res.StorageClass), }, }) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}}) } func getstring(s *string) string { @@ -193,45 +200,45 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } if ctx.Request().URI().QueryArgs().Has("acl") { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner}}) } res, err := auth.ParseACLOutput(data) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner}}) } if ctx.Request().URI().QueryArgs().Has("uploads") { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))}) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}}) } if ctx.QueryInt("list-type") == 2 { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjects", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.ListObjects(bucket, prefix, marker, delimiter, maxkeys) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjects", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { @@ -252,15 +259,29 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { if ctx.Request().URI().QueryArgs().Has("acl") { var input *s3.PutBucketAclInput + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl"}}) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl"}}) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil { + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) + } + if len(ctx.Body()) > 0 { if grants+acl != "" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } var accessControlPolicy auth.AccessControlPolicy err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) if err != nil { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } input = &s3.PutBucketAclInput{ @@ -271,10 +292,10 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { } if acl != "" { if acl != "private" && acl != "public-read" && acl != "public-read-write" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } if len(ctx.Body()) > 0 || grants != "" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } input = &s3.PutBucketAclInput{ @@ -296,31 +317,17 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { } } - data, err := c.be.GetBucketAcl(bucket) - if err != nil { - return SendResponse(ctx, err) - } - - parsedAcl, err := auth.ParseACL(data) - if err != nil { - return SendResponse(ctx, err) - } - - if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil { - return SendResponse(ctx, err) - } - updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } err = c.be.PutBucketAcl(bucket, updAcl) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}}) } err := c.be.PutBucket(bucket, access) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)}}) } func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { @@ -361,30 +368,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { keyStart = keyStart + "/" } - var contentLength int64 - if contentLengthStr != "" { - var err error - contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest)) - } - } - data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } if ctx.Request().URI().QueryArgs().Has("tagging") { var objTagging s3response.Tagging err := xml.Unmarshal(ctx.Body(), &objTagging) if err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}}) } tags := make(map[string]string, len(objTagging.TagSet.Tags)) @@ -394,18 +392,18 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}}) } err = c.be.SetTags(bucket, keyStart, tags) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}}) } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" { partNumber := ctx.QueryInt("partNumber", -1) if partNumber < 1 || partNumber > 10000 { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart)) + return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}}) } resp, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{ @@ -417,17 +415,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ExpectedBucketOwner: &bucketOwner, CopySourceRange: ©SrcRange, }) - return SendXMLResponse(ctx, resp, err) + return SendXMLResponse(ctx, resp, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}}) } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") { partNumber := ctx.QueryInt("partNumber", -1) if partNumber < 1 || partNumber > 10000 { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}}) + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}}) } body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body()))) @@ -435,7 +438,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { etag, err := c.be.PutObjectPart(bucket, keyStart, uploadId, partNumber, contentLength, body) ctx.Response().Header.Set("Etag", etag) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}}) } if ctx.Request().URI().QueryArgs().Has("acl") { @@ -443,13 +446,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { if len(ctx.Body()) > 0 { if grants+acl != "" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}}) } var accessControlPolicy auth.AccessControlPolicy err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) if err != nil { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}}) } input = &s3.PutObjectAclInput{ @@ -461,10 +464,10 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if acl != "" { if acl != "private" && acl != "public-read" && acl != "public-read-write" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}}) } if len(ctx.Body()) > 0 || grants != "" { - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest)) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}}) } input = &s3.PutObjectAclInput{ @@ -489,7 +492,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } err = c.be.PutObjectAcl(input) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}}) } if copySource != "" { @@ -499,17 +502,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:] if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CopyObject", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.CopyObject(srcBucket, strings.Join(srcObject, "/"), bucket, keyStart) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CopyObject", BucketOwner: parsedAcl.Owner}}) } metadata := utils.GetUserMetaData(&ctx.Request().Header) if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}}) + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}}) } ctx.Locals("logReqBody", false) @@ -521,7 +529,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Body: bytes.NewReader(ctx.Request().Body()), }) ctx.Response().Header.Set("ETag", etag) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { @@ -529,46 +537,46 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBuckets"}}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBuckets"}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}}) } err = c.be.DeleteBucket(bucket) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) var dObj types.Delete - if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest)) - } - data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects"}}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects"}}) + } + + if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil { + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}}) } err = c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj}) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { @@ -585,28 +593,28 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } if ctx.Request().URI().QueryArgs().Has("tagging") { if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner}}) } err = c.be.RemoveTags(bucket, key) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner}}) } if uploadId != "" { expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer") if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner}}) } err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{ @@ -616,15 +624,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { ExpectedBucketOwner: &expectedBucketOwner, RequestPayer: types.RequestPayer(requestPayer), }) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObject", BucketOwner: parsedAcl.Owner}}) } err = c.be.DeleteObject(bucket, key) - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObject", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { @@ -632,21 +640,21 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket"}}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket"}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket", BucketOwner: parsedAcl.Owner}}) } _, err = c.be.HeadBucket(bucket) // TODO: set bucket response headers - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket", BucketOwner: parsedAcl.Owner}}) } const ( @@ -663,24 +671,24 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject"}}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject"}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.HeadObject(bucket, key) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}}) } if res == nil { - return SendResponse(ctx, fmt.Errorf("head object nil response")) + return SendResponse(ctx, fmt.Errorf("head object nil response"), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}}) } utils.SetMetaHeaders(ctx, res.Metadata) @@ -719,7 +727,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { }, }) - return SendResponse(ctx, nil) + return SendResponse(ctx, nil, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}}) } func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { @@ -736,27 +744,27 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { data, err := c.be.GetBucketAcl(bucket) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } parsedAcl, err := auth.ParseACL(data) if err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger}) } var restoreRequest s3.RestoreObjectInput if ctx.Request().URI().QueryArgs().Has("restore") { - xmlErr := xml.Unmarshal(ctx.Body(), &restoreRequest) - if xmlErr != nil { - return errors.New("wrong api call") + err := xml.Unmarshal(ctx.Body(), &restoreRequest) + if err != nil { + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}}) } - err := c.be.RestoreObject(bucket, key, &restoreRequest) - return SendResponse(ctx, err) + err = c.be.RestoreObject(bucket, key, &restoreRequest) + return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}}) } if uploadId != "" { @@ -765,26 +773,34 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }{} if err := xml.Unmarshal(ctx.Body(), &data); err != nil { - return errors.New("wrong api call") + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err) + return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner}}) } res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key}) - return SendXMLResponse(ctx, res, err) + return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner}}) } -func SendResponse(ctx *fiber.Ctx, err error) error { +type LogOptions struct { + Logger s3log.AuditLogger + Meta s3log.LogMeta +} + +func SendResponse(ctx *fiber.Ctx, err error, l *LogOptions) error { + if l.Logger != nil { + l.Logger.Log(ctx, err, nil, l.Meta) + } if err != nil { serr, ok := err.(s3err.APIError) if ok { @@ -806,9 +822,11 @@ func SendResponse(ctx *fiber.Ctx, err error) error { return nil } -func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error { +func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *LogOptions) error { if err != nil { - fmt.Println(err) + if l.Logger != nil { + l.Logger.Log(ctx, err, nil, l.Meta) + } serr, ok := err.(s3err.APIError) if ok { ctx.Status(serr.HTTPStatusCode) @@ -835,6 +853,9 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error { } utils.LogCtxDetails(ctx, b) + if l.Logger != nil { + l.Logger.Log(ctx, nil, b, l.Meta) + } return ctx.Send(b) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index ba98481..cd080f5 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -31,6 +31,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" "github.com/versity/versitygw/s3response" ) @@ -49,8 +50,9 @@ func init() { func TestNew(t *testing.T) { type args struct { - be backend.Backend - iam auth.IAMService + be backend.Backend + iam auth.IAMService + logger s3log.AuditLogger } be := backend.BackendUnsupported{} @@ -74,7 +76,7 @@ func TestNew(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := New(tt.args.be, tt.args.iam); !reflect.DeepEqual(got, tt.want) { + if got := New(tt.args.be, tt.args.iam, tt.args.logger); !reflect.DeepEqual(got, tt.want) { t.Errorf("New() = %v, want %v", got, tt.want) } }) @@ -187,14 +189,15 @@ func TestS3ApiController_ListBuckets(t *testing.T) { } } +func getPtr(val string) *string { + return &val +} + func TestS3ApiController_GetActions(t *testing.T) { type args struct { req *http.Request } - getPtr := func(val string) *string { - return &val - } now := time.Now() app := fiber.New() @@ -1435,7 +1438,7 @@ func Test_XMLresponse(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err); (err != nil) != tt.wantErr { + if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err, &LogOptions{}); (err != nil) != tt.wantErr { t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) } @@ -1515,7 +1518,7 @@ func Test_response(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := SendResponse(tt.args.ctx, tt.args.err); (err != nil) != tt.wantErr { + if err := SendResponse(tt.args.ctx, tt.args.err, &LogOptions{}); (err != nil) != tt.wantErr { t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) } diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index bd9674b..25cdfdd 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -29,6 +29,7 @@ import ( "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" ) const ( @@ -38,15 +39,18 @@ const ( type RootUserConfig struct { Access string Secret string + Region string } -func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler { +func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler { acct := accounts{root: root, iam: iam} return func(ctx *fiber.Ctx) error { + ctx.Locals("region", root.Region) + ctx.Locals("startTime", time.Now()) authorization := ctx.Get("Authorization") if authorization == "" { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.LogOptions{Logger: logger}) } // Check the signature version @@ -56,48 +60,52 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, } if len(authParts) != 3 { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.LogOptions{Logger: logger}) } startParts := strings.Split(authParts[0], " ") if startParts[0] != "AWS4-HMAC-SHA256" { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.LogOptions{Logger: logger}) } credKv := strings.Split(startParts[1], "=") if len(credKv) != 2 { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger}) } creds := strings.Split(credKv[1], "/") if len(creds) < 4 { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger}) } + ctx.Locals("access", creds[0]) + ctx.Locals("isRoot", creds[0] == root.Access) + signHdrKv := strings.Split(authParts[1], "=") if len(signHdrKv) != 2 { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger}) } signedHdrs := strings.Split(signHdrKv[1], ";") account, err := acct.getAccount(creds[0]) if err == auth.ErrNoSuchUser { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.LogOptions{Logger: logger}) } if err != nil { - return controllers.SendResponse(ctx, err) + return controllers.SendResponse(ctx, err, &controllers.LogOptions{Logger: logger}) } + ctx.Locals("role", account.Role) // Check X-Amz-Date header date := ctx.Get("X-Amz-Date") if date == "" { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.LogOptions{Logger: logger}) } // Parse the date and check the date validity tdate, err := time.Parse(iso8601Format, date) if err != nil { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.LogOptions{Logger: logger}) } hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256") @@ -110,14 +118,14 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, // Compare the calculated hash with the hash provided if hashPayloadHeader != hexPayload { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.LogOptions{Logger: logger}) } } // Create a new http request instance from fasthttp request req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs) if err != nil { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.LogOptions{Logger: logger}) } signer := v4.NewSigner() @@ -132,24 +140,20 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, } }) if signErr != nil { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.LogOptions{Logger: logger}) } parts := strings.Split(req.Header.Get("Authorization"), " ") if len(parts) < 4 { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.LogOptions{Logger: logger}) } calculatedSign := strings.Split(parts[3], "=")[1] expectedSign := strings.Split(authParts[2], "=")[1] if expectedSign != calculatedSign { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.LogOptions{Logger: logger}) } - ctx.Locals("role", account.Role) - ctx.Locals("access", creds[0]) - ctx.Locals("isRoot", creds[0] == root.Access) - return ctx.Next() } } diff --git a/s3api/middlewares/md5.go b/s3api/middlewares/md5.go index e5b8233..7fcc1e9 100644 --- a/s3api/middlewares/md5.go +++ b/s3api/middlewares/md5.go @@ -21,9 +21,10 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" ) -func VerifyMD5Body() fiber.Handler { +func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { return func(ctx *fiber.Ctx) error { incomingSum := ctx.Get("Content-Md5") if incomingSum == "" { @@ -34,10 +35,9 @@ func VerifyMD5Body() fiber.Handler { calculatedSum := base64.StdEncoding.EncodeToString(sum[:]) if incomingSum != calculatedSum { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest)) + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.LogOptions{Logger: logger}) } return ctx.Next() - } } diff --git a/s3api/router.go b/s3api/router.go index 5e20e4b..bc50666 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -19,12 +19,13 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/controllers" + "github.com/versity/versitygw/s3log" ) type S3ApiRouter struct{} -func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) { - s3ApiController := controllers.New(be, iam) +func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) { + s3ApiController := controllers.New(be, iam, logger) adminController := controllers.AdminController{IAMService: iam} app.Patch("/create-user", adminController.CreateUser) diff --git a/s3api/router_test.go b/s3api/router_test.go index 5d80471..b2c9e0e 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam) + tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil) }) } } diff --git a/s3api/server.go b/s3api/server.go index b2c9f79..0b59fc5 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -22,6 +22,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3log" ) type S3ApiServer struct { @@ -33,7 +34,7 @@ type S3ApiServer struct { debug bool } -func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) { +func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...Option) (*S3ApiServer, error) { server := &S3ApiServer{ app: app, backend: be, @@ -50,10 +51,10 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po app.Use(middlewares.RequestLogger(server.debug)) // Authentication middlewares - app.Use(middlewares.VerifyV4Signature(root, iam, region, server.debug)) - app.Use(middlewares.VerifyMD5Body()) + app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug)) + app.Use(middlewares.VerifyMD5Body(l)) - server.router.Init(app, be, iam) + server.router.Init(app, be, iam, l) return server, nil } diff --git a/s3api/server_test.go b/s3api/server_test.go index 5313e41..9917cb7 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -64,7 +64,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root, - tt.args.port, "us-east-1", &auth.IAMServiceInternal{}) + tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil) if (err != nil) != tt.wantErr { t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/s3log/audit-logger.go b/s3log/audit-logger.go new file mode 100644 index 0000000..fc49d3b --- /dev/null +++ b/s3log/audit-logger.go @@ -0,0 +1,110 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3log + +import ( + "crypto/tls" + "encoding/hex" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/gofiber/fiber/v2" +) + +type AuditLogger interface { + Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) +} + +type LogMeta struct { + BucketOwner string + ObjectSize int64 + Action string +} + +type LogConfig struct { + IsFile bool + WebhookURL string +} + +type LogFields struct { + BucketOwner string + Bucket string + Time time.Time + RemoteIP string + Requester string + RequestID string + Operation string + Key string + RequestURI string + HttpStatus int + ErrorCode string + BytesSent int + ObjectSize int64 + TotalTime int64 + TurnAroundTime int64 + Referer string + UserAgent string + VersionID string + HostID string + SignatureVersion string + CipherSuite string + AuthenticationType string + HostHeader string + TLSVersion string + AccessPointARN string + AclRequired string +} + +func InitLogger(cfg *LogConfig) (AuditLogger, error) { + if cfg.WebhookURL != "" && cfg.IsFile { + return nil, fmt.Errorf("there should be specified one of the following: file, webhook") + } + if cfg.WebhookURL != "" { + return InitWebhookLogger(cfg.WebhookURL) + } + if cfg.IsFile { + return InitFileLogger() + } + + return nil, nil +} + +func genID() string { + src := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, 8) + + if _, err := src.Read(b); err != nil { + panic(err) + } + + return strings.ToUpper(hex.EncodeToString(b)) +} + +func getTLSVersionName(version uint16) string { + switch version { + case tls.VersionTLS10: + return "TLSv1.0" + case tls.VersionTLS11: + return "TLSv1.1" + case tls.VersionTLS12: + return "TLSv1.2" + case tls.VersionTLS13: + return "TLSv1.3" + default: + return "" + } +} diff --git a/s3log/file.go b/s3log/file.go new file mode 100644 index 0000000..d6cd09e --- /dev/null +++ b/s3log/file.go @@ -0,0 +1,204 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3log + +import ( + "crypto/tls" + "errors" + "fmt" + "io/fs" + "os" + "strings" + "sync" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/s3err" +) + +const ( + logFile = "access.log" + logFileMode = 0600 + timeFormat = "02/January/2006:15:04:05 -0700" +) + +type FileLogger struct { + LogFields + mu sync.Mutex +} + +var _ AuditLogger = &FileLogger{} + +func InitFileLogger() (AuditLogger, error) { + _, err := os.ReadFile(logFile) + if err != nil && errors.Is(err, fs.ErrNotExist) { + err := os.WriteFile(logFile, []byte{}, logFileMode) + if err != nil { + return nil, err + } else { + return nil, err + } + } + + return &FileLogger{}, nil +} + +func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) { + f.mu.Lock() + defer f.mu.Unlock() + + access := "-" + reqURI := ctx.Request().URI().String() + path := strings.Split(ctx.Path(), "/") + bucket, object := path[1], strings.Join(path[2:], "/") + errorCode := "" + httpStatus := 200 + startTime := ctx.Locals("startTime").(time.Time) + tlsConnState := ctx.Context().TLSConnectionState() + if tlsConnState != nil { + f.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite) + f.TLSVersion = getTLSVersionName(tlsConnState.Version) + } + + if err != nil { + serr, ok := err.(s3err.APIError) + if ok { + errorCode = serr.Code + httpStatus = serr.HTTPStatusCode + } else { + errorCode = err.Error() + httpStatus = 500 + } + } + + switch ctx.Locals("access").(type) { + case string: + access = ctx.Locals("access").(string) + } + + f.BucketOwner = meta.BucketOwner + f.Bucket = bucket + f.Time = time.Now() + f.RemoteIP = ctx.IP() + f.Requester = access + f.RequestID = genID() + f.Operation = meta.Action + f.Key = object + f.RequestURI = reqURI + f.HttpStatus = httpStatus + f.ErrorCode = errorCode + f.BytesSent = len(body) + f.ObjectSize = meta.ObjectSize + f.TotalTime = time.Since(startTime).Milliseconds() + f.TurnAroundTime = time.Since(startTime).Milliseconds() + f.Referer = ctx.Get("Referer") + f.UserAgent = ctx.Get("User-Agent") + f.VersionID = ctx.Query("versionId") + f.HostID = ctx.Get("X-Amz-Id-2") + f.SignatureVersion = "SigV4" + f.AuthenticationType = "AuthHeader" + f.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string)) + f.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")) + f.AclRequired = "Yes" + + f.writeLog() +} + +func (fl *FileLogger) writeLog() { + if fl.BucketOwner == "" { + fl.BucketOwner = "-" + } + if fl.Bucket == "" { + fl.Bucket = "-" + } + if fl.RemoteIP == "" { + fl.RemoteIP = "-" + } + if fl.Requester == "" { + fl.Requester = "-" + } + if fl.Operation == "" { + fl.Operation = "-" + } + if fl.Key == "" { + fl.Key = "-" + } + if fl.RequestURI == "" { + fl.RequestURI = "-" + } + if fl.ErrorCode == "" { + fl.ErrorCode = "-" + } + if fl.Referer == "" { + fl.Referer = "-" + } + if fl.UserAgent == "" { + fl.UserAgent = "-" + } + if fl.VersionID == "" { + fl.VersionID = "-" + } + if fl.HostID == "" { + fl.HostID = "-" + } + if fl.CipherSuite == "" { + fl.CipherSuite = "-" + } + if fl.HostHeader == "" { + fl.HostHeader = "-" + } + if fl.TLSVersion == "" { + fl.TLSVersion = "-" + } + + log := fmt.Sprintf("\n%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v", + fl.BucketOwner, + fl.Bucket, + fmt.Sprintf("[%v]", fl.Time.Format(timeFormat)), + fl.RemoteIP, + fl.Requester, + fl.RequestID, + fl.Operation, + fl.Key, + fl.RequestURI, + fl.HttpStatus, + fl.ErrorCode, + fl.BytesSent, + fl.ObjectSize, + fl.TotalTime, + fl.TurnAroundTime, + fl.Referer, + fl.UserAgent, + fl.VersionID, + fl.HostID, + fl.SignatureVersion, + fl.CipherSuite, + fl.AuthenticationType, + fl.HostHeader, + fl.TLSVersion, + fl.AccessPointARN, + fl.AclRequired, + ) + + file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, logFileMode) + if err != nil { + fmt.Printf("error opening the log file: %v", err.Error()) + } + defer file.Close() + _, err = file.WriteString(log) + if err != nil { + fmt.Printf("error writing in log file: %v", err.Error()) + } +} diff --git a/s3log/webhook.go b/s3log/webhook.go new file mode 100644 index 0000000..cd5a873 --- /dev/null +++ b/s3log/webhook.go @@ -0,0 +1,141 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3log + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/s3err" +) + +type WebhookLogger struct { + LogFields + mu sync.Mutex + url string +} + +var _ AuditLogger = &WebhookLogger{} + +func InitWebhookLogger(url string) (AuditLogger, error) { + client := &http.Client{ + Timeout: 3 * time.Second, + } + _, err := client.Post(url, "application/json", nil) + if err != nil { + if err, ok := err.(net.Error); ok && !err.Timeout() { + return nil, fmt.Errorf("unreachable webhook url") + } + } + return &WebhookLogger{ + url: url, + }, nil +} + +func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) { + wl.mu.Lock() + defer wl.mu.Unlock() + + access := "-" + reqURI := ctx.Request().URI().String() + path := strings.Split(ctx.Path(), "/") + bucket, object := path[1], strings.Join(path[2:], "/") + errorCode := "" + httpStatus := 200 + startTime := ctx.Locals("startTime").(time.Time) + tlsConnState := ctx.Context().TLSConnectionState() + if tlsConnState != nil { + wl.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite) + wl.TLSVersion = getTLSVersionName(tlsConnState.Version) + } + + if err != nil { + serr, ok := err.(s3err.APIError) + if ok { + errorCode = serr.Code + httpStatus = serr.HTTPStatusCode + } else { + errorCode = err.Error() + httpStatus = 500 + } + } + + switch ctx.Locals("access").(type) { + case string: + access = ctx.Locals("access").(string) + } + + wl.BucketOwner = meta.BucketOwner + wl.Bucket = bucket + wl.Time = time.Now() + wl.RemoteIP = ctx.IP() + wl.Requester = access + wl.RequestID = genID() + wl.Operation = meta.Action + wl.Key = object + wl.RequestURI = reqURI + wl.HttpStatus = httpStatus + wl.ErrorCode = errorCode + wl.BytesSent = len(body) + wl.ObjectSize = meta.ObjectSize + wl.TotalTime = time.Since(startTime).Milliseconds() + wl.TurnAroundTime = time.Since(startTime).Milliseconds() + wl.Referer = ctx.Get("Referer") + wl.UserAgent = ctx.Get("User-Agent") + wl.VersionID = ctx.Query("versionId") + wl.HostID = ctx.Get("X-Amz-Id-2") + wl.SignatureVersion = "SigV4" + wl.AuthenticationType = "AuthHeader" + wl.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string)) + wl.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")) + wl.AclRequired = "Yes" + + wl.sendLog() +} + +func (wl *WebhookLogger) sendLog() { + jsonLog, err := json.Marshal(wl) + if err != nil { + fmt.Printf("\n failed to parse the log data: %v", err.Error()) + } + + req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog)) + if err != nil { + fmt.Println(err) + } + req.Header.Set("Content-Type", "application/json; charset=utf-8") + + go makeRequest(req) +} + +func makeRequest(req *http.Request) { + client := &http.Client{ + Timeout: 1 * time.Second, + } + _, err := client.Do(req) + if err != nil { + if err, ok := err.(net.Error); ok && !err.Timeout() { + fmt.Println("error sending the log to the specified url") + } + } +}