mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-05 03:24:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #129 from versity/audit-logging-setup

Browse Source 
feat: Set up audit logging basic structure, set up webhook logger, bu…
This commit is contained in:
Ben McClelland
2023-07-14 12:50:32 -07:00
committed by GitHub
parent 5aeb96f138 87d61a1eb3
commit df7f01f7e2
12 changed files with 676 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/versitygw/main.go
View File

@@ -27,6 +27,7 @@ import (
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3log"
)
var (
@@ -35,6 +36,8 @@ var (
rootUserSecret string
region string
certFile, keyFile string
logWebhookURL string
accessLog bool
debug bool
)
@@ -141,6 +144,16 @@ func initFlags() []cli.Flag {
Usage: "enable debug output",
Destination: &debug,
},
&cli.BoolFlag{
Name: "access-log",
Usage: "enable server access logging in the root directory",
Destination: &accessLog,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
Destination: &logWebhookURL,
},
}
}
@@ -182,10 +195,19 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
return fmt.Errorf("setup internal iam service: %w", err)
}
logger, err := s3log.InitLogger(&s3log.LogConfig{
IsFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
return fmt.Errorf("setup logger: %w", err)
}
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
Access: rootUserAccess,
Secret: rootUserSecret,
}, port, region, iam, opts...)
Region: region,
}, port, region, iam, logger, opts...)
if err != nil {
return fmt.Errorf("init gateway: %v", err)
}

279
s3api/controllers/base.go
View File

@@ -17,7 +17,6 @@ package controllers
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"log"
@@ -33,25 +32,27 @@ import (
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
"github.com/versity/versitygw/s3response"
)
type S3ApiController struct {
be backend.Backend
iam auth.IAMService
be backend.Backend
iam auth.IAMService
logger s3log.AuditLogger
}
func New(be backend.Backend, iam auth.IAMService) S3ApiController {
return S3ApiController{be: be, iam: iam}
func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) S3ApiController {
return S3ApiController{be: be, iam: iam, logger: logger}
}
func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
if err := auth.IsAdmin(access, isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListBucket"}})
}
res, err := c.be.ListBuckets()
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListBucket"}})
}
func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
@@ -70,22 +71,22 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}})
}
tags, err := c.be.GetTags(bucket, key)
if err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}})
}
resp := s3response.Tagging{TagSet: s3response.TagSet{Tags: []s3response.Tag{}}}
@@ -93,52 +94,58 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
resp.TagSet.Tags = append(resp.TagSet.Tags, s3response.Tag{Key: key, Value: val})
}
return SendXMLResponse(ctx, resp, nil)
return SendXMLResponse(ctx, resp, nil, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner}})
}
if uploadId != "" {
if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &LogOptions{
Logger: c.logger,
Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner},
})
}
if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), &LogOptions{
Logger: c.logger,
Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner},
})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.GetObjectAcl(bucket, key)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}})
}
if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ","))
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}})
}
ctx.Locals("logResBody", false)
res, err := c.be.GetObject(bucket, key, acceptRange, ctx.Response().BodyWriter())
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}})
}
if res == nil {
return SendResponse(ctx, fmt.Errorf("get object nil response"))
return SendResponse(ctx, fmt.Errorf("get object nil response"), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}})
}
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -172,7 +179,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
},
})
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetObject", BucketOwner: parsedAcl.Owner}})
}
func getstring(s *string) string {
@@ -193,45 +200,45 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner}})
}
res, err := auth.ParseACLOutput(data)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner}})
}
if ctx.Request().URI().QueryArgs().Has("uploads") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}})
}
if ctx.QueryInt("list-type") == 2 {
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjects", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "ListObjects", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
@@ -252,15 +259,29 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
if ctx.Request().URI().QueryArgs().Has("acl") {
var input *s3.PutBucketAclInput
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl"}})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl"}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
if len(ctx.Body()) > 0 {
if grants+acl != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
var accessControlPolicy auth.AccessControlPolicy
err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
input = &s3.PutBucketAclInput{
@@ -271,10 +292,10 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
}
if acl != "" {
if acl != "private" && acl != "public-read" && acl != "public-read-write" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
if len(ctx.Body()) > 0 || grants != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
input = &s3.PutBucketAclInput{
@@ -296,31 +317,17 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
}
}
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
return SendResponse(ctx, err)
}
updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
err = c.be.PutBucketAcl(bucket, updAcl)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}})
}
err := c.be.PutBucket(bucket, access)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)}})
}
func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
@@ -361,30 +368,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
keyStart = keyStart + "/"
}
var contentLength int64
if contentLengthStr != "" {
var err error
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
}
}
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
var objTagging s3response.Tagging
err := xml.Unmarshal(ctx.Body(), &objTagging)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}})
}
tags := make(map[string]string, len(objTagging.TagSet.Tags))
@@ -394,18 +392,18 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}})
}
err = c.be.SetTags(bucket, keyStart, tags)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner}})
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" {
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}})
}
resp, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{
@@ -417,17 +415,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ExpectedBucketOwner: &bucketOwner,
CopySourceRange: &copySrcRange,
})
return SendXMLResponse(ctx, resp, err)
return SendXMLResponse(ctx, resp, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}})
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") {
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}})
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}})
}
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
@@ -435,7 +438,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
etag, err := c.be.PutObjectPart(bucket, keyStart, uploadId,
partNumber, contentLength, body)
ctx.Response().Header.Set("Etag", etag)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
@@ -443,13 +446,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
if len(ctx.Body()) > 0 {
if grants+acl != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}})
}
var accessControlPolicy auth.AccessControlPolicy
err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}})
}
input = &s3.PutObjectAclInput{
@@ -461,10 +464,10 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if acl != "" {
if acl != "private" && acl != "public-read" && acl != "public-read-write" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}})
}
if len(ctx.Body()) > 0 || grants != "" {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidRequest))
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}})
}
input = &s3.PutObjectAclInput{
@@ -489,7 +492,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
err = c.be.PutObjectAcl(input)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObjectAcl", BucketOwner: parsedAcl.Owner}})
}
if copySource != "" {
@@ -499,17 +502,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:]
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CopyObject", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.CopyObject(srcBucket, strings.Join(srcObject, "/"), bucket, keyStart)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CopyObject", BucketOwner: parsedAcl.Owner}})
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}})
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}})
}
ctx.Locals("logReqBody", false)
@@ -521,7 +529,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Body: bytes.NewReader(ctx.Request().Body()),
})
ctx.Response().Header.Set("ETag", etag)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "PutObject", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
@@ -529,46 +537,46 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBuckets"}})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBuckets"}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}})
}
err = c.be.DeleteBucket(bucket)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
var dObj types.Delete
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
}
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects"}})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects"}})
}
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}})
}
err = c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj})
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
@@ -585,28 +593,28 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner}})
}
err = c.be.RemoveTags(bucket, key)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner}})
}
if uploadId != "" {
expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
@@ -616,15 +624,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
ExpectedBucketOwner: &expectedBucketOwner,
RequestPayer: types.RequestPayer(requestPayer),
})
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObject", BucketOwner: parsedAcl.Owner}})
}
err = c.be.DeleteObject(bucket, key)
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "DeleteObject", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
@@ -632,21 +640,21 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket"}})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket"}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket", BucketOwner: parsedAcl.Owner}})
}
_, err = c.be.HeadBucket(bucket)
// TODO: set bucket response headers
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadBucket", BucketOwner: parsedAcl.Owner}})
}
const (
@@ -663,24 +671,24 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject"}})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject"}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.HeadObject(bucket, key)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}})
}
if res == nil {
return SendResponse(ctx, fmt.Errorf("head object nil response"))
return SendResponse(ctx, fmt.Errorf("head object nil response"), &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}})
}
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -719,7 +727,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
},
})
return SendResponse(ctx, nil)
return SendResponse(ctx, nil, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "HeadObject", BucketOwner: parsedAcl.Owner}})
}
func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
@@ -736,27 +744,27 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
data, err := c.be.GetBucketAcl(bucket)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger})
}
var restoreRequest s3.RestoreObjectInput
if ctx.Request().URI().QueryArgs().Has("restore") {
xmlErr := xml.Unmarshal(ctx.Body(), &restoreRequest)
if xmlErr != nil {
return errors.New("wrong api call")
err := xml.Unmarshal(ctx.Body(), &restoreRequest)
if err != nil {
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendResponse(ctx, err)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}})
}
err := c.be.RestoreObject(bucket, key, &restoreRequest)
return SendResponse(ctx, err)
err = c.be.RestoreObject(bucket, key, &restoreRequest)
return SendResponse(ctx, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "RestoreObject", BucketOwner: parsedAcl.Owner}})
}
if uploadId != "" {
@@ -765,26 +773,34 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
}{}
if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
return errors.New("wrong api call")
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts)
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err)
return SendXMLResponse(ctx, nil, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
return SendXMLResponse(ctx, res, err)
return SendXMLResponse(ctx, res, err, &LogOptions{Logger: c.logger, Meta: s3log.LogMeta{Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner}})
}
func SendResponse(ctx *fiber.Ctx, err error) error {
type LogOptions struct {
Logger s3log.AuditLogger
Meta s3log.LogMeta
}
func SendResponse(ctx *fiber.Ctx, err error, l *LogOptions) error {
if l.Logger != nil {
l.Logger.Log(ctx, err, nil, l.Meta)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
@@ -806,9 +822,11 @@ func SendResponse(ctx *fiber.Ctx, err error) error {
return nil
}
func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error {
func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *LogOptions) error {
if err != nil {
fmt.Println(err)
if l.Logger != nil {
l.Logger.Log(ctx, err, nil, l.Meta)
}
serr, ok := err.(s3err.APIError)
if ok {
ctx.Status(serr.HTTPStatusCode)
@@ -835,6 +853,9 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error {
}
utils.LogCtxDetails(ctx, b)
if l.Logger != nil {
l.Logger.Log(ctx, nil, b, l.Meta)
}
return ctx.Send(b)
}

19
s3api/controllers/base_test.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
"github.com/versity/versitygw/s3response"
)
@@ -49,8 +50,9 @@ func init() {
func TestNew(t *testing.T) {
type args struct {
be backend.Backend
iam auth.IAMService
be backend.Backend
iam auth.IAMService
logger s3log.AuditLogger
}
be := backend.BackendUnsupported{}
@@ -74,7 +76,7 @@ func TestNew(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := New(tt.args.be, tt.args.iam); !reflect.DeepEqual(got, tt.want) {
if got := New(tt.args.be, tt.args.iam, tt.args.logger); !reflect.DeepEqual(got, tt.want) {
t.Errorf("New() = %v, want %v", got, tt.want)
}
})
@@ -187,14 +189,15 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
}
}
func getPtr(val string) *string {
return &val
}
func TestS3ApiController_GetActions(t *testing.T) {
type args struct {
req *http.Request
}
getPtr := func(val string) *string {
return &val
}
now := time.Now()
app := fiber.New()
@@ -1435,7 +1438,7 @@ func Test_XMLresponse(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err); (err != nil) != tt.wantErr {
if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err, &LogOptions{}); (err != nil) != tt.wantErr {
t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
}
@@ -1515,7 +1518,7 @@ func Test_response(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := SendResponse(tt.args.ctx, tt.args.err); (err != nil) != tt.wantErr {
if err := SendResponse(tt.args.ctx, tt.args.err, &LogOptions{}); (err != nil) != tt.wantErr {
t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
}

44
s3api/middlewares/authentication.go
View File

@@ -29,6 +29,7 @@ import (
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
const (
@@ -38,15 +39,18 @@ const (
type RootUserConfig struct {
Access string
Secret string
Region string
}
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler {
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
ctx.Locals("region", root.Region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.LogOptions{Logger: logger})
}
// Check the signature version
@@ -56,48 +60,52 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
}
if len(authParts) != 3 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.LogOptions{Logger: logger})
}
startParts := strings.Split(authParts[0], " ")
if startParts[0] != "AWS4-HMAC-SHA256" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.LogOptions{Logger: logger})
}
credKv := strings.Split(startParts[1], "=")
if len(credKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger})
}
creds := strings.Split(credKv[1], "/")
if len(creds) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger})
}
ctx.Locals("access", creds[0])
ctx.Locals("isRoot", creds[0] == root.Access)
signHdrKv := strings.Split(authParts[1], "=")
if len(signHdrKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.LogOptions{Logger: logger})
}
signedHdrs := strings.Split(signHdrKv[1], ";")
account, err := acct.getAccount(creds[0])
if err == auth.ErrNoSuchUser {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.LogOptions{Logger: logger})
}
if err != nil {
return controllers.SendResponse(ctx, err)
return controllers.SendResponse(ctx, err, &controllers.LogOptions{Logger: logger})
}
ctx.Locals("role", account.Role)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.LogOptions{Logger: logger})
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.LogOptions{Logger: logger})
}
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
@@ -110,14 +118,14 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
// Compare the calculated hash with the hash provided
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.LogOptions{Logger: logger})
}
}
// Create a new http request instance from fasthttp request
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.LogOptions{Logger: logger})
}
signer := v4.NewSigner()
@@ -132,24 +140,20 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
}
})
if signErr != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.LogOptions{Logger: logger})
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.LogOptions{Logger: logger})
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.LogOptions{Logger: logger})
}
ctx.Locals("role", account.Role)
ctx.Locals("access", creds[0])
ctx.Locals("isRoot", creds[0] == root.Access)
return ctx.Next()
}
}

6
s3api/middlewares/md5.go
View File

@@ -21,9 +21,10 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyMD5Body() fiber.Handler {
func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
incomingSum := ctx.Get("Content-Md5")
if incomingSum == "" {
@@ -34,10 +35,9 @@ func VerifyMD5Body() fiber.Handler {
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest))
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.LogOptions{Logger: logger})
}
return ctx.Next()
}
}

5
s3api/router.go
View File

@@ -19,12 +19,13 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3log"
)
type S3ApiRouter struct{}
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
s3ApiController := controllers.New(be, iam)
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
s3ApiController := controllers.New(be, iam, logger)
adminController := controllers.AdminController{IAMService: iam}
app.Patch("/create-user", adminController.CreateUser)

2
s3api/router_test.go
View File

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam)
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil)
})
}
}

9
s3api/server.go
View File

@@ -22,6 +22,7 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3log"
)
type S3ApiServer struct {
@@ -33,7 +34,7 @@ type S3ApiServer struct {
debug bool
}
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) {
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...Option) (*S3ApiServer, error) {
server := &S3ApiServer{
app: app,
backend: be,
@@ -50,10 +51,10 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, region, server.debug))
app.Use(middlewares.VerifyMD5Body())
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyMD5Body(l))
server.router.Init(app, be, iam)
server.router.Init(app, be, iam, l)
return server, nil
}

2
s3api/server_test.go
View File

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
tt.args.port, "us-east-1", &auth.IAMServiceInternal{})
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return

110
s3log/audit-logger.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"encoding/hex"
"fmt"
"math/rand"
"strings"
"time"
"github.com/gofiber/fiber/v2"
)
type AuditLogger interface {
Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
}
type LogMeta struct {
BucketOwner string
ObjectSize int64
Action string
}
type LogConfig struct {
IsFile bool
WebhookURL string
}
type LogFields struct {
BucketOwner string
Bucket string
Time time.Time
RemoteIP string
Requester string
RequestID string
Operation string
Key string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
ObjectSize int64
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
VersionID string
HostID string
SignatureVersion string
CipherSuite string
AuthenticationType string
HostHeader string
TLSVersion string
AccessPointARN string
AclRequired string
}
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
if cfg.WebhookURL != "" && cfg.IsFile {
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
}
if cfg.WebhookURL != "" {
return InitWebhookLogger(cfg.WebhookURL)
}
if cfg.IsFile {
return InitFileLogger()
}
return nil, nil
}
func genID() string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, 8)
if _, err := src.Read(b); err != nil {
panic(err)
}
return strings.ToUpper(hex.EncodeToString(b))
}
func getTLSVersionName(version uint16) string {
switch version {
case tls.VersionTLS10:
return "TLSv1.0"
case tls.VersionTLS11:
return "TLSv1.1"
case tls.VersionTLS12:
return "TLSv1.2"
case tls.VersionTLS13:
return "TLSv1.3"
default:
return ""
}
}

204
s3log/file.go Normal file
View File

@@ -0,0 +1,204 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"errors"
"fmt"
"io/fs"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
const (
logFile = "access.log"
logFileMode = 0600
timeFormat = "02/January/2006:15:04:05 -0700"
)
type FileLogger struct {
LogFields
mu sync.Mutex
}
var _ AuditLogger = &FileLogger{}
func InitFileLogger() (AuditLogger, error) {
_, err := os.ReadFile(logFile)
if err != nil && errors.Is(err, fs.ErrNotExist) {
err := os.WriteFile(logFile, []byte{}, logFileMode)
if err != nil {
return nil, err
} else {
return nil, err
}
}
return &FileLogger{}, nil
}
func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
f.mu.Lock()
defer f.mu.Unlock()
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
f.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
f.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
f.BucketOwner = meta.BucketOwner
f.Bucket = bucket
f.Time = time.Now()
f.RemoteIP = ctx.IP()
f.Requester = access
f.RequestID = genID()
f.Operation = meta.Action
f.Key = object
f.RequestURI = reqURI
f.HttpStatus = httpStatus
f.ErrorCode = errorCode
f.BytesSent = len(body)
f.ObjectSize = meta.ObjectSize
f.TotalTime = time.Since(startTime).Milliseconds()
f.TurnAroundTime = time.Since(startTime).Milliseconds()
f.Referer = ctx.Get("Referer")
f.UserAgent = ctx.Get("User-Agent")
f.VersionID = ctx.Query("versionId")
f.HostID = ctx.Get("X-Amz-Id-2")
f.SignatureVersion = "SigV4"
f.AuthenticationType = "AuthHeader"
f.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
f.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
f.AclRequired = "Yes"
f.writeLog()
}
func (fl *FileLogger) writeLog() {
if fl.BucketOwner == "" {
fl.BucketOwner = "-"
}
if fl.Bucket == "" {
fl.Bucket = "-"
}
if fl.RemoteIP == "" {
fl.RemoteIP = "-"
}
if fl.Requester == "" {
fl.Requester = "-"
}
if fl.Operation == "" {
fl.Operation = "-"
}
if fl.Key == "" {
fl.Key = "-"
}
if fl.RequestURI == "" {
fl.RequestURI = "-"
}
if fl.ErrorCode == "" {
fl.ErrorCode = "-"
}
if fl.Referer == "" {
fl.Referer = "-"
}
if fl.UserAgent == "" {
fl.UserAgent = "-"
}
if fl.VersionID == "" {
fl.VersionID = "-"
}
if fl.HostID == "" {
fl.HostID = "-"
}
if fl.CipherSuite == "" {
fl.CipherSuite = "-"
}
if fl.HostHeader == "" {
fl.HostHeader = "-"
}
if fl.TLSVersion == "" {
fl.TLSVersion = "-"
}
log := fmt.Sprintf("\n%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v",
fl.BucketOwner,
fl.Bucket,
fmt.Sprintf("[%v]", fl.Time.Format(timeFormat)),
fl.RemoteIP,
fl.Requester,
fl.RequestID,
fl.Operation,
fl.Key,
fl.RequestURI,
fl.HttpStatus,
fl.ErrorCode,
fl.BytesSent,
fl.ObjectSize,
fl.TotalTime,
fl.TurnAroundTime,
fl.Referer,
fl.UserAgent,
fl.VersionID,
fl.HostID,
fl.SignatureVersion,
fl.CipherSuite,
fl.AuthenticationType,
fl.HostHeader,
fl.TLSVersion,
fl.AccessPointARN,
fl.AclRequired,
)
file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, logFileMode)
if err != nil {
fmt.Printf("error opening the log file: %v", err.Error())
}
defer file.Close()
_, err = file.WriteString(log)
if err != nil {
fmt.Printf("error writing in log file: %v", err.Error())
}
}

141
s3log/webhook.go Normal file
View File

@@ -0,0 +1,141 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
type WebhookLogger struct {
LogFields
mu sync.Mutex
url string
}
var _ AuditLogger = &WebhookLogger{}
func InitWebhookLogger(url string) (AuditLogger, error) {
client := &http.Client{
Timeout: 3 * time.Second,
}
_, err := client.Post(url, "application/json", nil)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
return nil, fmt.Errorf("unreachable webhook url")
}
}
return &WebhookLogger{
url: url,
}, nil
}
func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
wl.mu.Lock()
defer wl.mu.Unlock()
access := "-"
reqURI := ctx.Request().URI().String()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
wl.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
wl.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
wl.BucketOwner = meta.BucketOwner
wl.Bucket = bucket
wl.Time = time.Now()
wl.RemoteIP = ctx.IP()
wl.Requester = access
wl.RequestID = genID()
wl.Operation = meta.Action
wl.Key = object
wl.RequestURI = reqURI
wl.HttpStatus = httpStatus
wl.ErrorCode = errorCode
wl.BytesSent = len(body)
wl.ObjectSize = meta.ObjectSize
wl.TotalTime = time.Since(startTime).Milliseconds()
wl.TurnAroundTime = time.Since(startTime).Milliseconds()
wl.Referer = ctx.Get("Referer")
wl.UserAgent = ctx.Get("User-Agent")
wl.VersionID = ctx.Query("versionId")
wl.HostID = ctx.Get("X-Amz-Id-2")
wl.SignatureVersion = "SigV4"
wl.AuthenticationType = "AuthHeader"
wl.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
wl.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
wl.AclRequired = "Yes"
wl.sendLog()
}
func (wl *WebhookLogger) sendLog() {
jsonLog, err := json.Marshal(wl)
if err != nil {
fmt.Printf("\n failed to parse the log data: %v", err.Error())
}
req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
if err != nil {
fmt.Println(err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
go makeRequest(req)
}
func makeRequest(req *http.Request) {
client := &http.Client{
Timeout: 1 * time.Second,
}
_, err := client.Do(req)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
fmt.Println("error sending the log to the specified url")
}
}
}