diff --git a/integration/tests.go b/integration/tests.go index 7f390bc..aed74d9 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -74,6 +74,7 @@ func TestPutGetObject(s *S3Conf) { dstBucket := "testdstbucket" obj := "myobject" obj2 := "myobject2" + obj3 := "myobject%%3" copySource := bucket + "/" + obj s3client := s3.NewFromConfig(s.Config()) @@ -107,6 +108,16 @@ func TestPutGetObject(s *S3Conf) { failF("%v: %v", testname, err) return } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj3, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ @@ -120,6 +131,17 @@ func TestPutGetObject(s *S3Conf) { return } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj3, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, @@ -130,7 +152,6 @@ func TestPutGetObject(s *S3Conf) { failF("%v: %v", testname, err) return } - fmt.Println(out.Metadata) if !areMapsSame(out.Metadata, meta) { failF("%v: incorrect object metadata", testname) return @@ -207,7 +228,7 @@ func TestPutGetObject(s *S3Conf) { } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}}}}) + _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}, {Key: &obj3}}}}) cancel() if err != nil { failF("%v: %v", testname, err) @@ -234,7 +255,7 @@ func TestPutGetObject(s *S3Conf) { } if objCount != 0 { - failF("%v: expected object count %v instead got %v", testname, 2, objCount) + failF("%v: expected object count %v instead got %v", testname, 0, objCount) return } diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 8fed3c4..b46aeb3 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -133,6 +133,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au AccessKeyID: creds[0], SecretAccessKey: account.Secret, }, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) { + options.DisableURIPathEscaping = true if debug { options.LogSigning = true options.Logger = logging.NewStandardLogger(os.Stderr) diff --git a/s3api/middlewares/url-decoder.go b/s3api/middlewares/url-decoder.go new file mode 100644 index 0000000..ea0019b --- /dev/null +++ b/s3api/middlewares/url-decoder.go @@ -0,0 +1,41 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "net/url" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/s3api/controllers" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" +) + +func DecodeURL(logger s3log.AuditLogger) fiber.Handler { + return func(ctx *fiber.Ctx) error { + reqURL := ctx.Request().URI().String() + decoded, err := url.Parse(reqURL) + if err != nil { + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger}) + } + ctx.Path(decoded.Path) + decodedURL, err := url.QueryUnescape(reqURL) + if err != nil { + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger}) + } + ctx.Request().SetRequestURI(decodedURL) + return ctx.Next() + } +} diff --git a/s3api/server.go b/s3api/server.go index a5d803e..57bb46f 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -49,6 +49,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po // Logging middlewares app.Use(logger.New()) + app.Use(middlewares.DecodeURL(l)) app.Use(middlewares.RequestLogger(server.debug)) // Authentication middlewares diff --git a/s3err/s3err.go b/s3err/s3err.go index f445d2f..b67cedf 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -107,6 +107,7 @@ const ( ErrPreconditionFailed ErrInvalidObjectState ErrInvalidRange + ErrInvalidURI // Non-AWS errors ErrExistingObjectIsDirectory @@ -381,6 +382,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The requested range is not valid for the request. Try another range.", HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, }, + ErrInvalidURI: { + Code: "InvalidURI", + Description: "The specified URI couldn't be parsed.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrExistingObjectIsDirectory: { Code: "ExistingObjectIsDirectory", Description: "Existing Object is a directory.", diff --git a/s3log/file.go b/s3log/file.go index e96f291..33e70b4 100644 --- a/s3log/file.go +++ b/s3log/file.go @@ -65,7 +65,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) { lf := LogFields{} access := "-" - reqURI := ctx.Request().URI().String() + reqURI := ctx.OriginalURL() path := strings.Split(ctx.Path(), "/") bucket, object := path[1], strings.Join(path[2:], "/") errorCode := "" diff --git a/s3log/webhook.go b/s3log/webhook.go index 4fcf31b..bc82726 100644 --- a/s3log/webhook.go +++ b/s3log/webhook.go @@ -62,7 +62,7 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet lf := LogFields{} access := "-" - reqURI := ctx.Request().URI().String() + reqURI := ctx.OriginalURL() path := strings.Split(ctx.Path(), "/") bucket, object := path[1], strings.Join(path[2:], "/") errorCode := ""