feat: Fixes #181, Added support to add object with special character keys, disabled URI path escaping in v4 signing, add a middleware to parse the URL and store the decoded version as a new URL, added test cases for adding/getting/listing objects with special characters

integration/tests.go

@@ -74,6 +74,7 @@ func TestPutGetObject(s *S3Conf) {
 	dstBucket := "testdstbucket"
 	obj := "myobject"
 	obj2 := "myobject2"
+	obj3 := "myobject%%3"
 	copySource := bucket + "/" + obj
 
 	s3client := s3.NewFromConfig(s.Config())
@@ -107,6 +108,16 @@ func TestPutGetObject(s *S3Conf) {
 		failF("%v: %v", testname, err)
 		return
 	}
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: &bucket,
+		Key:    &obj3,
+	})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
 
 	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
 	_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
@@ -120,6 +131,17 @@ func TestPutGetObject(s *S3Conf) {
 		return
 	}
 
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &obj3,
+	})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
 	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
 	out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
 		Bucket: &bucket,
@@ -130,7 +152,6 @@ func TestPutGetObject(s *S3Conf) {
 		failF("%v: %v", testname, err)
 		return
 	}
-	fmt.Println(out.Metadata)
 	if !areMapsSame(out.Metadata, meta) {
 		failF("%v: incorrect object metadata", testname)
 		return
@@ -207,7 +228,7 @@ func TestPutGetObject(s *S3Conf) {
 	}
 
 	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
-	_, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}}}})
+	_, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}, {Key: &obj3}}}})
 	cancel()
 	if err != nil {
 		failF("%v: %v", testname, err)
@@ -234,7 +255,7 @@ func TestPutGetObject(s *S3Conf) {
 	}
 
 	if objCount != 0 {
-		failF("%v: expected object count %v instead got %v", testname, 2, objCount)
+		failF("%v: expected object count %v instead got %v", testname, 0, objCount)
 		return
 	}
 

s3api/middlewares/authentication.go

@@ -133,6 +133,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			AccessKeyID:     creds[0],
 			SecretAccessKey: account.Secret,
 		}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
+			options.DisableURIPathEscaping = true
 			if debug {
 				options.LogSigning = true
 				options.Logger = logging.NewStandardLogger(os.Stderr)

s3api/middlewares/url-decoder.go

@@ -0,0 +1,41 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Path(decoded.Path)
+		decodedURL, err := url.QueryUnescape(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Request().SetRequestURI(decodedURL)
+		return ctx.Next()
+	}
+}

s3api/server.go

@@ -49,6 +49,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 
 	// Logging middlewares
 	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares

s3err/s3err.go

@@ -107,6 +107,7 @@ const (
 	ErrPreconditionFailed
 	ErrInvalidObjectState
 	ErrInvalidRange
+	ErrInvalidURI
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -381,6 +382,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The requested range is not valid for the request. Try another range.",
 		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
 	},
+	ErrInvalidURI: {
+		Code:           "InvalidURI",
+		Description:    "The specified URI couldn't be parsed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",

s3log/file.go

@@ -65,7 +65,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	lf := LogFields{}
 
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""

s3log/webhook.go

@@ -62,7 +62,7 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 	lf := LogFields{}
 
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""