diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 32fe3b8a..f94138c5 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -50,6 +50,7 @@ var ( logWebhookURL, accessLog string adminLogFile string healthPath string + virtualDomain string debug bool pprof string quiet bool @@ -227,6 +228,13 @@ func initFlags() []cli.Flag { Destination: &quiet, Aliases: []string{"q"}, }, + &cli.StringFlag{ + Name: "virtual-domain", + Usage: "enables the host-style bucket addressing with the specified virtual domain as base.", + EnvVars: []string{"VGW_VIRTUAL_DOMAIN"}, + Destination: &virtualDomain, + Aliases: []string{"vd"}, + }, &cli.StringFlag{ Name: "access-log", Usage: "enable server access logging to specified file", @@ -603,6 +611,9 @@ func runGateway(ctx context.Context, be backend.Backend) error { if readonly { opts = append(opts, s3api.WithReadOnly()) } + if virtualDomain != "" { + opts = append(opts, s3api.WithHostStyle(virtualDomain)) + } admApp := fiber.New(fiber.Config{ AppName: "versitygw", diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index a4f9e753..541ad2b2 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -34,7 +34,7 @@ var ( totalReqs int upload bool download bool - pathStyle bool + hostStyle bool checksumDisable bool versioningEnabled bool azureTests bool @@ -74,6 +74,12 @@ func initTestFlags() []cli.Flag { Destination: &endpoint, Aliases: []string{"e"}, }, + &cli.BoolFlag{ + Name: "host-style", + Usage: "Use host-style bucket addressing", + Value: false, + Destination: &hostStyle, + }, &cli.BoolFlag{ Name: "debug", Usage: "enable debug mode", @@ -191,12 +197,6 @@ func initTestCommands() []*cli.Command { Value: 1, Destination: &concurrency, }, - &cli.BoolFlag{ - Name: "pathStyle", - Usage: "Use Pathstyle bucket addressing", - Value: false, - Destination: &pathStyle, - }, &cli.BoolFlag{ Name: "checksumDis", Usage: "Disable server checksum", @@ -228,8 +228,8 @@ func initTestCommands() []*cli.Command { if debug { opts = append(opts, integration.WithDebug()) } - if pathStyle { - opts = append(opts, integration.WithPathStyle()) + if hostStyle { + opts = append(opts, integration.WithHostStyle()) } if checksumDisable { opts = append(opts, integration.WithDisableChecksum()) @@ -292,6 +292,9 @@ func initTestCommands() []*cli.Command { if checksumDisable { opts = append(opts, integration.WithDisableChecksum()) } + if hostStyle { + opts = append(opts, integration.WithHostStyle()) + } s3conf := integration.NewS3Conf(opts...) @@ -321,6 +324,9 @@ func getAction(tf testFunc) func(*cli.Context) error { if azureTests { opts = append(opts, integration.WithAzureMode()) } + if hostStyle { + opts = append(opts, integration.WithHostStyle()) + } s := integration.NewS3Conf(opts...) tf(s) @@ -356,6 +362,9 @@ func extractIntTests() (commands []*cli.Command) { if versioningEnabled { opts = append(opts, integration.WithVersioningEnabled()) } + if hostStyle { + opts = append(opts, integration.WithHostStyle()) + } s := integration.NewS3Conf(opts...) err := testFunc(s) diff --git a/s3api/middlewares/host-style-parser.go b/s3api/middlewares/host-style-parser.go new file mode 100644 index 00000000..f670cafc --- /dev/null +++ b/s3api/middlewares/host-style-parser.go @@ -0,0 +1,40 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "fmt" + "strings" + + "github.com/gofiber/fiber/v2" +) + +// HostStyleParser is a middleware which parses the bucket name +// from the 'Host' header and appends in the request URL path +func HostStyleParser(virtualDomain string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + host := string(ctx.Request().Host()) + // the host should match this pattern: '.' + bucket, _, found := strings.Cut(host, "."+virtualDomain) + if !found || bucket == "" { + return ctx.Next() + } + path := ctx.Path() + pathStyleUrl := fmt.Sprintf("/%v%v", bucket, path) + ctx.Path(pathStyleUrl) + + return ctx.Next() + } +} diff --git a/s3api/middlewares/url-decoder.go b/s3api/middlewares/url-decoder.go index e7e4c891..fc275ac1 100644 --- a/s3api/middlewares/url-decoder.go +++ b/s3api/middlewares/url-decoder.go @@ -26,7 +26,7 @@ import ( func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { return func(ctx *fiber.Ctx) error { - unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal())) + unescp, err := url.PathUnescape(string(ctx.Request().URI().PathOriginal())) if err != nil { return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm}) } diff --git a/s3api/server.go b/s3api/server.go index 80469302..96bf7d13 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -29,15 +29,16 @@ import ( ) type S3ApiServer struct { - app *fiber.App - backend backend.Backend - router *S3ApiRouter - port string - cert *tls.Certificate - quiet bool - debug bool - readonly bool - health string + app *fiber.App + backend backend.Backend + router *S3ApiRouter + port string + cert *tls.Certificate + quiet bool + debug bool + readonly bool + health string + virtualDomain string } func New( @@ -76,6 +77,13 @@ func New( }) } app.Use(middlewares.DecodeURL(l, mm)) + + // initialize host-style parser in virtual domain is specified + if server.virtualDomain != "" { + app.Use(middlewares.HostStyleParser(server.virtualDomain)) + } + + // initialize the debug logger in debug mode if server.debug { app.Use(middlewares.DebugLogger()) } @@ -123,6 +131,11 @@ func WithReadOnly() Option { return func(s *S3ApiServer) { s.readonly = true } } +// WithHostStyle enabled host-style bucket addressing on the server +func WithHostStyle(virtualDomain string) Option { + return func(s *S3ApiServer) { s.virtualDomain = virtualDomain } +} + func (sa *S3ApiServer) Serve() (err error) { if sa.cert != nil { return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert) diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 6eb906f9..30d81c75 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -29,7 +29,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go/encoding/httpbinding" "github.com/gofiber/fiber/v2" "github.com/valyala/fasthttp" "github.com/versity/versitygw/s3api/debuglogger" @@ -42,10 +41,6 @@ var ( bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`) ) -const ( - upperhex = "0123456789ABCDEF" -) - func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) { metadata = make(map[string]string) headers.DisableNormalizing() @@ -71,9 +66,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength body = bytes.NewReader(req.Body()) } - escapedURI := escapeOriginalURI(ctx) + uri := ctx.OriginalURL() - httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body) + httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body) if err != nil { return nil, errors.New("error in creating an http request") } @@ -126,8 +121,7 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont body = bytes.NewReader(req.Body()) } - uri := string(ctx.Request().URI().Path()) - uri = httpbinding.EscapePath(uri, false) + uri, _, _ := strings.Cut(ctx.OriginalURL(), "?") isFirst := true ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) { @@ -400,77 +394,6 @@ func IsValidOwnership(val types.ObjectOwnership) bool { } } -func escapeOriginalURI(ctx *fiber.Ctx) string { - path := ctx.Path() - - // Escape the URI original path - escapedURI := escapePath(path) - - // Add the URI query params - query := string(ctx.Request().URI().QueryArgs().QueryString()) - if query != "" { - escapedURI = escapedURI + "?" + query - } - - return escapedURI -} - -// Escapes the path string -// Most of the parts copied from std url -func escapePath(s string) string { - hexCount := 0 - for i := 0; i < len(s); i++ { - c := s[i] - if shouldEscape(c) { - hexCount++ - } - } - - if hexCount == 0 { - return s - } - - var buf [64]byte - var t []byte - - required := len(s) + 2*hexCount - if required <= len(buf) { - t = buf[:required] - } else { - t = make([]byte, required) - } - - j := 0 - for i := 0; i < len(s); i++ { - switch c := s[i]; { - case shouldEscape(c): - t[j] = '%' - t[j+1] = upperhex[c>>4] - t[j+2] = upperhex[c&15] - j += 3 - default: - t[j] = s[i] - j++ - } - } - - return string(t) -} - -// Checks if the character needs to be escaped -func shouldEscape(c byte) bool { - if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { - return false - } - - switch c { - case '-', '_', '.', '~', '/': - return false - } - - return true -} - type ChecksumValues map[types.ChecksumAlgorithm]string // Headers concatinates checksum algorithm by prefixing each diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 247f0e3a..26e8bc14 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -418,128 +418,6 @@ func TestIsValidOwnership(t *testing.T) { } } -func Test_shouldEscape(t *testing.T) { - type args struct { - c byte - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "shouldn't-escape-alphanum", - args: args{ - c: 'h', - }, - want: false, - }, - { - name: "shouldn't-escape-unreserved-char", - args: args{ - c: '_', - }, - want: false, - }, - { - name: "shouldn't-escape-unreserved-number", - args: args{ - c: '0', - }, - want: false, - }, - { - name: "shouldn't-escape-path-separator", - args: args{ - c: '/', - }, - want: false, - }, - { - name: "should-escape-special-char-1", - args: args{ - c: '&', - }, - want: true, - }, - { - name: "should-escape-special-char-2", - args: args{ - c: '*', - }, - want: true, - }, - { - name: "should-escape-special-char-3", - args: args{ - c: '(', - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := shouldEscape(tt.args.c); got != tt.want { - t.Errorf("shouldEscape() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_escapePath(t *testing.T) { - type args struct { - s string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "empty-string", - args: args{ - s: "", - }, - want: "", - }, - { - name: "alphanum-path", - args: args{ - s: "/test-bucket/test-key", - }, - want: "/test-bucket/test-key", - }, - { - name: "path-with-unescapable-chars", - args: args{ - s: "/test~bucket/test.key", - }, - want: "/test~bucket/test.key", - }, - { - name: "path-with-escapable-chars", - args: args{ - s: "/bucket-*(/test=key&", - }, - want: "/bucket-%2A%28/test%3Dkey%26", - }, - { - name: "path-with-space", - args: args{ - s: "/test-bucket/my key", - }, - want: "/test-bucket/my%20key", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := escapePath(tt.args.s); got != tt.want { - t.Errorf("escapePath() = %v, want %v", got, tt.want) - } - }) - } -} - func TestIsChecksumAlgorithmValid(t *testing.T) { type args struct { alg types.ChecksumAlgorithm diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index e2cee844..10fe585e 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -36,8 +36,8 @@ type S3Conf struct { awsSecret string awsRegion string endpoint string + hostStyle bool checksumDisable bool - pathStyle bool PartSize int64 Concurrency int debug bool @@ -87,8 +87,8 @@ func WithEndpoint(e string) Option { func WithDisableChecksum() Option { return func(s *S3Conf) { s.checksumDisable = true } } -func WithPathStyle() Option { - return func(s *S3Conf) { s.pathStyle = true } +func WithHostStyle() Option { + return func(s *S3Conf) { s.hostStyle = true } } func WithPartSize(p int64) Option { return func(s *S3Conf) { s.PartSize = p } @@ -122,7 +122,12 @@ func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider { } func (c *S3Conf) GetClient() *s3.Client { - return s3.NewFromConfig(c.Config()) + return s3.NewFromConfig(c.Config(), func(o *s3.Options) { + if c.hostStyle { + o.BaseEndpoint = &c.endpoint + o.UsePathStyle = false + } + }) } func (c *S3Conf) Config() aws.Config { diff --git a/tests/integration/tests.go b/tests/integration/tests.go index ec5499bb..e71dc0e7 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -1406,7 +1406,7 @@ func PresignedAuth_UploadPart(s *S3Conf) error { return err } - clt := s3.NewFromConfig(s.Config()) + clt := s.GetClient() mp, err := createMp(clt, bucket, key) if err != nil { return err @@ -1600,7 +1600,7 @@ func CreateBucket_ownership_with_acl(s *S3Conf) error { testName := "CreateBucket_ownership_with_acl" runF(testName) - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ @@ -1697,7 +1697,7 @@ func CreateBucket_non_default_acl(s *S3Conf) error { } bucket := getBucketName() - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = client.CreateBucket(ctx, &s3.CreateBucketInput{ @@ -1743,7 +1743,7 @@ func CreateBucket_default_object_lock(s *S3Conf) error { bucket := getBucketName() lockEnabled := true - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ @@ -1863,7 +1863,7 @@ func ListBuckets_as_user(s *S3Conf) error { return err } - userClient := s3.NewFromConfig(cfg.Config()) + userClient := cfg.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := userClient.ListBuckets(ctx, &s3.ListBucketsInput{}) @@ -1938,7 +1938,7 @@ func ListBuckets_as_admin(s *S3Conf) error { return err } - adminClient := s3.NewFromConfig(cfg.Config()) + adminClient := cfg.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{}) @@ -2216,7 +2216,7 @@ func DeleteBucket_non_existing_bucket(s *S3Conf) error { testName := "DeleteBucket_non_existing_bucket" runF(testName) bucket := getBucketName() - s3client := s3.NewFromConfig(s.Config()) + s3client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ @@ -3008,7 +3008,7 @@ func PutObject_with_object_lock(s *S3Conf) error { runF(testName) bucket, obj, lockStatus := getBucketName(), "my-obj", true - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ Bucket: &bucket, @@ -3414,7 +3414,7 @@ func PutObject_racey_success(s *S3Conf) error { runF(testName) bucket, obj, lockStatus := getBucketName(), "my-obj", true - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ Bucket: &bucket, @@ -3471,7 +3471,7 @@ func PutObject_invalid_credentials(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { newconf := *s newconf.awsSecret = newconf.awsSecret + "badpassword" - client := s3.NewFromConfig(newconf.Config()) + client := newconf.GetClient() _, err := putObjects(client, []string{"my-obj"}, bucket) return checkApiErr(err, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)) }) @@ -6301,7 +6301,7 @@ func CopyObject_not_owned_source_bucket(s *S3Conf) error { cfg.awsID = usr.access cfg.awsSecret = usr.secret - userS3Client := s3.NewFromConfig(cfg.Config()) + userS3Client := cfg.GetClient() err = createUsers(s, []user{usr}) if err != nil { @@ -11974,7 +11974,7 @@ func PutBucketAcl_success_access_denied(s *S3Conf) error { newConf := *s newConf.awsID = "grt1" newConf.awsSecret = "grt1secret" - userClient := s3.NewFromConfig(newConf.Config()) + userClient := newConf.GetClient() _, err = putObjects(userClient, []string{"my-obj"}, bucket) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { @@ -12006,7 +12006,7 @@ func PutBucketAcl_success_canned_acl(s *S3Conf) error { newConf := *s newConf.awsID = "grt1" newConf.awsSecret = "grt1secret" - userClient := s3.NewFromConfig(newConf.Config()) + userClient := newConf.GetClient() _, err = putObjects(userClient, []string{"my-obj"}, bucket) if err != nil { @@ -12038,7 +12038,7 @@ func PutBucketAcl_success_acp(s *S3Conf) error { newConf := *s newConf.awsID = "grt1" newConf.awsSecret = "grt1secret" - userClient := s3.NewFromConfig(newConf.Config()) + userClient := newConf.GetClient() _, err = putObjects(userClient, []string{"my-obj"}, bucket) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { @@ -12092,7 +12092,7 @@ func PutBucketAcl_success_grants(s *S3Conf) error { newConf := *s newConf.awsID = "grt1" newConf.awsSecret = "grt1secret" - userClient := s3.NewFromConfig(newConf.Config()) + userClient := newConf.GetClient() _, err = putObjects(userClient, []string{"my-obj"}, bucket) if err != nil { @@ -12286,7 +12286,7 @@ func GetBucketAcl_access_denied(s *S3Conf) error { newConf := *s newConf.awsID = "grt1" newConf.awsSecret = "grt1secret" - userClient := s3.NewFromConfig(newConf.Config()) + userClient := newConf.GetClient() ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = userClient.GetBucketAcl(ctx, &s3.GetBucketAclInput{ @@ -14863,7 +14863,7 @@ func AccessControl_default_ACL_user_access_denied(s *S3Conf) error { cfg.awsID = usr.access cfg.awsSecret = usr.secret - _, err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket) + _, err = putObjects(cfg.GetClient(), []string{"my-obj"}, bucket) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { return err } @@ -14889,7 +14889,7 @@ func AccessControl_default_ACL_userplus_access_denied(s *S3Conf) error { cfg.awsID = usr.access cfg.awsSecret = usr.secret - _, err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket) + _, err = putObjects(cfg.GetClient(), []string{"my-obj"}, bucket) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { return err } @@ -14915,7 +14915,7 @@ func AccessControl_default_ACL_admin_successful_access(s *S3Conf) error { cfg.awsID = admin.access cfg.awsSecret = admin.secret - _, err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket) + _, err = putObjects(cfg.GetClient(), []string{"my-obj"}, bucket) if err != nil { return err } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 14c1e326..f87576d7 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -59,7 +59,7 @@ func getBucketName() string { } func setup(s *S3Conf, bucket string, opts ...setupOpt) error { - s3client := s3.NewFromConfig(s.Config()) + s3client := s.GetClient() cfg := new(setupCfg) for _, opt := range opts { @@ -95,7 +95,7 @@ func setup(s *S3Conf, bucket string, opts ...setupOpt) error { } func teardown(s *S3Conf, bucket string) error { - s3client := s3.NewFromConfig(s.Config()) + s3client := s.GetClient() deleteObject := func(bucket, key, versionId *string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) @@ -200,7 +200,7 @@ func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, failF("%v: failed to create a bucket: %v", testName, err) return fmt.Errorf("%v: failed to create a bucket: %w", testName, err) } - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() handlerErr := handler(client, bucketName) if handlerErr != nil { failF("%v: %v", testName, handlerErr) @@ -222,7 +222,7 @@ func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, func actionHandlerNoSetup(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, _ ...setupOpt) error { runF(testName) - client := s3.NewFromConfig(s.Config()) + client := s.GetClient() handlerErr := handler(client, "") if handlerErr != nil { failF("%v: %v", testName, handlerErr) @@ -263,7 +263,7 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error { runF(testName) - clt := s3.NewPresignClient(s3.NewFromConfig(s.Config())) + clt := s3.NewPresignClient(s.GetClient()) err := handler(clt) if err != nil { @@ -989,7 +989,7 @@ func getUserS3Client(usr user, cfg *S3Conf) *s3.Client { config.awsID = usr.access config.awsSecret = usr.secret - return s3.NewFromConfig(config.Config()) + return config.GetClient() } // if true enables, otherwise disables