Merge pull request #397 from versity/presigned-url-authentication

Presigned URL authentication

s3api/middlewares/authentication.go

@@ -44,6 +44,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
+		// If account is set in context locals, it means it was presigned url case
+		_, ok := ctx.Locals("account").(auth.Account)
+		if ok {
+			return ctx.Next()
+		}
+
 		ctx.Locals("region", region)
 		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
@@ -96,7 +102,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		}
 
 		// Validate the dates difference
-		err = validateDate(tdate)
+		err = utils.ValidateDate(tdate)
 		if err != nil {
 			return sendResponse(ctx, err, logger)
 		}
@@ -158,29 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 	return a.iam.GetUserAccount(access)
 }
 
-func validateDate(date time.Time) error {
-	now := time.Now().UTC()
-	diff := date.Unix() - now.Unix()
-
-	// Checks the dates difference to be less than a minute
-	if diff > 60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-	if diff < -60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-
-	return nil
-}
-
 func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
 	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 }

s3api/middlewares/presign-auth.go

@@ -0,0 +1,69 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+	acct := accounts{root: root, iam: iam}
+
+	return func(ctx *fiber.Ctx) error {
+		if ctx.Query("X-Amz-Signature") == "" {
+			return ctx.Next()
+		}
+
+		ctx.Locals("region", region)
+		ctx.Locals("startTime", time.Now())
+
+		authData, err := utils.ParsePresignedURIParts(ctx)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		ctx.Locals("isRoot", authData.Access == root.Access)
+		account, err := acct.getAccount(authData.Access)
+		if err == auth.ErrNoSuchUser {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+		}
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+		ctx.Locals("account", account)
+
+		if utils.IsBigDataAction(ctx) {
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
+			})
+
+			return ctx.Next()
+		}
+
+		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		return ctx.Next()
+	}
+}

s3api/server.go

@@ -56,6 +56,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
+	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
 	app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
 	app.Use(middlewares.VerifyMD5Body(l))

s3api/utils/presign-auth-reader.go

@@ -0,0 +1,243 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	unsignedPayload string = "UNSIGNED-PAYLOAD"
+)
+
+// PresignedAuthReader is an io.Reader that validates presigned request authorization
+// once the underlying reader returns io.EOF.  This is needed for streaming
+// data requests where the data size is not known until
+// the data is completely read.
+type PresignedAuthReader struct {
+	ctx    *fiber.Ctx
+	auth   AuthData
+	secret string
+	r      io.Reader
+	debug  bool
+}
+
+func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *PresignedAuthReader {
+	return &PresignedAuthReader{
+		ctx:    ctx,
+		r:      r,
+		auth:   auth,
+		secret: secret,
+		debug:  debug,
+	}
+}
+
+// Read allows *PresignedAuthReader to be used as an io.Reader
+func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
+	n, err := pr.r.Read(p)
+
+	if errors.Is(err, io.EOF) {
+		cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, pr.debug)
+		if cerr != nil {
+			return n, cerr
+		}
+	}
+
+	return n, err
+}
+
+// CheckPresignedSignature validates presigned request signature
+func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, debug bool) error {
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	var contentLength int64
+	var err error
+	contentLengthStr := ctx.Get("Content-Length")
+	if contentLengthStr != "" {
+		contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+		if err != nil {
+			return s3err.GetAPIError(s3err.ErrInvalidRequest)
+		}
+	}
+
+	// Create a new http request instance from fasthttp request
+	req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
+	if err != nil {
+		return fmt.Errorf("create http request from context: %w", err)
+	}
+
+	date, _ := time.Parse(iso8601Format, auth.Date)
+
+	signer := v4.NewSigner()
+	uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
+		AccessKeyID:     auth.Access,
+		SecretAccessKey: secret,
+	}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
+		options.DisableURIPathEscaping = true
+		if debug {
+			options.LogSigning = true
+			options.Logger = logging.NewStandardLogger(os.Stderr)
+		}
+	})
+	if signErr != nil {
+		return fmt.Errorf("presign generated http request: %w", err)
+	}
+
+	urlParts, err := url.Parse(uri)
+	if err != nil {
+		return fmt.Errorf("parse presigned url: %w", err)
+	}
+
+	signature := urlParts.Query().Get("X-Amz-Signature")
+	if signature != auth.Signature {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+//
+// # ParsePresignedURIParts parses and validates request URL query parameters
+//
+// ?X-Amz-Algorithm=AWS4-HMAC-SHA256
+// &X-Amz-Credential=access-key-id/20130721/us-east-1/s3/aws4_request
+// &X-Amz-Date=20130721T201207Z
+// &X-Amz-Expires=86400
+// &X-Amz-SignedHeaders=host
+// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
+func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
+	a := AuthData{}
+
+	// Get and verify algorithm query parameter
+	algo := ctx.Query("X-Amz-Algorithm")
+	if algo == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+	if algo != "AWS4-HMAC-SHA256" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)
+	}
+
+	// Parse and validate credentials query parameter
+	credsQuery := ctx.Query("X-Amz-Credential")
+	if credsQuery == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	creds := strings.Split(credsQuery, "/")
+	if len(creds) != 5 {
+		return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+	}
+	if creds[3] != "s3" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
+	}
+	if creds[4] != "aws4_request" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
+	}
+	_, err := time.Parse(yyyymmdd, creds[1])
+	if err != nil {
+		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+	}
+
+	// Parse and validate Date query param
+	date := ctx.Query("X-Amz-Date")
+	if date == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	tdate, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return a, s3err.GetAPIError(s3err.ErrMalformedDate)
+	}
+
+	if date[:8] != creds[1] {
+		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+	}
+
+	if ctx.Locals("region") != creds[2] {
+		return a, s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	signature := ctx.Query("X-Amz-Signature")
+	if signature == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	signedHdrs := ctx.Query("X-Amz-SignedHeaders")
+	if signedHdrs == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	// Validate X-Amz-Expires query param and check if request is expired
+	err = validateExpiration(ctx.Query("X-Amz-Expires"), tdate)
+	if err != nil {
+		return a, err
+	}
+
+	a.Signature = signature
+	a.Access = creds[0]
+	a.Algorithm = algo
+	a.Region = creds[2]
+	a.SignedHeaders = signedHdrs
+	a.Date = date
+
+	return a, nil
+}
+
+func validateExpiration(str string, date time.Time) error {
+	if str == "" {
+		return s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	exp, err := strconv.Atoi(str)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrMalformedExpires)
+	}
+
+	if exp < 0 {
+		return s3err.GetAPIError(s3err.ErrNegativeExpires)
+	}
+
+	if exp > 604800 {
+		return s3err.GetAPIError(s3err.ErrMaximumExpires)
+	}
+
+	now := time.Now()
+	passed := int(now.Sub(date).Seconds())
+
+	if passed > exp {
+		return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
+	}
+
+	return nil
+}

s3api/utils/presign-auth-reader_test.go

@@ -0,0 +1,100 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"testing"
+	"time"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+func Test_validateExpiration(t *testing.T) {
+	type args struct {
+		str  string
+		date time.Time
+	}
+	tests := []struct {
+		name string
+		args args
+		err  error
+	}{
+		{
+			name: "empty-expiration",
+			args: args{
+				str:  "",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrInvalidQueryParams),
+		},
+		{
+			name: "invalid-expiration",
+			args: args{
+				str:  "invalid_expiration",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrMalformedExpires),
+		},
+		{
+			name: "negative-expiration",
+			args: args{
+				str:  "-320",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrNegativeExpires),
+		},
+		{
+			name: "exceeding-expiration",
+			args: args{
+				str:  "6048000",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrMaximumExpires),
+		},
+		{
+			name: "expired value",
+			args: args{
+				str:  "200",
+				date: time.Now().AddDate(0, 0, -1),
+			},
+			err: s3err.GetAPIError(s3err.ErrExpiredPresignRequest),
+		},
+		{
+			name: "valid expiration",
+			args: args{
+				str:  "300",
+				date: time.Now(),
+			},
+			err: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateExpiration(tt.args.str, tt.args.date)
+			// Check for nil case
+			if tt.err == nil && err != nil {
+				t.Errorf("Expected nil error, got: %v", err)
+				return
+			} else if tt.err == nil && err == nil {
+				// Both are nil, no need for further comparison
+				return
+			}
+
+			if err.Error() != tt.err.Error() {
+				t.Errorf("Expected error: %v, got: %v", tt.err, err)
+			}
+		})
+	}
+}

s3api/utils/utils.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
@@ -86,6 +87,66 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 	return httpReq, nil
 }
 
+var (
+	signedQueryArgs = map[string]bool{
+		"X-Amz-Algorithm":     true,
+		"X-Amz-Credential":    true,
+		"X-Amz-Date":          true,
+		"X-Amz-SignedHeaders": true,
+		"X-Amz-Signature":     true,
+	}
+)
+
+func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
+	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
+
+	uri := string(ctx.Request().URI().Path())
+	isFirst := true
+
+	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
+		_, ok := signedQueryArgs[string(key)]
+		if !ok {
+			if isFirst {
+				uri += fmt.Sprintf("?%s=%s", key, value)
+				isFirst = false
+			} else {
+				uri += fmt.Sprintf("&%s=%s", key, value)
+			}
+		}
+	})
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
+	if err != nil {
+		return nil, errors.New("error in creating an http request")
+	}
+	// Set the request headers
+	req.Header.VisitAll(func(key, value []byte) {
+		keyStr := string(key)
+		if includeHeader(keyStr, signedHdrs) {
+			httpReq.Header.Add(keyStr, string(value))
+		}
+	})
+
+	// Check if Content-Length in signed headers
+	// If content length is non 0, then the header will be included
+	if !includeHeader("Content-Length", signedHdrs) {
+		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
+	}
+
+	// Set the Host header
+	httpReq.Host = string(req.Header.Host())
+
+	return httpReq, nil
+}
+
 func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
 	ctx.Response().Header.DisableNormalizing()
 	for key, val := range meta {
@@ -149,3 +210,26 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
 	}
 	return false
 }
+
+func ValidateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if diff > 60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+	if diff < -60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	return nil
+}