Merge pull request #397 from versity/presigned-url-authentication

Presigned URL authentication
This commit is contained in:
Ben McClelland
2024-02-13 11:33:49 -08:00
committed by GitHub
9 changed files with 1497 additions and 25 deletions

View File

@@ -22,6 +22,30 @@ func TestAuthentication(s *S3Conf) {
Authentication_signature_error_incorrect_secret_key(s)
}
func TestPresignedAuthentication(s *S3Conf) {
PresignedAuth_missing_algo_query_param(s)
PresignedAuth_unsupported_algorithm(s)
PresignedAuth_missing_credentials_query_param(s)
PresignedAuth_malformed_creds_invalid_parts(s)
PresignedAuth_malformed_creds_invalid_parts(s)
PresignedAuth_creds_incorrect_service(s)
PresignedAuth_creds_incorrect_region(s)
PresignedAuth_creds_invalid_date(s)
PresignedAuth_missing_date_query(s)
PresignedAuth_dates_mismatch(s)
PresignedAuth_non_existing_access_key_id(s)
PresignedAuth_missing_signed_headers_query_param(s)
PresignedAuth_missing_expiration_query_param(s)
PresignedAuth_invalid_expiration_query_param(s)
PresignedAuth_negative_expiration_query_param(s)
PresignedAuth_exceeding_expiration_query_param(s)
PresignedAuth_expired_request(s)
PresignedAuth_incorrect_secret_key(s)
PresignedAuth_PutObject_success(s)
PresignedAuth_Put_GetObject_with_data(s)
PresignedAuth_UploadPart(s)
}
func TestCreateBucket(s *S3Conf) {
CreateBucket_invalid_bucket_name(s)
CreateBucket_existing_bucket(s)
@@ -219,6 +243,7 @@ func TestGetBucketAcl(s *S3Conf) {
func TestFullFlow(s *S3Conf) {
TestAuthentication(s)
TestPresignedAuthentication(s)
TestCreateBucket(s)
TestHeadBucket(s)
TestListBuckets(s)
@@ -277,6 +302,27 @@ func GetIntTests() IntTests {
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
"Authentication_incorrect_md5": Authentication_incorrect_md5,
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
"PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param,
"PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm,
"PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param,
"PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts,
"PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator,
"PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service,
"PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region,
"PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date,
"PresignedAuth_missing_date_query": PresignedAuth_missing_date_query,
"PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch,
"PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id,
"PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param,
"PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param,
"PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param,
"PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param,
"PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param,
"PresignedAuth_expired_request": PresignedAuth_expired_request,
"PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key,
"PresignedAuth_PutObject_success": PresignedAuth_PutObject_success,
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
"CreateBucket_as_user": CreateBucket_as_user,

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
@@ -19,7 +20,8 @@ import (
)
var (
shortTimeout = 10 * time.Second
shortTimeout = 10 * time.Second
iso8601Format = "20060102T150405Z"
)
func Authentication_empty_auth_header(s *S3Conf) error {
@@ -624,6 +626,912 @@ func Authentication_signature_error_incorrect_secret_key(s *S3Conf) error {
})
}
func PresignedAuth_missing_algo_query_param(s *S3Conf) error {
testName := "PresignedAuth_missing_algo_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Del("X-Amz-Algorithm")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_unsupported_algorithm(s *S3Conf) error {
testName := "PresignedAuth_unsupported_algorithm"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri := strings.Replace(v4req.URL, "AWS4-HMAC-SHA256", "AWS4-SHA256", 1)
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_missing_credentials_query_param(s *S3Conf) error {
testName := "PresignedAuth_missing_credentials_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Del("X-Amz-Credential")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_malformed_creds_invalid_parts(s *S3Conf) error {
testName := "PresignedAuth_malformed_creds_invalid_parts"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Set("X-Amz-Credential", "access/hello/world")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_creds_invalid_terminator(s *S3Conf) error {
testName := "PresignedAuth_creds_invalid_terminator"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri, err := changeAuthCred(v4req.URL, "aws5_request", credTerminator)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_creds_incorrect_service(s *S3Conf) error {
testName := "PresignedAuth_creds_incorrect_service"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri, err := changeAuthCred(v4req.URL, "sns", credService)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_creds_incorrect_region(s *S3Conf) error {
testName := "PresignedAuth_creds_incorrect_region"
cfg := *s
if cfg.awsRegion == "us-east-1" {
cfg.awsRegion = "us-west-1"
} else {
cfg.awsRegion = "us-east-1"
}
return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", cfg.awsRegion),
HTTPStatusCode: http.StatusForbidden,
}); err != nil {
return err
}
return nil
})
}
func PresignedAuth_creds_invalid_date(s *S3Conf) error {
testName := "PresignedAuth_creds_invalid_date"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri, err := changeAuthCred(v4req.URL, "32234Z34", credDate)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_non_existing_access_key_id(s *S3Conf) error {
testName := "PresignedAuth_non_existing_access_key_id"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri, err := changeAuthCred(v4req.URL, "a_rarely_existing_access_key_id890asd6f807as6ydf870say", credAccess)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_missing_date_query(s *S3Conf) error {
testName := "PresignedAuth_missing_date_query"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Del("X-Amz-Date")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_dates_mismatch(s *S3Conf) error {
testName := "PresignedAuth_dates_mismatch"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
uri, err := changeAuthCred(v4req.URL, "20060102", credDate)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_missing_signed_headers_query_param(s *S3Conf) error {
testName := "PresignedAuth_missing_signed_headers_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Del("X-Amz-SignedHeaders")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_missing_expiration_query_param(s *S3Conf) error {
testName := "PresignedAuth_missing_expiration_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Del("X-Amz-Expires")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_invalid_expiration_query_param(s *S3Conf) error {
testName := "PresignedAuth_invalid_expiration_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Set("X-Amz-Expires", "invalid_value")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMalformedExpires)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_negative_expiration_query_param(s *S3Conf) error {
testName := "PresignedAuth_negative_expiration_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Set("X-Amz-Expires", "-3")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrNegativeExpires)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_exceeding_expiration_query_param(s *S3Conf) error {
testName := "PresignedAuth_exceeding_expiration_query_param"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
queries := urlParsed.Query()
queries.Set("X-Amz-Expires", "60580000")
urlParsed.RawQuery = queries.Encode()
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMaximumExpires)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_expired_request(s *S3Conf) error {
testName := "PresignedAuth_expired_request"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
urlParsed, err := url.Parse(v4req.URL)
if err != nil {
return err
}
expDate := time.Now().AddDate(0, -1, 0).Format(iso8601Format)
queries := urlParsed.Query()
queries.Set("X-Amz-Date", expDate)
urlParsed.RawQuery = queries.Encode()
uri, err := changeAuthCred(urlParsed.String(), expDate[:8], credDate)
if err != nil {
return err
}
req, err := http.NewRequest(v4req.Method, uri, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrExpiredPresignRequest)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_incorrect_secret_key(s *S3Conf) error {
testName := "PresignedAuth_incorrect_secret_key"
cfg := *s
cfg.awsSecret += "x"
return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil {
return err
}
return nil
})
}
func PresignedAuth_PutObject_success(s *S3Conf) error {
testName := "PresignedAuth_PutObject_success"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
bucket := getBucketName()
err := setup(s, bucket)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("my-obj")})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
req, err := http.NewRequest(http.MethodPut, v4req.URL, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected my-obj to be successfully uploaded and get 200 response status, instead got %v", resp.StatusCode)
}
err = teardown(s, bucket)
if err != nil {
return err
}
return nil
})
}
func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
testName := "PresignedAuth_Put_GetObject_with_data"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
bucket, obj := getBucketName(), "my-obj"
err := setup(s, bucket)
if err != nil {
return err
}
data := "Hello world"
body := strings.NewReader(data)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Body: body})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
req, err := http.NewRequest(v4req.Method, v4req.URL, body)
if err != nil {
return err
}
req.Header = v4req.SignedHeader
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected my-obj to be successfully uploaded and get %v response status, instead got %v", http.StatusOK, resp.StatusCode)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
v4GetReq, err := client.PresignGetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &obj})
cancel()
if err != nil {
return err
}
req, err = http.NewRequest(v4GetReq.Method, v4GetReq.URL, nil)
if err != nil {
return err
}
resp, err = httpClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read get object response body %w", err)
}
fmt.Println(resp.Request.Method, resp.ContentLength, string(respBody))
if string(respBody) != data {
return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody)
}
err = teardown(s, bucket)
if err != nil {
return err
}
return nil
})
}
func PresignedAuth_UploadPart(s *S3Conf) error {
testName := "PresignedAuth_UploadPart"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
bucket, key, partNumber := getBucketName(), "my-mp", int32(1)
err := setup(s, bucket)
if err != nil {
return err
}
clt := s3.NewFromConfig(s.Config())
mp, err := createMp(clt, bucket, key)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignUploadPart(ctx, &s3.UploadPartInput{Bucket: &bucket, Key: &key, UploadId: mp.UploadId, PartNumber: &partNumber})
cancel()
if err != nil {
return err
}
httpClient := http.Client{
Timeout: shortTimeout,
}
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
if err != nil {
return err
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected response status code to be %v, instead got %v", http.StatusOK, resp.StatusCode)
}
etag := resp.Header.Get("Etag")
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := clt.ListParts(ctx, &s3.ListPartsInput{Bucket: &bucket, Key: &key, UploadId: mp.UploadId})
cancel()
if err != nil {
return err
}
if len(out.Parts) != 1 {
return fmt.Errorf("expected mp upload parts length to be 1, instead got %v", len(out.Parts))
}
if *out.Parts[0].ETag != etag {
return fmt.Errorf("expected uploaded part etag to be %v, instead got %v", etag, *out.Parts[0].ETag)
}
if *out.Parts[0].PartNumber != partNumber {
return fmt.Errorf("expected uploaded part part-number to be %v, instead got %v", partNumber, *out.Parts[0].PartNumber)
}
err = teardown(s, bucket)
if err != nil {
return err
}
return nil
})
}
func CreateBucket_invalid_bucket_name(s *S3Conf) error {
testName := "CreateBucket_invalid_bucket_name"
runF(testName)

View File

@@ -12,6 +12,7 @@ import (
"io"
rnd "math/rand"
"net/http"
"net/url"
"os"
"os/exec"
"strings"
@@ -150,6 +151,20 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err
return nil
}
func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
runF(testName)
clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
err := handler(clt)
if err != nil {
failF("%v: %v", testName, err)
return fmt.Errorf("%v: %w", testName, err)
}
passF(testName)
return nil
}
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
if err != nil {
@@ -551,3 +566,26 @@ func genRandString(length int) string {
}
return string(result)
}
const (
credAccess int = iota
credDate
credRegion
credService
credTerminator
)
func changeAuthCred(uri, newVal string, index int) (string, error) {
urlParsed, err := url.Parse(uri)
if err != nil {
return "", err
}
queries := urlParsed.Query()
creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
creds[index] = newVal
queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
urlParsed.RawQuery = queries.Encode()
return urlParsed.String(), nil
}

View File

@@ -44,6 +44,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
// If account is set in context locals, it means it was presigned url case
_, ok := ctx.Locals("account").(auth.Account)
if ok {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
@@ -96,7 +102,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
}
// Validate the dates difference
err = validateDate(tdate)
err = utils.ValidateDate(tdate)
if err != nil {
return sendResponse(ctx, err, logger)
}
@@ -158,29 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
return a.iam.GetUserAccount(access)
}
func validateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be less than a minute
if diff > 60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
if diff < -60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
return nil
}
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}

View File

@@ -0,0 +1,69 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
if ctx.Query("X-Amz-Signature") == "" {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authData, err := utils.ParsePresignedURIParts(ctx)
if err != nil {
return sendResponse(ctx, err, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
if utils.IsBigDataAction(ctx) {
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
})
return ctx.Next()
}
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
if err != nil {
return sendResponse(ctx, err, logger)
}
return ctx.Next()
}
}

View File

@@ -56,6 +56,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
app.Use(middlewares.VerifyMD5Body(l))

View File

@@ -0,0 +1,243 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
const (
unsignedPayload string = "UNSIGNED-PAYLOAD"
)
// PresignedAuthReader is an io.Reader that validates presigned request authorization
// once the underlying reader returns io.EOF. This is needed for streaming
// data requests where the data size is not known until
// the data is completely read.
type PresignedAuthReader struct {
ctx *fiber.Ctx
auth AuthData
secret string
r io.Reader
debug bool
}
func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *PresignedAuthReader {
return &PresignedAuthReader{
ctx: ctx,
r: r,
auth: auth,
secret: secret,
debug: debug,
}
}
// Read allows *PresignedAuthReader to be used as an io.Reader
func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
n, err := pr.r.Read(p)
if errors.Is(err, io.EOF) {
cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, pr.debug)
if cerr != nil {
return n, cerr
}
}
return n, err
}
// CheckPresignedSignature validates presigned request signature
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, debug bool) error {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
var contentLength int64
var err error
contentLengthStr := ctx.Get("Content-Length")
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return s3err.GetAPIError(s3err.ErrInvalidRequest)
}
}
// Create a new http request instance from fasthttp request
req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
if err != nil {
return fmt.Errorf("create http request from context: %w", err)
}
date, _ := time.Parse(iso8601Format, auth.Date)
signer := v4.NewSigner()
uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
AccessKeyID: auth.Access,
SecretAccessKey: secret,
}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return fmt.Errorf("presign generated http request: %w", err)
}
urlParts, err := url.Parse(uri)
if err != nil {
return fmt.Errorf("parse presigned url: %w", err)
}
signature := urlParts.Query().Get("X-Amz-Signature")
if signature != auth.Signature {
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
}
return nil
}
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
//
// # ParsePresignedURIParts parses and validates request URL query parameters
//
// ?X-Amz-Algorithm=AWS4-HMAC-SHA256
// &X-Amz-Credential=access-key-id/20130721/us-east-1/s3/aws4_request
// &X-Amz-Date=20130721T201207Z
// &X-Amz-Expires=86400
// &X-Amz-SignedHeaders=host
// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
a := AuthData{}
// Get and verify algorithm query parameter
algo := ctx.Query("X-Amz-Algorithm")
if algo == "" {
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
if algo != "AWS4-HMAC-SHA256" {
return a, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)
}
// Parse and validate credentials query parameter
credsQuery := ctx.Query("X-Amz-Credential")
if credsQuery == "" {
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
creds := strings.Split(credsQuery, "/")
if len(creds) != 5 {
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
}
if creds[3] != "s3" {
return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
}
if creds[4] != "aws4_request" {
return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
}
_, err := time.Parse(yyyymmdd, creds[1])
if err != nil {
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
}
// Parse and validate Date query param
date := ctx.Query("X-Amz-Date")
if date == "" {
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return a, s3err.GetAPIError(s3err.ErrMalformedDate)
}
if date[:8] != creds[1] {
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
}
if ctx.Locals("region") != creds[2] {
return a, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
HTTPStatusCode: http.StatusForbidden,
}
}
signature := ctx.Query("X-Amz-Signature")
if signature == "" {
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
signedHdrs := ctx.Query("X-Amz-SignedHeaders")
if signedHdrs == "" {
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
// Validate X-Amz-Expires query param and check if request is expired
err = validateExpiration(ctx.Query("X-Amz-Expires"), tdate)
if err != nil {
return a, err
}
a.Signature = signature
a.Access = creds[0]
a.Algorithm = algo
a.Region = creds[2]
a.SignedHeaders = signedHdrs
a.Date = date
return a, nil
}
func validateExpiration(str string, date time.Time) error {
if str == "" {
return s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
exp, err := strconv.Atoi(str)
if err != nil {
return s3err.GetAPIError(s3err.ErrMalformedExpires)
}
if exp < 0 {
return s3err.GetAPIError(s3err.ErrNegativeExpires)
}
if exp > 604800 {
return s3err.GetAPIError(s3err.ErrMaximumExpires)
}
now := time.Now()
passed := int(now.Sub(date).Seconds())
if passed > exp {
return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
}
return nil
}

View File

@@ -0,0 +1,100 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"testing"
"time"
"github.com/versity/versitygw/s3err"
)
func Test_validateExpiration(t *testing.T) {
type args struct {
str string
date time.Time
}
tests := []struct {
name string
args args
err error
}{
{
name: "empty-expiration",
args: args{
str: "",
date: time.Now(),
},
err: s3err.GetAPIError(s3err.ErrInvalidQueryParams),
},
{
name: "invalid-expiration",
args: args{
str: "invalid_expiration",
date: time.Now(),
},
err: s3err.GetAPIError(s3err.ErrMalformedExpires),
},
{
name: "negative-expiration",
args: args{
str: "-320",
date: time.Now(),
},
err: s3err.GetAPIError(s3err.ErrNegativeExpires),
},
{
name: "exceeding-expiration",
args: args{
str: "6048000",
date: time.Now(),
},
err: s3err.GetAPIError(s3err.ErrMaximumExpires),
},
{
name: "expired value",
args: args{
str: "200",
date: time.Now().AddDate(0, 0, -1),
},
err: s3err.GetAPIError(s3err.ErrExpiredPresignRequest),
},
{
name: "valid expiration",
args: args{
str: "300",
date: time.Now(),
},
err: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateExpiration(tt.args.str, tt.args.date)
// Check for nil case
if tt.err == nil && err != nil {
t.Errorf("Expected nil error, got: %v", err)
return
} else if tt.err == nil && err == nil {
// Both are nil, no need for further comparison
return
}
if err.Error() != tt.err.Error() {
t.Errorf("Expected error: %v, got: %v", tt.err, err)
}
})
}
}

View File

@@ -23,6 +23,7 @@ import (
"regexp"
"strconv"
"strings"
"time"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
@@ -86,6 +87,66 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
return httpReq, nil
}
var (
signedQueryArgs = map[string]bool{
"X-Amz-Algorithm": true,
"X-Amz-Credential": true,
"X-Amz-Date": true,
"X-Amz-SignedHeaders": true,
"X-Amz-Signature": true,
}
)
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
if IsBigDataAction(ctx) {
body = req.BodyStream()
} else {
body = bytes.NewReader(req.Body())
}
uri := string(ctx.Request().URI().Path())
isFirst := true
ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
_, ok := signedQueryArgs[string(key)]
if !ok {
if isFirst {
uri += fmt.Sprintf("?%s=%s", key, value)
isFirst = false
} else {
uri += fmt.Sprintf("&%s=%s", key, value)
}
}
})
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
if err != nil {
return nil, errors.New("error in creating an http request")
}
// Set the request headers
req.Header.VisitAll(func(key, value []byte) {
keyStr := string(key)
if includeHeader(keyStr, signedHdrs) {
httpReq.Header.Add(keyStr, string(value))
}
})
// Check if Content-Length in signed headers
// If content length is non 0, then the header will be included
if !includeHeader("Content-Length", signedHdrs) {
httpReq.ContentLength = 0
} else {
httpReq.ContentLength = contentLength
}
// Set the Host header
httpReq.Host = string(req.Header.Host())
return httpReq, nil
}
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
ctx.Response().Header.DisableNormalizing()
for key, val := range meta {
@@ -149,3 +210,26 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
}
return false
}
func ValidateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be less than a minute
if diff > 60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
if diff < -60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
return nil
}