versitygw/s3api/utils/utils.go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package utils

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/aws/smithy-go/encoding/httpbinding"
	"github.com/gofiber/fiber/v2"
	"github.com/valyala/fasthttp"
	"github.com/versity/versitygw/s3err"
	"github.com/versity/versitygw/s3response"
)

var (
	bucketNameRegexp   = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
)

const (
	upperhex = "0123456789ABCDEF"
)

func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
	metadata = make(map[string]string)
	headers.DisableNormalizing()
	headers.VisitAllInOrder(func(key, value []byte) {
		hKey := string(key)
		if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
			trimmedKey := hKey[11:]
			headerValue := string(value)
			metadata[trimmedKey] = headerValue
		}
	})
	headers.EnableNormalizing()

	return
}

func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
	req := ctx.Request()
	var body io.Reader
	if IsBigDataAction(ctx) {
		body = req.BodyStream()
	} else {
		body = bytes.NewReader(req.Body())
	}

	escapedURI := escapeOriginalURI(ctx)

	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
	if err != nil {
		return nil, errors.New("error in creating an http request")
	}

	// Set the request headers
	req.Header.VisitAll(func(key, value []byte) {
		keyStr := string(key)
		if includeHeader(keyStr, signedHdrs) {
			httpReq.Header.Add(keyStr, string(value))
		}
	})

	// make sure all headers in the signed headers are present
	for _, header := range signedHdrs {
		if httpReq.Header.Get(header) == "" {
			httpReq.Header.Set(header, "")
		}
	}

	// Check if Content-Length in signed headers
	// If content length is non 0, then the header will be included
	if !includeHeader("Content-Length", signedHdrs) {
		httpReq.ContentLength = 0
	} else {
		httpReq.ContentLength = contentLength
	}

	// Set the Host header
	httpReq.Host = string(req.Header.Host())

	return httpReq, nil
}

var (
	signedQueryArgs = map[string]bool{
		"X-Amz-Algorithm":     true,
		"X-Amz-Credential":    true,
		"X-Amz-Date":          true,
		"X-Amz-SignedHeaders": true,
		"X-Amz-Signature":     true,
	}
)

func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
	req := ctx.Request()
	var body io.Reader
	if IsBigDataAction(ctx) {
		body = req.BodyStream()
	} else {
		body = bytes.NewReader(req.Body())
	}

	uri := string(ctx.Request().URI().Path())
	uri = httpbinding.EscapePath(uri, false)
	isFirst := true

	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
		_, ok := signedQueryArgs[string(key)]
		if !ok {
			escapeValue := url.QueryEscape(string(value))
			if isFirst {
				uri += fmt.Sprintf("?%s=%s", key, escapeValue)
				isFirst = false
			} else {
				uri += fmt.Sprintf("&%s=%s", key, escapeValue)
			}
		}
	})

	httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
	if err != nil {
		return nil, errors.New("error in creating an http request")
	}
	// Set the request headers
	req.Header.VisitAll(func(key, value []byte) {
		keyStr := string(key)
		if includeHeader(keyStr, signedHdrs) {
			httpReq.Header.Add(keyStr, string(value))
		}
	})

	// Check if Content-Length in signed headers
	// If content length is non 0, then the header will be included
	if !includeHeader("Content-Length", signedHdrs) {
		httpReq.ContentLength = 0
	} else {
		httpReq.ContentLength = contentLength
	}

	// Set the Host header
	httpReq.Host = string(req.Header.Host())

	return httpReq, nil
}

func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
	ctx.Response().Header.DisableNormalizing()
	for key, val := range meta {
		ctx.Response().Header.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
	}
	ctx.Response().Header.EnableNormalizing()
}

func ParseUint(str string) (int32, error) {
	if str == "" {
		return 1000, nil
	}
	num, err := strconv.ParseInt(str, 10, 32)
	if err != nil {
		return 1000, fmt.Errorf("invalid int: %w", err)
	}
	if num < 0 {
		return 1000, fmt.Errorf("negative uint: %v", num)
	}
	if num > 1000 {
		num = 1000
	}
	return int32(num), nil
}

type CustomHeader struct {
	Key   string
	Value string
}

func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
	for _, header := range headers {
		ctx.Set(header.Key, header.Value)
	}
}

// Streams the response body by chunks
func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
	// SetBodyStream will call Close() on the reader when the stream is done
	// since rdr is a ReadCloser
	ctx.Context().SetBodyStream(rdr, bodysize)
}

func IsValidBucketName(bucket string) bool {
	if len(bucket) < 3 || len(bucket) > 63 {
		return false
	}
	// Checks to contain only digits, lowercase letters, dot, hyphen.
	// Checks to start and end with only digits and lowercase letters.
	if !bucketNameRegexp.MatchString(bucket) {
		return false
	}
	// Checks not to be a valid IP address
	if bucketNameIpRegexp.MatchString(bucket) {
		return false
	}
	return true
}

func includeHeader(hdr string, signedHdrs []string) bool {
	for _, shdr := range signedHdrs {
		if strings.EqualFold(hdr, shdr) {
			return true
		}
	}
	return false
}

func IsBigDataAction(ctx *fiber.Ctx) bool {
	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
			return true
		}
	}
	return false
}

// expiration time window
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp
const timeExpirationSec = 15 * 60

func ValidateDate(date time.Time) error {
	now := time.Now().UTC()
	diff := date.Unix() - now.Unix()

	// Checks the dates difference to be within allotted window
	if diff > timeExpirationSec || diff < -timeExpirationSec {
		return s3err.GetAPIError(s3err.ErrRequestTimeTooSkewed)
	}

	return nil
}

func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
	for _, obj := range objs {
		result = append(result, *obj.Key)
	}

	return
}

func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
	// These properties shouldn't appear in the final response body
	output.LastModified = nil
	output.VersionId = nil
	output.DeleteMarker = nil

	if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
		output.ETag = nil
	}
	if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
		output.ObjectParts = nil
	}
	if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
		output.ObjectSize = nil
	}
	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
		output.StorageClass = ""
	}
	if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
		output.Checksum = nil
	}

	return output
}

func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
	attrs := map[s3response.ObjectAttributes]struct{}{}
	var err error
	ctx.Request().Header.VisitAll(func(key, value []byte) {
		if string(key) == "X-Amz-Object-Attributes" {
			oattrs := strings.Split(string(value), ",")
			for _, a := range oattrs {
				attr := s3response.ObjectAttributes(a)
				if !attr.IsValid() {
					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
					break
				}
				attrs[attr] = struct{}{}
			}
		}
	})

	if len(attrs) == 0 {
		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
	}

	return attrs, err
}

type objLockCfg struct {
	RetainUntilDate time.Time
	ObjectLockMode  types.ObjectLockMode
	LegalHoldStatus types.ObjectLockLegalHoldStatus
}

func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
	legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
	objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
	objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")

	if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
		return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
	}

	var retainUntilDate time.Time
	if objLockDate != "" {
		rDate, err := time.Parse(time.RFC3339, objLockDate)
		if err != nil {
			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
		}
		if rDate.Before(time.Now()) {
			return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
		}
		retainUntilDate = rDate
	}

	objLockMode := types.ObjectLockMode(objLockModeHdr)

	if objLockMode != "" &&
		objLockMode != types.ObjectLockModeCompliance &&
		objLockMode != types.ObjectLockModeGovernance {
		return nil, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode)
	}

	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)

	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
		return nil, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)
	}

	return &objLockCfg{
		RetainUntilDate: retainUntilDate,
		ObjectLockMode:  objLockMode,
		LegalHoldStatus: legalHold,
	}, nil
}

func IsValidOwnership(val types.ObjectOwnership) bool {
	switch val {
	case types.ObjectOwnershipBucketOwnerEnforced:
		return true
	case types.ObjectOwnershipBucketOwnerPreferred:
		return true
	case types.ObjectOwnershipObjectWriter:
		return true
	default:
		return false
	}
}

func escapeOriginalURI(ctx *fiber.Ctx) string {
	path := ctx.Path()

	// Escape the URI original path
	escapedURI := escapePath(path)

	// Add the URI query params
	query := string(ctx.Request().URI().QueryArgs().QueryString())
	if query != "" {
		escapedURI = escapedURI + "?" + query
	}

	return escapedURI
}

// Escapes the path string
// Most of the parts copied from std url
func escapePath(s string) string {
	hexCount := 0
	for i := 0; i < len(s); i++ {
		c := s[i]
		if shouldEscape(c) {
			hexCount++
		}
	}

	if hexCount == 0 {
		return s
	}

	var buf [64]byte
	var t []byte

	required := len(s) + 2*hexCount
	if required <= len(buf) {
		t = buf[:required]
	} else {
		t = make([]byte, required)
	}

	j := 0
	for i := 0; i < len(s); i++ {
		switch c := s[i]; {
		case shouldEscape(c):
			t[j] = '%'
			t[j+1] = upperhex[c>>4]
			t[j+2] = upperhex[c&15]
			j += 3
		default:
			t[j] = s[i]
			j++
		}
	}

	return string(t)
}

// Checks if the character needs to be escaped
func shouldEscape(c byte) bool {
	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
		return false
	}

	switch c {
	case '-', '_', '.', '~', '/':
		return false
	}

	return true
}

func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
	sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))

	err := IsChecksumAlgorithmValid(sdkAlgorithm)
	if err != nil {
		return "", nil, err
	}

	checksums := map[types.ChecksumAlgorithm]string{}

	var hdrErr error
	// Parse and validate checksum headers
	ctx.Request().Header.VisitAll(func(key, value []byte) {
		// Skip `X-Amz-Checksum-Type` as it's a special header
		if hdrErr != nil || !strings.HasPrefix(string(key), "X-Amz-Checksum-") || string(key) == "X-Amz-Checksum-Type" {
			return
		}

		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(key), "X-Amz-Checksum-")))
		err := IsChecksumAlgorithmValid(algo)
		if err != nil {
			hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
			return
		}

		checksums[algo] = string(value)
	})

	if hdrErr != nil {
		return sdkAlgorithm, nil, hdrErr
	}

	headerCtr := 0
	for al, val := range checksums {
		if val != "" && !IsValidChecksum(val, al) {
			return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
		}
		// If any other checksum value is provided,
		// rather than x-amz-sdk-checksum-algorithm
		if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" {
			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
		}
		if val != "" {
			sdkAlgorithm = al
			headerCtr++
		}

		if headerCtr > 1 {
			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
		}
	}

	return sdkAlgorithm, checksums, nil
}

var checksumLengths = map[types.ChecksumAlgorithm]int{
	types.ChecksumAlgorithmCrc32:     4,
	types.ChecksumAlgorithmCrc32c:    4,
	types.ChecksumAlgorithmCrc64nvme: 8,
	types.ChecksumAlgorithmSha1:      20,
	types.ChecksumAlgorithmSha256:    32,
}

func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
	decoded, err := base64.StdEncoding.DecodeString(checksum)
	if err != nil {
		return false
	}

	expectedLength, exists := checksumLengths[algorithm]
	if !exists {
		return false
	}

	return len(decoded) == expectedLength
}

func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
	alg = types.ChecksumAlgorithm(strings.ToUpper(string(alg)))
	if alg != "" &&
		alg != types.ChecksumAlgorithmCrc32 &&
		alg != types.ChecksumAlgorithmCrc32c &&
		alg != types.ChecksumAlgorithmSha1 &&
		alg != types.ChecksumAlgorithmSha256 &&
		alg != types.ChecksumAlgorithmCrc64nvme {
		return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
	}

	return nil
}

// Validates the provided checksum type
func IsChecksumTypeValid(t types.ChecksumType) error {
	if t != "" &&
		t != types.ChecksumTypeComposite &&
		t != types.ChecksumTypeFullObject {
		return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
	}
	return nil
}

type checksumTypeSchema map[types.ChecksumType]struct{}
type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema

// A table defining the checksum algorithm/type support
var checksumMap checksumSchema = checksumSchema{
	types.ChecksumAlgorithmCrc32: checksumTypeSchema{
		types.ChecksumTypeComposite:  struct{}{},
		types.ChecksumTypeFullObject: struct{}{},
		"":                           struct{}{},
	},
	types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
		types.ChecksumTypeComposite:  struct{}{},
		types.ChecksumTypeFullObject: struct{}{},
		"":                           struct{}{},
	},
	types.ChecksumAlgorithmSha1: checksumTypeSchema{
		types.ChecksumTypeComposite: struct{}{},
		"":                          struct{}{},
	},
	types.ChecksumAlgorithmSha256: checksumTypeSchema{
		types.ChecksumTypeComposite: struct{}{},
		"":                          struct{}{},
	},
	types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
		types.ChecksumTypeFullObject: struct{}{},
		"":                           struct{}{},
	},
	// Both could be empty
	"": checksumTypeSchema{
		"": struct{}{},
	},
}

// Checks if checksum type and algorithm are supported together
func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error {
	typeSchema := checksumMap[algo]
	_, ok := typeSchema[t]
	if !ok {
		return s3err.GetChecksumSchemaMismatchErr(algo, t)
	}

	return nil
}

// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
	algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
	if err := IsChecksumAlgorithmValid(algo); err != nil {
		return "", "", err
	}

	chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
	if err := IsChecksumTypeValid(chType); err != nil {
		return "", "", err
	}

	// Verify if checksum algorithm is provided, if
	// checksum type is specified
	if chType != "" && algo == "" {
		return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
	}

	// Verify if the checksum type is supported for
	// the provided checksum algorithm
	if err := checkChecksumTypeAndAlgo(algo, chType); err != nil {
		return algo, chType, err
	}

	// x-amz-checksum-type defaults to COMPOSITE
	// if x-amz-checksum-algorithm is set except
	// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
	if algo != "" && chType == "" {
		if algo == types.ChecksumAlgorithmCrc64nvme {
			chType = types.ChecksumTypeFullObject
		} else {
			chType = types.ChecksumTypeComposite
		}
	}

	return algo, chType, nil
}