test: bucket setup/resetting cleanup, go virtual path handling, s3api profile setup

tests/env.sh

@@ -116,6 +116,10 @@ check_aws_vars() {
     log 1 "AWS_PROFILE missing"
     return 1
   fi
+  if ! check_user_profile_and_add_if_needed; then
+    log 1 "user profile '$AWS_PROFILE' not found and unable to be added"
+    return 1
+  fi
   if [ "$DIRECT" != "true" ]; then
     if [ -z "$AWS_ENDPOINT_URL" ]; then
       log 1 "AWS_ENDPOINT_URL missing"
@@ -348,6 +352,34 @@ check_user_vars() {
   return 1
 }
 
+check_user_profile_and_add_if_needed() {
+  local response profile aws_profiles=()
+
+  if ! response=$(aws configure list-profiles 2>&1); then
+    log 2 "error listing aws profiles: $response"
+    return 1
+  fi
+  mapfile -t aws_profiles <<< "$response"
+  for profile in "${aws_profiles[@]}"; do
+    if [ "$profile" != "" ] && [ "$AWS_PROFILE" == "$profile" ]; then
+      return 0
+    fi
+  done
+  if ! response=$(aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile "$AWS_PROFILE" 2>&1); then
+    log 2 "error calculating response: $response"
+    return 1
+  fi
+  if ! response=$(aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile "$AWS_PROFILE" 2>&1); then
+    log 2 "error calculating response: $response"
+    return 1
+  fi
+  if ! response=$(aws configure set aws_region "$AWS_REGION" --profile "$AWS_PROFILE" 2>&1); then
+    log 2 "error calculating response: $response"
+    return 1
+  fi
+  return 0
+}
+
 delete_command_log() {
   if [ -f "$COMMAND_LOG" ]; then
     if ! error=$(rm "$COMMAND_LOG"); then

tests/rest_scripts/command/curlRequest.go

@@ -39,7 +39,7 @@ func (c *CurlCommand) Render() error {
 	if c.Config.Method != "GET" {
 		curlCommand = append(curlCommand, fmt.Sprintf("-X %s ", c.Config.Method))
 	}
-	fullPath := c.Config.Url + c.path
+	fullPath := c.url + c.path
 	awsUrl, err := url.Parse(fullPath)
 	if err != nil {
 		return fmt.Errorf("error parsing URL: %w", err)

tests/rest_scripts/command/s3Request.go

@@ -28,6 +28,7 @@ type S3Request struct {
 	dataSource           DataSource
 	canonicalRequestHash string
 	signingKey           []byte
+	url                  string
 }
 
 func (s *S3Request) CalculateDateTimeParams() {
@@ -50,12 +51,21 @@ func (s *S3Request) DeriveHost() error {
 	if len(protocolAndHost) != 2 {
 		return fmt.Errorf("invalid URL value: %s", s.Config.Url)
 	}
-	s.host = protocolAndHost[1]
+	if s.Config.AddressFormat == AddressFormatVirtual && s.Config.BucketName != "" {
+		s.host = fmt.Sprintf("%s.%s", s.Config.BucketName, protocolAndHost[1])
+		s.url = strings.Join([]string{protocolAndHost[0], s.host}, "://")
+	} else {
+		s.host = protocolAndHost[1]
+		s.url = s.Config.Url
+	}
 	return nil
 }
 
 func (s *S3Request) DeriveBucketAndKeyPath() {
-	s.path = "/" + s.Config.BucketName
+	s.path = "/"
+	if s.Config.BucketName != "" && s.Config.AddressFormat == AddressFormatPath {
+		s.path += s.Config.BucketName
+	}
 	if s.Config.ObjectKey != "" {
 		s.path += "/" + s.Config.ObjectKey
 	}

tests/rest_scripts/command/s3RequestBuilder.go

@@ -12,6 +12,11 @@ const (
 	OPENSSL = "openssl"
 )
 
+const (
+	AddressFormatPath    = "path"
+	AddressFormatVirtual = "virtual"
+)
+
 const (
 	UnsignedPayload                            = "UNSIGNED-PAYLOAD"
 	StreamingAWS4HMACSHA256Payload             = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
@@ -80,6 +85,7 @@ type S3RequestConfigData struct {
 	WriteXMLPayloadToFile        string
 	OutputFile                   string
 	HeaderFile                   string
+	AddressFormat                string
 }
 
 type S3RequestBuilder struct {

tests/rest_scripts/generateCommand.go

@@ -82,6 +82,8 @@ var paramSeparator *string
 
 var writeXMLPayloadToFile *string
 
+var addressFormat *string
+
 func (r *restParams) String() string {
 	return fmt.Sprintf("%v", *r)
 }
@@ -158,6 +160,7 @@ func main() {
 			WriteXMLPayloadToFile: *writeXMLPayloadToFile,
 			OutputFile:            *outputFile,
 			HeaderFile:            *headerFile,
+			AddressFormat:         *addressFormat,
 		},
 	}
 
@@ -282,6 +285,7 @@ func checkFlags() error {
 	headerFile = flag.String("headerFile", "", "for curl commands, location to save header file data (to stdout or outputFile if empty)")
 	flag.Var(&objectsToDelete, "objectsToDelete", "Objects to delete in DeleteObjects command (can add multiple)")
 	deleteObjectsQuietMode = flag.Bool("deleteObjectsQuietMode", false, "Quiet mode for DeleteObjects command")
+	addressFormat = flag.String("addressFormat", command.AddressFormatPath, "S3 command address format ('path' or 'virtual')")
 	// Parse the flags
 	flag.Parse()
 
@@ -292,6 +296,11 @@ func checkFlags() error {
 		if f.Name == "locationConstraint" {
 			locationConstraintSet = true
 		}
+		if f.Name == "addressFormat" {
+			if f.Value.String() != command.AddressFormatPath && f.Value.String() != command.AddressFormatVirtual {
+				panic("Invalid address format: " + f.Value.String())
+			}
+		}
 	})
 
 	return nil