test: list-buckets tests

tests/rest_scripts/README.md

@@ -0,0 +1,33 @@
+# REST scripts
+
+## Parameters
+
+The common S3 parameters for the scripts can be found in **rest.sh**.  They are:
+* AWS_ACCESS_KEY_ID
+* AWS_SECRET_ACCESS_KEY
+* AWS_ENDPOINT_URL
+* AWS_REGION
+
+The `OUTPUT_FILE` parameter can be set to write the response data to a file.
+
+Operation-specific parameters can be found by looking at the tops of the scripts.  For example, for the CreateBucket operation, this would be `BUCKET_NAME`.
+
+## cURL
+
+Most scripts will send a cURL message to an s3 server.  The scripts all use the following parameters:
+
+In the root folder, for example, the **create_bucket.sh** script can be run with:
+
+`AWS_ACCESS_KEY_ID=<key> AWS_SECRET_ACCESS_KEY=<key> AWS_ENDPOINT_URL=<url> AWS_REGION=<region> BUCKET_NAME=<name> OUTPUT_FILE=<file> ./tests/rest_scripts/create_bucket.sh`
+
+A successful bucket creation will return a 200 code on the command line.
+
+## openssl
+
+Some scripts will generate a raw REST API message that can be sent via openssl.
+
+For example, to create this file with **put_object_openssl.sh**:
+
+`AWS_ACCESS_KEY_ID=<key> AWS_SECRET_ACCESS_KEY=<key> AWS_ENDPOINT_URL=<url> AWS_REGION=<region> DATA_FILE=<data_file> BUCKET_NAME=<name> OBJECT_KEY=<desired key name> COMMAND_FILE=<output file> ./tests/rest_scripts/put_object_openssl.sh`
+
+This should generate a raw REST command file in the `COMMAND_FILE` location.  This command can be sent to the S3 server with:  `openssl s_client -connect <server host and port> -ign_eof < <command file>`

tests/rest_scripts/complete_multipart_upload.sh

@@ -34,6 +34,8 @@ checksum_algorithm="$CHECKSUM_ALGORITHM"
 checksum_hash="$CHECKSUM_HASH"
 # shellcheck disable=SC2154
 algorithm_parameter="${ALGORITHM_PARAMETER:=false}"
+# shellcheck disable=SC2153
+multipart_object_size="$MULTIPART_OBJECT_SIZE"
 
 payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
 <CompleteMultipartUpload xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">$parts</CompleteMultipartUpload>"
@@ -53,6 +55,9 @@ if [ "$checksum_type" != "" ]; then
   cr_data+=("x-amz-checksum-type:$checksum_type")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
+if [ "$multipart_object_size" != "" ]; then
+  cr_data+=("x-amz-mp-object-size:$multipart_object_size")
+fi
 build_canonical_request "${cr_data[@]}"
 
 # shellcheck disable=SC2119

tests/rest_scripts/create_bucket.sh

@@ -44,7 +44,7 @@ build_canonical_request "${cr_data[@]}"
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name")
+curl_command+=(curl -ks -w "%{http_code}" -X PUT "$AWS_ENDPOINT_URL/$bucket_name")
 curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")

tests/rest_scripts/list_buckets.sh

@@ -16,25 +16,45 @@
 
 source ./tests/rest_scripts/rest.sh
 
+# shellcheck disable=SC2153
+prefix="$PREFIX"
+# shellcheck disable=SC2153
+max_buckets="$MAX_BUCKETS"
+# shellcheck disable=SC2153
+continuation_token="$CONTINUATION_TOKEN"
+
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
-canonical_request="GET
-/
-
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
+canonical_request_data=("GET" "/")
+queries=""
+if [ "$continuation_token" != "" ]; then
+  queries=$(add_parameter "$queries" "continuation-token=$continuation_token")
+fi
+if [ "$max_buckets" != "" ]; then
+  queries=$(add_parameter "$queries" "max-buckets=$max_buckets")
+fi
+if [ "$prefix" != "" ]; then
+  queries=$(add_parameter "$queries" "prefix=$prefix")
+fi
+canonical_request_data+=("$queries")
+canonical_request_data+=("host:$host" "x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
 
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host"
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
--H "\"x-amz-date: $current_date_time\""
--o "$OUTPUT_FILE")
+curl_command+=(curl -ks -w "\"%{http_code}\"")
+url="'$AWS_ENDPOINT_URL"
+if [ "$queries" != "" ]; then
+  url+="?$queries"
+fi
+url+="'"
+curl_command+=("$url")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
 # shellcheck disable=SC2154
-eval "${curl_command[*]}" 2>&1
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/list_objects.sh

@@ -63,7 +63,7 @@ if [ "$queries" != "" ]; then
 fi
 url+="'"
 curl_command+=("$url")
-curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")
 # shellcheck disable=SC2154