diff --git a/tests/README.md b/tests/README.md index a7c0fb2e..30ab7431 100644 --- a/tests/README.md +++ b/tests/README.md @@ -81,4 +81,56 @@ For the s3 backend, see the **S3 Backend** instructions above. If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries. -A single instance can be run with `docker-compose -f docker-compose-bats.yml up ` \ No newline at end of file +A single instance can be run with `docker-compose -f docker-compose-bats.yml up ` + +## Environment Parameters + +**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**: identical to the same parameters in **s3**. + +**VERSITY_EXE**: location of the versity executable relative to test folder. + +**RUN_VERSITYGW**: whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**. + +**BACKEND**: the storage backend type for the gateway, e.g. **posix** or **s3**. + +**LOCAL_FOLDER**: if running with a **posix** backend, the backend storage folder. + +**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**: test bucket names. + +**RECREATE_BUCKETS**: whether to delete buckets between tests. If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted. + +**CERT**, **KEY**: certificate and key locations if using SSL. + +**S3CMD_CONFIG**: location of **s3cmd** config file if running **s3cmd** tests. + +**SECRETS_FILE**: file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored. + +**MC_ALIAS**: Minio MC alias if running MC tests. + +**LOG_LEVEL**: level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing) + +**GOCOVERDIR**: folder to put golang coverage info in, if checking coverage info. + +**USERS_FOLDER**: folder to use if storing IAM data in a folder. + +**IAM_TYPE**: how to store IAM data (**s3** or **folder**). + +**TEST_LOG_FILE**: log file location for these bats tests. + +**VERSITY_LOG_FILE**: log file for versity application as it is tested by bats tests. + +**DIRECT**: if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes). + +**DIRECT_DISPLAY_NAME**: username if **DIRECT** is set to **true**. + +**COVERAGE_DB**: database to store client command coverage info and usage counts, if using. + +**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**: credentials for users created and tested for non-root user **versitygw** operations. + +**TEST_FILE_FOLDER**: where to put temporary test files. + +**REMOVE_TEST_FILE_FOLDER**: whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files. + +**VERSIONING_DIR**: where to put gateway file versioning info. + +**COMMAND_LOG**: where to store list of client commands, which if using will be reported during test failures. diff --git a/tests/commands/abort_multipart_upload.sh b/tests/commands/abort_multipart_upload.sh index 38950811..20c9d0c2 100644 --- a/tests/commands/abort_multipart_upload.sh +++ b/tests/commands/abort_multipart_upload.sh @@ -39,4 +39,4 @@ abort_multipart_upload_with_user() { return 1 fi return 0 -} \ No newline at end of file +} diff --git a/tests/commands/create_multipart_upload.sh b/tests/commands/create_multipart_upload.sh index 46ec4d6a..60ab176d 100644 --- a/tests/commands/create_multipart_upload.sh +++ b/tests/commands/create_multipart_upload.sh @@ -107,3 +107,19 @@ create_multipart_upload_custom() { log 5 "upload id: $upload_id" return 0 } + +create_multipart_upload_rest() { + if [ $# -ne 2 ]; then + log 2 "'create_multipart_upload_rest' requires bucket name, key" + return 1 + fi + if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then + log 2 "error creating multipart upload: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")" + return 1 + fi + return 0 +} diff --git a/tests/commands/get_bucket_versioning.sh b/tests/commands/get_bucket_versioning.sh index ae6ad09d..debc49ab 100644 --- a/tests/commands/get_bucket_versioning.sh +++ b/tests/commands/get_bucket_versioning.sh @@ -37,40 +37,12 @@ get_bucket_versioning_rest() { log 2 "'get_bucket_versioning_rest' requires bucket name" return 1 fi - - #generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - # shellcheck disable=SC2154 - canonical_request="GET -/$1 -versioning= -host:$aws_endpoint_url_address -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -UNSIGNED-PAYLOAD" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OUTPUT_FILE="$TEST_FILE_FOLDER/versioning.txt" ./tests/rest_scripts/get_bucket_versioning.sh); then + log 2 "error getting bucket versioning: $result" return 1 fi - get_signature - # shellcheck disable=SC2154 - if ! reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?versioning" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ - -H "x-amz-date: $current_date_time" \ - -o "$TEST_FILE_FOLDER/versioning.txt" 2>&1); then - log 2 "error retrieving curl reply: $reply" - return 1 - fi - log 5 "reply: $reply" - if [[ "$reply" != "200" ]]; then - log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/versioning.txt")" + if [ "$result" != "200" ]; then + log 2 "get-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/versioning.txt")" return 1 fi return 0 diff --git a/tests/commands/get_object_legal_hold.sh b/tests/commands/get_object_legal_hold.sh index b530e586..269a9ff7 100644 --- a/tests/commands/get_object_legal_hold.sh +++ b/tests/commands/get_object_legal_hold.sh @@ -33,43 +33,13 @@ get_object_legal_hold_rest() { log 2 "'get_object_legal_hold_rest' requires bucket, key" return 1 fi - - generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - # shellcheck disable=SC2154 - canonical_request="GET -/$1/$2 -legal-hold= -host:$aws_endpoint_url_address -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then + log 2 "error getting object legal hold: $result" return 1 fi - get_signature - # shellcheck disable=SC2154 - reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?legal-hold" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -o "$TEST_FILE_FOLDER"/object_legal_hold.txt 2>&1) - log 5 "reply status code: $reply" - if [[ "$reply" != "200" ]]; then - if [ "$reply" == "404" ]; then - return 1 - fi - log 2 "reply error: $reply" - log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_legal_hold.txt)" - return 2 + if [ "$result" != "200" ]; then + log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")" + return 1 fi - log 5 "object legal hold: $(cat "$TEST_FILE_FOLDER"/object_legal_hold.txt)" return 0 } \ No newline at end of file diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index 3afa77ad..028a098c 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -117,30 +117,14 @@ list_buckets_s3api() { } list_buckets_rest() { - generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - # shellcheck disable=SC2154 - canonical_request="GET -/ - -host:${AWS_ENDPOINT_URL#*//} -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG=$COMMAND_LOG OUTPUT_FILE="$TEST_FILE_FOLDER/buckets.txt" ./tests/rest_scripts/list_buckets.sh); then + log 2 "error listing buckets: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "list-buckets returned code $result: $(cat "$TEST_FILE_FOLDER/buckets.txt")" return 1 fi - - get_signature - # shellcheck disable=SC2034,SC2154 - reply=$(send_command curl -ks "$AWS_ENDPOINT_URL" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" 2>&1) parse_bucket_list + return 0 } diff --git a/tests/commands/put_bucket_versioning.sh b/tests/commands/put_bucket_versioning.sh index 4b390047..910cfcbb 100644 --- a/tests/commands/put_bucket_versioning.sh +++ b/tests/commands/put_bucket_versioning.sh @@ -36,47 +36,12 @@ put_bucket_versioning_rest() { log 2 "'put_bucket_versioning_rest' requires bucket, 'Enabled' or 'Suspended'" return 1 fi - - versioning=" - - $2 -" - - generate_hash_for_payload "$versioning" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - content_md5=$(echo -n "$versioning" | openssl dgst -binary -md5 | openssl base64) - # shellcheck disable=SC2154 - canonical_request="PUT -/$1 -versioning= -content-md5:$content_md5 -host:$aws_endpoint_url_address -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -content-md5;host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 STATUS="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_bucket_versioning.sh); then + log 2 "error putting bucket versioning: $result" return 1 fi - get_signature - - # shellcheck disable=SC2154 - reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1?versioning" \ - -H "Content-MD5: $content_md5" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -d "$versioning" -o "$TEST_FILE_FOLDER"/put_versioning_error.txt 2>&1) - log 5 "reply status code: $reply" - if [[ "$reply" != "200" ]]; then - log 2 "reply error: $reply" - log 2 "put bucket versioning command returned error: $(cat "$TEST_FILE_FOLDER"/put_versioning_error.txt)" + if [ "$result" != "200" ]; then + log 2 "put-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")" return 1 fi return 0 diff --git a/tests/commands/put_object_retention.sh b/tests/commands/put_object_retention.sh index e329fb2a..eeb04830 100644 --- a/tests/commands/put_object_retention.sh +++ b/tests/commands/put_object_retention.sh @@ -33,49 +33,12 @@ put_object_retention_rest() { log 2 "'put_object_retention_rest' requires bucket, key, retention mode, retention date" return 1 fi - - retention=" - - $3 - $4 -" - - log 5 "retention payload: $retention" - generate_hash_for_payload "$retention" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - content_md5=$(echo -n "$retention" | openssl dgst -binary -md5 | openssl base64) - # shellcheck disable=SC2154 - canonical_request="PUT -/$1/$2 -retention= -content-md5:$content_md5 -host:$aws_endpoint_url_address -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -content-md5;host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" RETENTION_MODE="$3" RETAIN_UNTIL_DATE="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_object_retention.sh); then + log 2 "error putting object retention: $result" return 1 fi - get_signature - - # shellcheck disable=SC2154 - reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?retention" \ - -H "Content-MD5: $content_md5" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -d "$retention" -o "$TEST_FILE_FOLDER"/put_object_retention_error.txt 2>&1) - log 5 "reply status code: $reply" - if [[ "$reply" != "200" ]]; then - log 2 "reply error: $reply" - log 2 "put object retention command returned error: $(cat "$TEST_FILE_FOLDER"/put_object_retention_error.txt)" + if [ "$result" != "200" ]; then + log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")" return 1 fi return 0 diff --git a/tests/commands/upload_part.sh b/tests/commands/upload_part.sh index 54e8bdf1..da376ca0 100644 --- a/tests/commands/upload_part.sh +++ b/tests/commands/upload_part.sh @@ -30,4 +30,4 @@ upload_part() { return 1 fi export etag -} \ No newline at end of file +} diff --git a/tests/rest_scripts/abort_multipart_upload.sh b/tests/rest_scripts/abort_multipart_upload.sh new file mode 100755 index 00000000..4d5c6100 --- /dev/null +++ b/tests/rest_scripts/abort_multipart_upload.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153,SC2034 +upload_id="$UPLOAD_ID" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="DELETE +/$bucket_name/$key +uploadId=$UPLOAD_ID +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "https://$host/$bucket_name/$key?uploadId=$upload_id" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/create_multipart_upload.sh b/tests/rest_scripts/create_multipart_upload.sh new file mode 100755 index 00000000..d9ab4bf6 --- /dev/null +++ b/tests/rest_scripts/create_multipart_upload.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" + +# Step 1: generate canonical request hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="POST +/$bucket_name/$key +uploads= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploads=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/get_bucket_versioning.sh b/tests/rest_scripts/get_bucket_versioning.sh index 98ac90bf..10463110 100755 --- a/tests/rest_scripts/get_bucket_versioning.sh +++ b/tests/rest_scripts/get_bucket_versioning.sh @@ -17,22 +17,8 @@ source ./tests/rest_scripts/rest.sh # Fields - -payload="" -# shellcheck disable=SC2153 -aws_access_key_id="$AWS_ACCESS_KEY_ID" -# shellcheck disable=SC2153 -aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" -get_host -get_aws_region - -# Step 1: generate payload hash - -payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" - -# Step 2: generate canonical hash current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") @@ -40,36 +26,18 @@ canonical_request="GET /$bucket_name versioning= host:$host -x-amz-content-sha256:$payload_hash +x-amz-content-sha256:UNSIGNED-PAYLOAD x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date -$payload_hash" +UNSIGNED-PAYLOAD" -canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +create_canonical_hash_sts_and_signature -# Step 3: create STS data string - -year_month_day="$(echo "$current_date_time" | cut -c1-8)" - -sts_data="AWS4-HMAC-SHA256 -$current_date_time -$year_month_day/$aws_region/s3/aws4_request -$canonical_request_hash" - -# Step 4: generate signature - -date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') -date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') -date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') -signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') -signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ - -mac HMAC \ - -macopt hexkey:"$signing_key" | awk '{print $2}') - -# Step 5: send curl command - -curl -ks "https://$host/$BUCKET_NAME?versioning" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ No newline at end of file +curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?versioning=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh old mode 100644 new mode 100755 index a7c97727..93b9cf2d --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -18,58 +18,29 @@ source ./tests/rest_scripts/rest.sh # Fields -payload="" -# shellcheck disable=SC2153 -aws_access_key_id="$AWS_ACCESS_KEY_ID" -# shellcheck disable=SC2153 -aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" -get_host -get_aws_region - -# Step 1: generate payload hash - -payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" - -# Step 2: generate canonical hash +# shellcheck disable=SC2154 +key="$OBJECT_KEY" current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request="GET -/$bucket_name +/$bucket_name/$key legal-hold= host:$host -x-amz-content-sha256:$payload_hash +x-amz-content-sha256:UNSIGNED-PAYLOAD x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date -$payload_hash" +UNSIGNED-PAYLOAD" -canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +create_canonical_hash_sts_and_signature -# Step 3: create STS data string - -year_month_day="$(echo "$current_date_time" | cut -c1-8)" - -sts_data="AWS4-HMAC-SHA256 -$current_date_time -$year_month_day/$aws_region/s3/aws4_request -$canonical_request_hash" - -# Step 4: generate signature - -date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') -date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') -date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') -signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') -signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ - -mac HMAC \ - -macopt hexkey:"$signing_key" | awk '{print $2}') - -# Step 5: send curl command - -curl -ks "https://$host/$BUCKET_NAME?legal-hold" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ No newline at end of file +curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/list_buckets.sh b/tests/rest_scripts/list_buckets.sh index aa431308..8ad4cc22 100755 --- a/tests/rest_scripts/list_buckets.sh +++ b/tests/rest_scripts/list_buckets.sh @@ -16,58 +16,24 @@ source ./tests/rest_scripts/rest.sh -# Fields - -payload="" -# shellcheck disable=SC2153 -aws_access_key_id="$AWS_ACCESS_KEY_ID" -# shellcheck disable=SC2153 -aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" -get_host -get_aws_region - -# Step 1: generate payload hash - -payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" - -# Step 2: generate canonical hash - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request="GET / host:$host -x-amz-content-sha256:$payload_hash +x-amz-content-sha256:UNSIGNED-PAYLOAD x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date -$payload_hash" +UNSIGNED-PAYLOAD" -canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +create_canonical_hash_sts_and_signature -# Step 3: create STS data string - -year_month_day="$(echo "$current_date_time" | cut -c1-8)" - -sts_data="AWS4-HMAC-SHA256 -$current_date_time -$year_month_day/$aws_region/s3/aws4_request -$canonical_request_hash" - -# Step 4: generate signature - -date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') -date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') -date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') -signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') -signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ - -mac HMAC \ - -macopt hexkey:"$signing_key" | awk '{print $2}') - -# Step 5: send curl command - -curl -ks "https://$host/" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ No newline at end of file +curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/list_parts.sh b/tests/rest_scripts/list_parts.sh new file mode 100755 index 00000000..20eb5560 --- /dev/null +++ b/tests/rest_scripts/list_parts.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +upload_id="$UPLOAD_ID" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +# shellcheck disable=SC2034 +canonical_request="GET +/$bucket_name/$key +uploadId=$upload_id +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploadId=$upload_id" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/put_bucket_versioning.sh b/tests/rest_scripts/put_bucket_versioning.sh index 9d52c6bc..8432ce5a 100755 --- a/tests/rest_scripts/put_bucket_versioning.sh +++ b/tests/rest_scripts/put_bucket_versioning.sh @@ -21,15 +21,7 @@ source ./tests/rest_scripts/rest.sh # shellcheck disable=SC2153 status="$STATUS" # shellcheck disable=SC2153 -aws_access_key_id="$AWS_ACCESS_KEY_ID" -# shellcheck disable=SC2153 -aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" -# shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" -get_host -get_aws_region - -# Step 1: generate payload hash payload=" @@ -37,9 +29,6 @@ payload=" " payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" - -# Step 2: generate canonical hash - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request="PUT @@ -52,31 +41,13 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" -canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +create_canonical_hash_sts_and_signature -# Step 3: create STS data string - -year_month_day="$(echo "$current_date_time" | cut -c1-8)" - -sts_data="AWS4-HMAC-SHA256 -$current_date_time -$year_month_day/$aws_region/s3/aws4_request -$canonical_request_hash" - -# Step 4: generate signature - -date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') -date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') -date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') -signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') -signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ - -mac HMAC \ - -macopt hexkey:"$signing_key" | awk '{print $2}') - -# Step 5: send curl command - -curl -ks -X PUT "https://$host/$BUCKET_NAME?versioning" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -d "$payload" \ No newline at end of file +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?versioning=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"" +-d "\"${payload//\"/\\\"}\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/put_object_retention.sh b/tests/rest_scripts/put_object_retention.sh index f54aa250..f7135f02 100755 --- a/tests/rest_scripts/put_object_retention.sh +++ b/tests/rest_scripts/put_object_retention.sh @@ -26,14 +26,6 @@ key="$OBJECT_KEY" mode="$RETENTION_MODE" # shellcheck disable=SC2153 retain_until_date="$RETAIN_UNTIL_DATE" -# shellcheck disable=SC2153 -aws_access_key_id="$AWS_ACCESS_KEY_ID" -# shellcheck disable=SC2153 -aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" -get_host -get_aws_region - -# Step 1: generate payload hash payload=" @@ -43,9 +35,6 @@ payload=" payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64) - -# Step 2: generate canonical hash - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request="PUT @@ -59,32 +48,14 @@ x-amz-date:$current_date_time content-md5;host;x-amz-content-sha256;x-amz-date $payload_hash" -canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +create_canonical_hash_sts_and_signature -# Step 3: create STS data string - -year_month_day="$(echo "$current_date_time" | cut -c1-8)" - -sts_data="AWS4-HMAC-SHA256 -$current_date_time -$year_month_day/$aws_region/s3/aws4_request -$canonical_request_hash" - -# Step 4: generate signature - -date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') -date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') -date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') -signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') -signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ - -mac HMAC \ - -macopt hexkey:"$signing_key" | awk '{print $2}') - -# Step 5: send curl command - -curl -ks -X PUT "https://$host/$bucket_name/$key?retention" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "Content-MD5: $content_md5" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -d "$payload" +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?retention=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"Content-MD5: $content_md5\"" +-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"" +-d "\"${payload//\"/\\\"}\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh index ef1e6327..eb20ea0b 100644 --- a/tests/rest_scripts/rest.sh +++ b/tests/rest_scripts/rest.sh @@ -14,20 +14,54 @@ # specific language governing permissions and limitations # under the License. -get_host() { - if [ -z "$HOST" ]; then - host="localhost:7070" - return - fi - # shellcheck disable=SC2034 - host="$HOST" -} +source ./tests/commands/command.sh -get_aws_region() { - if [ -z "$AWS_REGION" ]; then - aws_region="us-east-1" - return - fi +# shellcheck disable=SC2153,SC2034 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153,SC2034 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" + +if [ -z "$AWS_ENDPOINT_URL" ]; then + host="localhost:7070" +else + # shellcheck disable=SC2034 + host="$(echo "$AWS_ENDPOINT_URL" | awk -F'//' '{print $2}')" +fi + +if [ -z "$AWS_REGION" ]; then + aws_region="us-east-1" +else # shellcheck disable=SC2034 aws_region="$AWS_REGION" +fi + +add_command_recording_if_enabled() { + if [ -n "$COMMAND_LOG" ]; then + curl_command+=(send_command) + fi +} + +create_canonical_hash_sts_and_signature() { + # shellcheck disable=SC2154 + canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + + # shellcheck disable=SC2154 + year_month_day="$(echo "$current_date_time" | cut -c1-8)" + + sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + + date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') + date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') + date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') + signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') + # shellcheck disable=SC2034 + signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + + curl_command=() + add_command_recording_if_enabled } diff --git a/tests/test_common.sh b/tests/test_common.sh index dc2bc3be..bfc09509 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -373,112 +373,6 @@ test_common_get_bucket_location() { assert_success } -test_put_bucket_acl_s3cmd() { - if [[ $DIRECT != "true" ]]; then - # https://github.com/versity/versitygw/issues/695 - skip - fi - run setup_bucket "s3cmd" "$BUCKET_ONE_NAME" - assert_success - - run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" - assert_success - - username=$USERNAME_ONE - if [[ $DIRECT != "true" ]]; then - run setup_user "$username" "HIJKLMN" "user" - assert_success - fi - sleep 5 - - run get_check_default_acl_s3cmd "$BUCKET_ONE_NAME" - assert_success - - if [[ $DIRECT == "true" ]]; then - run put_public_access_block_enable_public_acls "$BUCKET_ONE_NAME" - assert_success - fi - run put_bucket_canned_acl_s3cmd "$BUCKET_ONE_NAME" "--acl-public" - assert_success - - run get_check_post_change_acl_s3cmd "$BUCKET_ONE_NAME" - assert_success -} - -test_common_put_bucket_acl() { - if [[ $RECREATE_BUCKETS == "false" ]]; then - # https://github.com/versity/versitygw/issues/716 - skip - fi - assert [ $# -eq 1 ] - - run setup_bucket "$1" "$BUCKET_ONE_NAME" - assert_success - - run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" - assert_success - - username=$USERNAME_ONE - run setup_user "$username" "HIJKLMN" "user" - assert_success - - run get_check_acl_id "$1" "$BUCKET_ONE_NAME" - assert_success - - acl_file="test-acl" - run create_test_files "$acl_file" - assert_success - - if [[ $DIRECT == "true" ]]; then - grantee="{\"Type\": \"Group\", \"URI\": \"http://acs.amazonaws.com/groups/global/AllUsers\"}" - else - grantee="{\"ID\": \"$username\", \"Type\": \"CanonicalUser\"}" - fi - -cat < "$TEST_FILE_FOLDER"/"$acl_file" - { - "Grants": [ - { - "Grantee": $grantee, - "Permission": "READ" - } - ], - "Owner": { - "ID": "$AWS_ACCESS_KEY_ID" - } - } -EOF - - run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" - assert_success - - run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME" - assert_success - -cat < "$TEST_FILE_FOLDER"/"$acl_file" - { - "Grants": [ - { - "Grantee": { - "ID": "$username", - "Type": "CanonicalUser" - }, - "Permission": "FULL_CONTROL" - } - ], - "Owner": { - "ID": "$AWS_ACCESS_KEY_ID" - } - } -EOF - - run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" - assert_success - - run get_check_acl_after_second_put "$1" "$BUCKET_ONE_NAME" - assert_success -} - test_common_get_put_delete_bucket_policy() { assert [ $# -eq 1 ] diff --git a/tests/test_common_acl.sh b/tests/test_common_acl.sh new file mode 100644 index 00000000..7efa65c3 --- /dev/null +++ b/tests/test_common_acl.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +test_put_bucket_acl_s3cmd() { + if [[ $DIRECT != "true" ]]; then + skip "https://github.com/versity/versitygw/issues/695" + fi + run setup_bucket "s3cmd" "$BUCKET_ONE_NAME" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + username=$USERNAME_ONE + if [[ $DIRECT != "true" ]]; then + run setup_user "$username" "HIJKLMN" "user" + assert_success + fi + sleep 5 + + run get_check_default_acl_s3cmd "$BUCKET_ONE_NAME" + assert_success + + if [[ $DIRECT == "true" ]]; then + run put_public_access_block_enable_public_acls "$BUCKET_ONE_NAME" + assert_success + fi + run put_bucket_canned_acl_s3cmd "$BUCKET_ONE_NAME" "--acl-public" + assert_success + + run get_check_post_change_acl_s3cmd "$BUCKET_ONE_NAME" + assert_success +} + +test_common_put_bucket_acl() { + if [[ $RECREATE_BUCKETS == "false" ]]; then + skip "https://github.com/versity/versitygw/issues/716" + fi + assert [ $# -eq 1 ] + + run setup_bucket "$1" "$BUCKET_ONE_NAME" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + username=$USERNAME_ONE + run setup_user "$username" "HIJKLMN" "user" + assert_success + + run get_check_acl_id "$1" "$BUCKET_ONE_NAME" + assert_success + + acl_file="test-acl" + run create_test_files "$acl_file" + assert_success + + if [[ $DIRECT == "true" ]]; then + grantee="{\"Type\": \"Group\", \"URI\": \"http://acs.amazonaws.com/groups/global/AllUsers\"}" + else + grantee="{\"ID\": \"$username\", \"Type\": \"CanonicalUser\"}" + fi + +cat < "$TEST_FILE_FOLDER"/"$acl_file" + { + "Grants": [ + { + "Grantee": $grantee, + "Permission": "READ" + } + ], + "Owner": { + "ID": "$AWS_ACCESS_KEY_ID" + } + } +EOF + + run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" + assert_success + + run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME" + assert_success + +cat < "$TEST_FILE_FOLDER"/"$acl_file" + { + "Grants": [ + { + "Grantee": { + "ID": "$username", + "Type": "CanonicalUser" + }, + "Permission": "FULL_CONTROL" + } + ], + "Owner": { + "ID": "$AWS_ACCESS_KEY_ID" + } + } +EOF + + run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" + assert_success + + run get_check_acl_after_second_put "$1" "$BUCKET_ONE_NAME" + assert_success +} \ No newline at end of file diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 6f115def..da088a80 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -1,5 +1,20 @@ #!/usr/bin/env bats +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/commands/create_multipart_upload.sh source ./tests/commands/delete_object_tagging.sh source ./tests/commands/get_bucket_versioning.sh source ./tests/commands/get_object.sh @@ -224,7 +239,9 @@ source ./tests/util_versioning.sh } @test "versioning - retrieve after delete" { - skip "https://github.com/versity/versitygw/issues/888" + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/888" + fi test_file="test_file" run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -264,3 +281,13 @@ source ./tests/util_versioning.sh run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" assert_success } + +@test "REST - multipart upload create then abort" { + test_file="test_file" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run create_abort_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" + assert_success +} diff --git a/tests/test_s3api.sh b/tests/test_s3api.sh index 040a3a62..2fa0209f 100755 --- a/tests/test_s3api.sh +++ b/tests/test_s3api.sh @@ -25,6 +25,7 @@ source ./tests/util_tags.sh source ./tests/util_users.sh source ./tests/test_aws_root_inner.sh source ./tests/test_common.sh +source ./tests/test_common_acl.sh source ./tests/commands/copy_object.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/delete_object_tagging.sh @@ -234,18 +235,21 @@ export RUN_USERS=true run create_test_file "$bucket_file" assert_success - dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 || fail "error adding data to test file" + + run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 + assert_success run setup_bucket "aws" "$BUCKET_ONE_NAME" assert_success - multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 || fail "error performing multipart upload" + run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 + assert_success - get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy" || fail "error getting object" - compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file || fail "data doesn't match" + run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy" + assert_success - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file + run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file + assert_success } @test "test_multipart_upload_from_bucket_range_too_large" { @@ -256,13 +260,8 @@ export RUN_USERS=true run setup_bucket "aws" "$BUCKET_ONE_NAME" assert_success - multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 "bytes=0-1000000000" || local upload_result=$? - [[ $upload_result -eq 1 ]] || fail "multipart upload with overly large range should have failed" - log 5 "error: $upload_part_copy_error" - [[ $upload_part_copy_error == *"Range specified is not valid"* ]] || [[ $upload_part_copy_error == *"InvalidRange"* ]] || fail "unexpected error: $upload_part_copy_error" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file + run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" + assert_success } @test "test_multipart_upload_from_bucket_range_valid" { diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 03a4cbca..2947eb35 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -16,6 +16,7 @@ source ./tests/setup.sh source ./tests/test_common.sh +source ./tests/test_common_acl.sh source ./tests/util.sh source ./tests/util_create_bucket.sh source ./tests/util_users.sh diff --git a/tests/util.sh b/tests/util.sh index 8cc739d3..a3e0c75e 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -29,10 +29,8 @@ source ./tests/commands/delete_object.sh source ./tests/commands/get_bucket_acl.sh source ./tests/commands/get_bucket_ownership_controls.sh source ./tests/commands/get_bucket_policy.sh -source ./tests/commands/get_bucket_tagging.sh source ./tests/commands/get_object_legal_hold.sh source ./tests/commands/get_object_lock_configuration.sh -source ./tests/commands/get_object_tagging.sh source ./tests/commands/head_bucket.sh source ./tests/commands/head_object.sh source ./tests/commands/list_multipart_uploads.sh @@ -720,85 +718,6 @@ get_object_acl() { export acl } -check_tags_empty() { - if [[ $# -ne 1 ]]; then - echo "check tags empty requires command type" - return 1 - fi - if [[ $1 == 'aws' ]]; then - if [[ $tags != "" ]]; then - tag_set=$(echo "$tags" | jq '.TagSet') - if [[ $tag_set != "[]" ]]; then - echo "error: tags not empty: $tags" - return 1 - fi - fi - else - if [[ $tags != "" ]] && [[ $tags != *"No tags found"* ]]; then - echo "Error: tags not empty: $tags" - return 1 - fi - fi - return 0 -} - -check_object_tags_empty() { - if [[ $# -ne 3 ]]; then - echo "bucket tags empty check requires command type, bucket, and key" - return 2 - fi - if ! get_object_tagging "$1" "$2" "$3"; then - echo "failed to get tags" - return 2 - fi - check_tags_empty "$1" || local check_result=$? - # shellcheck disable=SC2086 - return $check_result -} - -check_bucket_tags_empty() { - if [[ $# -ne 2 ]]; then - echo "bucket tags empty check requires command type, bucket" - return 2 - fi - if ! get_bucket_tagging "$1" "$2"; then - echo "failed to get tags" - return 2 - fi - check_tags_empty "$1" || local check_result=$? - # shellcheck disable=SC2086 - return $check_result -} - -get_and_verify_object_tags() { - if [[ $# -ne 5 ]]; then - echo "get and verify object tags missing command type, bucket, key, tag key, tag value" - return 1 - fi - get_object_tagging "$1" "$2" "$3" || get_result=$? - if [[ $get_result -ne 0 ]]; then - echo "failed to get tags" - return 1 - fi - if [[ $1 == 'aws' ]]; then - tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key') - tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value') - if [[ $tag_set_key != '"'$4'"' ]]; then - echo "Key mismatch ($tag_set_key, \"$4\")" - return 1 - fi - if [[ $tag_set_value != '"'$5'"' ]]; then - echo "Value mismatch ($tag_set_value, \"$5\")" - return 1 - fi - else - read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')" - [[ $tag_set_key == "$4" ]] || fail "Key mismatch" - [[ $tag_set_value == "$5" ]] || fail "Value mismatch" - fi - return 0 -} - # copy a file to/from S3 # params: source, destination # return 0 for success, 1 for failure diff --git a/tests/util_file.sh b/tests/util_file.sh index d7608a84..6f39f321 100644 --- a/tests/util_file.sh +++ b/tests/util_file.sh @@ -124,6 +124,10 @@ delete_test_files() { # param: file location # return 0 for success, 1 for error split_file() { + if [ $# -ne 2 ]; then + log 2 "'split_file' requires file name, number of pieces" + return 1 + fi file_size=$(stat -c %s "$1" 2>/dev/null || stat -f %z "$1" 2>/dev/null) part_size=$((file_size / $2)) remainder=$((file_size % $2)) diff --git a/tests/util_legal_hold.sh b/tests/util_legal_hold.sh index b8a1c87b..31f504a2 100644 --- a/tests/util_legal_hold.sh +++ b/tests/util_legal_hold.sh @@ -46,8 +46,8 @@ check_legal_hold_without_lock_enabled() { log 2 "get legal hold using REST succeeded without lock enabled" return 1 fi - log 5 "legal hold info: $(cat "$TEST_FILE_FOLDER/object_legal_hold.txt")" - if ! code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/object_legal_hold.txt" 2>&1); then + log 5 "legal hold info: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")" + if ! code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/legal_hold.txt" 2>&1); then log 2 "error getting error code: $code" return 1 fi diff --git a/tests/util_multipart.sh b/tests/util_multipart.sh index 12274b4f..8908fc11 100644 --- a/tests/util_multipart.sh +++ b/tests/util_multipart.sh @@ -552,3 +552,61 @@ abort_all_multipart_uploads() { fi done } + +create_upload_and_get_id_rest() { + if [ $# -ne 2 ]; then + log 2 "'create_upload_and_get_id_rest' requires bucket, key" + return 1 + fi + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then + log 2 "error creating multipart upload: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")" + return 1 + fi + log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")" + if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then + log 2 "error getting upload ID: $upload_id" + return 1 + fi + log 5 "upload ID: $upload_id" + return 0 +} + +create_abort_multipart_upload_rest() { + if [ $# -ne 2 ]; then + log 2 "'create_abort_upload_rest' requires bucket, key" + return 1 + fi + if ! create_upload_and_get_id_rest "$1" "$2"; then + log 2 "error creating upload" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then + log 2 "error aborting multipart upload: $result" + return 1 + fi + if [ "$result" != "204" ]; then + log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/output.txt)" + return 1 + fi +} + +multipart_upload_range_too_large() { + if [ $# -ne 3 ]; then + log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location" + return 1 + fi + if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then + log 2 "multipart upload succeeded despite overly large range" + return 1 + fi + log 5 "error: $upload_part_copy_error" + if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then + log 2 "unexpected error: $upload_part_copy_error" + return 1 + fi + return 0 +} diff --git a/tests/util_rest.sh b/tests/util_rest.sh index 4f29de5a..4ea4cbb0 100644 --- a/tests/util_rest.sh +++ b/tests/util_rest.sh @@ -2,8 +2,8 @@ parse_bucket_list() { # shellcheck disable=SC2154 - log 5 "bucket list: $reply" - bucket_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' -) + log 5 "bucket list: $(cat "$TEST_FILE_FOLDER/buckets.txt")" + bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$TEST_FILE_FOLDER/buckets.txt") bucket_array=() while read -r bucket; do bucket_array+=("$bucket") @@ -12,6 +12,7 @@ parse_bucket_list() { } parse_object_list() { + # shellcheck disable=SC2154 object_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' -) object_array=() while read -r object; do @@ -67,7 +68,6 @@ generate_hash_for_payload_file() { } get_creq_string_list_buckets() { - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request="GET / @@ -151,4 +151,3 @@ $creq_hash" echo "TEST STS" cat test.sts } - diff --git a/tests/util_tags.sh b/tests/util_tags.sh index 2c72931e..190384bf 100644 --- a/tests/util_tags.sh +++ b/tests/util_tags.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_object_tagging.sh + # params: username, password, bucket, expected key, expected value # return 0 for success, 1 for failure get_and_check_bucket_tags_with_user() { @@ -171,3 +173,83 @@ parse_object_tags_rest() { fi return 0 } + +check_tags_empty() { + if [[ $# -ne 1 ]]; then + echo "check tags empty requires command type" + return 1 + fi + if [[ $1 == 'aws' ]]; then + if [[ $tags == "" ]]; then + return 0 + fi + tag_set=$(echo "$tags" | jq '.TagSet') + if [[ $tag_set != "[]" ]]; then + echo "error: tags not empty: $tags" + return 1 + fi + else + if [[ $tags != "" ]] && [[ $tags != *"No tags found"* ]]; then + echo "Error: tags not empty: $tags" + return 1 + fi + fi + return 0 +} + +check_object_tags_empty() { + if [[ $# -ne 3 ]]; then + echo "bucket tags empty check requires command type, bucket, and key" + return 2 + fi + if ! get_object_tagging "$1" "$2" "$3"; then + echo "failed to get tags" + return 2 + fi + check_tags_empty "$1" || local check_result=$? + # shellcheck disable=SC2086 + return $check_result +} + +check_bucket_tags_empty() { + if [[ $# -ne 2 ]]; then + echo "bucket tags empty check requires command type, bucket" + return 2 + fi + if ! get_bucket_tagging "$1" "$2"; then + echo "failed to get tags" + return 2 + fi + check_tags_empty "$1" || local check_result=$? + # shellcheck disable=SC2086 + return $check_result +} + +get_and_verify_object_tags() { + if [[ $# -ne 5 ]]; then + echo "get and verify object tags missing command type, bucket, key, tag key, tag value" + return 1 + fi + get_object_tagging "$1" "$2" "$3" || get_result=$? + if [[ $get_result -ne 0 ]]; then + echo "failed to get tags" + return 1 + fi + if [[ $1 == 'aws' ]]; then + tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key') + tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value') + if [[ $tag_set_key != '"'$4'"' ]]; then + echo "Key mismatch ($tag_set_key, \"$4\")" + return 1 + fi + if [[ $tag_set_value != '"'$5'"' ]]; then + echo "Value mismatch ($tag_set_value, \"$5\")" + return 1 + fi + else + read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')" + [[ $tag_set_key == "$4" ]] || fail "Key mismatch" + [[ $tag_set_value == "$5" ]] || fail "Value mismatch" + fi + return 0 +}