diff --git a/tests/README.md b/tests/README.md index 273f9eb..b9b86d0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -151,11 +151,13 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **DIRECT**: if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes). -**DIRECT_DISPLAY_NAME**: username if **DIRECT** is set to **true**. +**DIRECT_DISPLAY_NAME**: AWS ACL main user display name if **DIRECT** is set to **true**. + +**DIRECT_AWS_USER_ID**: AWS policy 12-digit user ID if **DIRECT** is set to **true**. **COVERAGE_DB**: database to store client command coverage info and usage counts, if using. -**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**: credentials for users created and tested for non-root user **versitygw** operations (non-setup_user_v2). +**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**: setup_user (v1), credentials for users created and tested for non-root user **versitygw** operations (non-setup_user_v2). **TEST_FILE_FOLDER**: where to put temporary test files. @@ -167,13 +169,19 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **TIME_LOG**: optional log to show duration of individual tests -**DIRECT_S3_ROOT_ACCOUNT_NAME**: for direct mode, S3 username +**DIRECT_S3_ROOT_ACCOUNT_NAME**: for direct mode, S3 username for user with root permissions **DELETE_BUCKETS_AFTER_TEST**: whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked -**AUTOCREATE_USERS**: setup_user_v2, whether or not to autocreate users for tests. If set to **false**, users must be pre-created (see `Secret` section above). +**AUTOGENERATE_USERS**: setup_user_v2, whether or not to autocreate users for tests. If set to **false**, users must be pre-created (see `Secret` section above). -**USER_AUTOCREATION_PREFIX**: setup_user_v2, if **AUTOCREATE_USERS** is set to **true**, the prefix for the autocreated username. +**USER_AUTOGENERATION_PREFIX**: setup_user_v2, if **AUTOCREATE_USERS** is set to **true**, the prefix for the autocreated username. + +**CREATE_STATIC_USERS_IF_NONEXISTENT**: setup_user_v2, if **AUTOCREATE_USERS** is set to **false**, generate non-existing users if they don't exist, but don't delete them, as with user autogeneration + +**DIRECT_POST_COMMAND_DELAY**: in direct mode, time to wait before sending new commands to try to prevent propagation delay issues + +**SKIP_ACL_TESTING**: avoid ACL tests for systems which do not use ACLs ## REST Scripts diff --git a/tests/commands/command.sh b/tests/commands/command.sh index 3538ce7..af77b11 100644 --- a/tests/commands/command.sh +++ b/tests/commands/command.sh @@ -37,5 +37,8 @@ send_command() { echo " ($1 response code: $command_result)" fi fi + if [ "$DIRECT" == "true" ]; then + sleep "$DIRECT_POST_COMMAND_DELAY" + fi return $command_result } \ No newline at end of file diff --git a/tests/env.sh b/tests/env.sh index 4c9da8a..d828d30 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -131,6 +131,12 @@ check_universal_vars() { fi export LOG_LEVEL_INT=$LOG_LEVEL fi + if [ "$DIRECT" != "true" ]; then + if [ -z "$DIRECT_POST_COMMAND_DELAY" ]; then + DIRECT_POST_COMMAND_DELAY=0 + fi + export DIRECT_POST_COMMAND_DELAY + fi check_aws_vars diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh index 12345b5..7e23f0d 100755 --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -28,11 +28,10 @@ version_id="$VERSION_ID" current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request_data=("GET" "/$bucket_name/$key") -queries="" +queries="legal-hold=" if [ "$version_id" != "" ]; then queries=$(add_parameter "$queries" "versionId=$version_id") fi -queries=$(add_parameter "$queries" "legal-hold=") canonical_request_data+=("$queries" "host:$host") canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time") if ! build_canonical_request "${canonical_request_data[@]}"; then @@ -42,9 +41,10 @@ fi # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature +log_rest 5 "cr data: $canonical_request" -curl_command+=(curl -ks -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?$queries\"" --H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") +curl_command+=(curl -ks -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?$queries\"") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index f226dba..416f820 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -31,11 +31,7 @@ fi current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") -#x-amz-object-attributes:ETag -canonical_request="GET -/$bucket_name -" - +canonical_request_data=("GET" "/$bucket_name") queries="" if [ "$MARKER" != "" ]; then queries=$(add_parameter "$queries" "marker=$marker") @@ -49,14 +45,12 @@ fi if [ "$max_keys" -ne 0 ]; then queries=$(add_parameter "$queries" "max-keys=$max_keys") fi - -canonical_request+=" -host:$host -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -UNSIGNED-PAYLOAD" +canonical_request_data+=("$queries" "host:$host") +canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time") +if ! build_canonical_request "${canonical_request_data[@]}"; then + log_rest 2 "error building request" + exit 1 +fi # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature @@ -68,9 +62,8 @@ if [ "$queries" != "" ]; then fi url+="'" curl_command+=("$url") -curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" --H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" --H "\"x-amz-date: $current_date_time\"" --o "$OUTPUT_FILE") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"") +curl_command+=("${header_fields[@]}") +curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/put_object_legal_hold.sh b/tests/rest_scripts/put_object_legal_hold.sh index b6385c7..9c99747 100755 --- a/tests/rest_scripts/put_object_legal_hold.sh +++ b/tests/rest_scripts/put_object_legal_hold.sh @@ -28,6 +28,8 @@ status="$STATUS" omit_payload="${OMIT_PAYLOAD:=false}" # shellcheck disable=SC2153 version_id="$VERSION_ID" +# shellcheck disable=SC2153 +omit_content_md5="${OMIT_CONTENT_MD5:=false}" if [ "$omit_payload" == "false" ]; then payload=" @@ -39,15 +41,21 @@ else fi payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" +if [ "$omit_content_md5" == "false" ]; then + content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64) +fi current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request_data=("PUT" "/$bucket_name/$key") -queries="" +queries="legal-hold=" if [ "$version_id" != "" ]; then queries=$(add_parameter "$queries" "versionId=$version_id") fi -queries=$(add_parameter "$queries" "legal-hold=") -canonical_request_data+=("$queries" "host:$host") +canonical_request_data+=("$queries") +if [ "$omit_content_md5" == "false" ]; then + canonical_request_data+=("content-md5:$content_md5") +fi +canonical_request_data+=("host:$host") canonical_request_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time") if ! build_canonical_request "${canonical_request_data[@]}"; then log_rest 2 "error building request" @@ -57,7 +65,7 @@ fi # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?$queries" +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?$queries\"" -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") curl_command+=("${header_fields[@]}") if [ "$omit_payload" == "false" ]; then diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 3f0e9bd..17edb16 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -243,7 +243,7 @@ test_file="test_file" fi test_file_two="test_file_2" test_file_three="test_file_3" - run setup_bucket_and_files "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "$test_file_three" + run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "$test_file_three" assert_success run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" @@ -269,7 +269,7 @@ test_file="test_file" skip "https://github.com/versity/versitygw/issues/999" fi test_file_two="test_file_2" - run setup_bucket_and_files "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" + run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" assert_success run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" @@ -492,9 +492,9 @@ test_file="test_file" assert_success } -@test "REST - PutObjectLegalHold w/o payload" { +@test "REST - PutObjectLegalHold - missing content-md5" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1191" + skip "https://github.com/versity/versitygw/issues/1311" fi run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" assert_success @@ -505,6 +505,20 @@ test_file="test_file" run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success + run check_legal_hold_without_content_md5 "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - PutObjectLegalHold w/o payload" { + run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + run check_legal_hold_without_payload "$BUCKET_ONE_NAME" "$test_file" assert_success } diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index c2f74d1..4d1a792 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -10,8 +10,7 @@ source ./tests/util/util_retention.sh # param: bucket name # return 0 on success, 1 on error reset_bucket() { - log 6 "reset_bucket" - if ! check_param_count "clear_bucket_s3api" "bucket" 1 $#; then + if ! check_param_count "reset_bucket" "bucket" 1 $#; then return 1 fi diff --git a/tests/util/util_legal_hold.sh b/tests/util/util_legal_hold.sh index c0559a8..58446bb 100644 --- a/tests/util/util_legal_hold.sh +++ b/tests/util/util_legal_hold.sh @@ -73,10 +73,6 @@ check_remove_legal_hold_versions() { log 2 "error getting XML legal hold status" return 1 fi - #if ! status="$(echo "$legal_hold" | grep -v "InsecureRequestWarning" | jq -r '.LegalHold.Status' 2>&1)"; then - # log 2 "error getting legal hold status: $status" - # return 1 - #fi if [ "$status" == "ON" ]; then if ! put_object_legal_hold_rest_version_id "$1" "$2" "$3" "OFF"; then log 2 "error removing legal hold of version ID" @@ -99,7 +95,26 @@ check_legal_hold_without_payload() { return 1 fi if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "MalformedXML" "The XML you provided"; then - log 2 "error checking xml error, message" + log 2 "error checking xml error, message ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + return 0 +} + +check_legal_hold_without_content_md5() { + if ! check_param_count "check_legal_hold_without_content_md5" "bucket, key" 2 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OMIT_CONTENT_MD5="true" STATUS="OFF" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh); then + log 2 "error: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "expected '400', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "InvalidRequest" "Content-MD5"; then + log 2 "error checking xml error, message ($(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi return 0 diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index 9d32194..aff21d8 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -251,6 +251,10 @@ list_objects_check_params_get_token() { log 2 "error attempting to get bucket ACL response: $result" return 1 fi + if [ "$result" != "200" ]; then + log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))" + return 1 + fi log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")" if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then log 2 "error getting list bucket result: $list_bucket_result" diff --git a/tests/util/util_list_parts.sh b/tests/util/util_list_parts.sh index 52d588f..2982abe 100644 --- a/tests/util/util_list_parts.sh +++ b/tests/util/util_list_parts.sh @@ -123,7 +123,10 @@ upload_check_parts() { log 2 "error uploading and checking first part" return 1 fi - # shellcheck disable=SC2154 + if ! list_multipart_uploads_rest "$1"; then + log 2 "error listing uploads" + return 1 + fi etag_one=$etag if ! upload_check_part "$1" "$2" "$upload_id" 2 "$4" "$etag_one"; then log 2 "error uploading and checking second part" diff --git a/tests/util/util_lock_config.sh b/tests/util/util_lock_config.sh index 767c12b..317345f 100644 --- a/tests/util/util_lock_config.sh +++ b/tests/util/util_lock_config.sh @@ -20,7 +20,6 @@ get_and_check_object_lock_config() { if ! check_param_count "get_and_check_object_lock_config" "bucket, expected enabled value, expected governance mode, expected days" 4 $#; then return 1 fi - if ! get_object_lock_configuration "s3api" "$1"; then log 2 "error getting object lock config" return 1