diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index d5b9796..fa92e27 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -139,6 +139,7 @@ jobs: - name: Install s3cmd run: | + sudo apt-get update sudo apt-get install s3cmd - name: Install mc @@ -146,10 +147,10 @@ jobs: curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc chmod 755 /usr/local/bin/mc - - name: Install xmllint (for rest) + - name: Install xml libraries (for rest) run: | sudo apt-get update - sudo apt-get install libxml2-utils + sudo apt-get install libxml2-utils xmlstarlet # see https://github.com/versity/versitygw/issues/1034 - name: Install AWS cli diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 5f13469..a9f3c67 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -21,6 +21,7 @@ RUN apt-get update && \ jq \ bc \ libxml2-utils \ + xmlstarlet \ ca-certificates && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index f191b26..1faf4a0 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -20,15 +20,14 @@ source ./tests/report.sh # param: bucket name # return 0 for success, 1 for failure create_bucket() { - if [ $# -ne 2 ]; then - log 2 "create bucket missing command type, bucket name" + log 6 "create_bucket" + if ! check_param_count "create_bucket" "command type, bucket" 2 $#; then return 1 fi record_command "create-bucket" "client:$1" local exit_code=0 local error - log 6 "create bucket" if [[ $1 == 's3' ]]; then error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]]; then @@ -50,8 +49,8 @@ create_bucket() { } create_bucket_with_user() { - if [ $# -ne 4 ]; then - log 2 "create bucket missing command type, bucket name, access, secret" + log 6 "create_bucket_with_user" + if ! check_param_count "create_bucket_with_user" "command type, bucket, access ID, secret key" 4 $#; then return 1 fi local exit_code=0 @@ -73,9 +72,9 @@ create_bucket_with_user() { } create_bucket_object_lock_enabled() { + log 6 "create_bucket_object_lock_enabled" record_command "create-bucket" "client:s3api" - if [ $# -ne 1 ]; then - log 2 "create bucket missing bucket name" + if ! check_param_count "create_bucket_object_lock_enabled" "bucket" 1 $#; then return 1 fi diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index b46bd2b..5e40b86 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -18,8 +18,7 @@ delete_object() { log 6 "delete_object" record_command "delete-object" "client:$1" - if [ $# -ne 3 ]; then - log 2 "delete object command requires command type, bucket, key" + if ! check_param_count "delete_object" "command type, bucket, key" 3 $#; then return 1 fi local exit_code=0 @@ -47,8 +46,7 @@ delete_object() { } delete_object_bypass_retention() { - if [[ $# -ne 4 ]]; then - log 2 "'delete-object with bypass retention' requires bucket, key, user, password" + if ! check_param_count "delete_object_bypass_retention" "bucket, key, user, password" 4 $#; then return 1 fi if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then @@ -59,8 +57,7 @@ delete_object_bypass_retention() { } delete_object_version() { - if [[ $# -ne 3 ]]; then - log 2 "'delete_object_version' requires bucket, key, version ID" + if ! check_param_count "delete_object_version" "bucket, key, version ID" 3 $#; then return 1 fi if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then @@ -71,8 +68,7 @@ delete_object_version() { } delete_object_version_bypass_retention() { - if [[ $# -ne 3 ]]; then - log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID" + if ! check_param_count "delete_object_version_bypass_retention" "bucket, key, version ID" 3 $#; then return 1 fi if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then @@ -84,8 +80,7 @@ delete_object_version_bypass_retention() { delete_object_with_user() { record_command "delete-object" "client:$1" - if [ $# -ne 5 ]; then - log 2 "delete object with user command requires command type, bucket, key, access ID, secret key" + if ! check_param_count "delete_object_version_bypass_retention" "command type, bucket, key, access ID, secret key" 5 $#; then return 1 fi local exit_code=0 @@ -101,47 +96,22 @@ delete_object_with_user() { fi if [ $exit_code -ne 0 ]; then log 2 "error deleting object: $delete_object_error" - export delete_object_error return 1 fi return 0 } delete_object_rest() { - if [ $# -ne 2 ]; then - log 2 "'delete_object_rest' requires bucket name, object name" + if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then return 1 fi - - generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - # shellcheck disable=SC2154 - canonical_request="DELETE -/$1/$2 - -host:$aws_endpoint_url_address -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -UNSIGNED-PAYLOAD" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then + log 2 "error deleting object: $result" return 1 fi - get_signature - # shellcheck disable=SC2154 - reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ - -H "x-amz-date: $current_date_time" \ - -o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1) - if [[ "$reply" != "204" ]]; then - log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)" + if [ "$result" != "204" ]; then + delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt") + log 2 "expected '204', was '$result' ($delete_object_error)" return 1 fi return 0 diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index 7e61cbd..4284479 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -15,18 +15,20 @@ # under the License. get_bucket_policy() { + log 6 "get_bucket_policy '$1' '$2'" record_command "get-bucket-policy" "client:$1" - if [[ $# -ne 2 ]]; then - log 2 "get bucket policy command requires command type, bucket" + if ! check_param_count "get_bucket_policy" "command type, bucket" 2 $#; then return 1 fi local get_bucket_policy_result=0 if [[ $1 == 's3api' ]]; then - get_bucket_policy_aws "$2" || get_bucket_policy_result=$? + get_bucket_policy_s3api "$2" || get_bucket_policy_result=$? elif [[ $1 == 's3cmd' ]]; then get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$? elif [[ $1 == 'mc' ]]; then get_bucket_policy_mc "$2" || get_bucket_policy_result=$? + elif [ "$1" == 'rest' ]; then + get_bucket_policy_rest "$2" || get_bucket_policy_result=$? else log 2 "command 'get bucket policy' not implemented for '$1'" return 1 @@ -38,10 +40,10 @@ get_bucket_policy() { return 0 } -get_bucket_policy_aws() { +get_bucket_policy_s3api() { + log 6 "get_bucket_policy_s3api '$1'" record_command "get-bucket-policy" "client:s3api" - if [[ $# -ne 1 ]]; then - log 2 "aws 'get bucket policy' command requires bucket" + if ! check_param_count "get_bucket_policy_s3api" "bucket" 1 $#; then return 1 fi policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$? @@ -62,8 +64,7 @@ get_bucket_policy_aws() { get_bucket_policy_with_user() { record_command "get-bucket-policy" "client:s3api" - if [[ $# -ne 3 ]]; then - log 2 "'get bucket policy with user' command requires bucket, username, password" + if ! check_param_count "get_bucket_policy_with_user" "bucket, username, password" 3 $#; then return 1 fi if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then @@ -82,8 +83,7 @@ get_bucket_policy_with_user() { get_bucket_policy_s3cmd() { record_command "get-bucket-policy" "client:s3cmd" - if [[ $# -ne 1 ]]; then - log 2 "s3cmd 'get bucket policy' command requires bucket" + if ! check_param_count "get_bucket_policy_s3cmd" "bucket" 1 $#; then return 1 fi @@ -106,8 +106,7 @@ get_bucket_policy_s3cmd() { } get_bucket_policy_rest() { - if [[ $# -ne 1 ]]; then - log 2 "s3cmd 'get bucket policy' command requires bucket name" + if ! check_param_count "get_bucket_policy_rest" "bucket" 1 $#; then return 1 fi if ! get_bucket_policy_rest_expect_code "$1" "200"; then @@ -118,8 +117,7 @@ get_bucket_policy_rest() { } get_bucket_policy_rest_expect_code() { - if [[ $# -ne 2 ]]; then - log 2 "s3cmd 'get bucket policy' command requires bucket name, expected code" + if ! check_param_count "get_bucket_policy_rest_expect_code" "bucket, code" 2 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then @@ -169,8 +167,7 @@ search_for_first_policy_line_or_full_policy() { get_bucket_policy_mc() { record_command "get-bucket-policy" "client:mc" - if [[ $# -ne 1 ]]; then - log 2 "aws 'get bucket policy' command requires bucket" + if ! check_param_count "get_bucket_policy_mc" "bucket" 1 $#; then return 1 fi bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$? diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index eb3ebfa..e47d382 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -14,17 +14,26 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/drivers.sh + get_object_lock_configuration() { record_command "get-object-lock-configuration" "client:s3api" - if [[ $# -ne 1 ]]; then - log 2 "'get object lock configuration' command missing bucket name" + if ! check_param_count "get_object_lock_configuration" "client, bucket name" 2 $#; then return 1 fi - if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then - log 2 "error obtaining lock config: $lock_config" - # shellcheck disable=SC2034 - get_object_lock_config_err=$lock_config - return 1 + if [ "$1" == 'rest' ]; then + if ! get_object_lock_configuration_rest "$2"; then + log 2 "error getting REST object lock configuration" + get_object_lock_config_err=$(cat "$TEST_FILE_FOLDER/object-lock-config.txt") + return 1 + fi + else + if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$2" 2>&1); then + log 2 "error obtaining lock config: $lock_config" + # shellcheck disable=SC2034 + get_object_lock_config_err=$lock_config + return 1 + fi fi lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning") return 0 @@ -32,8 +41,7 @@ get_object_lock_configuration() { get_object_lock_configuration_rest() { log 6 "get_object_lock_configuration_rest" - if [ $# -ne 1 ]; then - log 2 "'get_object_lock_configuration_rest' requires bucket name" + if ! check_param_count "get_object_lock_configuration_rest" "bucket name" 1 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then @@ -44,5 +52,6 @@ get_object_lock_configuration_rest() { log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" return 1 fi + lock_config="$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" return 0 } \ No newline at end of file diff --git a/tests/commands/head_bucket.sh b/tests/commands/head_bucket.sh index 6077091..8c335f2 100644 --- a/tests/commands/head_bucket.sh +++ b/tests/commands/head_bucket.sh @@ -22,10 +22,9 @@ source ./tests/report.sh # 1 - bucket does not exist # 2 - misc error head_bucket() { - log 6 "head_bucket" + log 6 "head_bucket '$1' '$2'" record_command "head-bucket" "client:$1" - if [ $# -ne 2 ]; then - log 2 "'head_bucket' command requires client, bucket name" + if ! check_param_count "head_bucket" "client, bucket name" 2 $#; then return 1 fi local exit_code=0 @@ -37,6 +36,7 @@ head_bucket() { bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then bucket_info=$(head_bucket_rest "$2") || exit_code=$? + log 5 "head bucket rest exit code: $exit_code" return $exit_code else log 2 "invalid command type $1" @@ -54,19 +54,21 @@ head_bucket() { } head_bucket_rest() { - if [ $# -ne 1 ]; then - log 2 "'head_bucket_rest' requires bucket name" + log 6 "head_bucket_rest '$1'" + if ! check_param_count "head_bucket_rest" "bucket" 1 $#; then return 2 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_bucket.sh 2>&1); then - log 2 "error getting head bucket" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_bucket.sh 2>&1); then + log 2 "error getting head bucket: $result" return 2 fi if [ "$result" == "200" ]; then bucket_info="$(cat "$TEST_FILE_FOLDER/result.txt")" echo "$bucket_info" + log 5 "bucket info: $bucket_info" return 0 elif [ "$result" == "404" ]; then + log 5 "bucket '$1' not found" return 1 fi log 2 "unexpected response code '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" diff --git a/tests/commands/list_objects.sh b/tests/commands/list_objects.sh index efb7dbf..9f616a6 100644 --- a/tests/commands/list_objects.sh +++ b/tests/commands/list_objects.sh @@ -22,25 +22,24 @@ source ./tests/commands/command.sh list_objects() { log 6 "list_objects" record_command "list-objects" "client:$1" - if [ $# -ne 2 ]; then - log 2 "'list_objects' command requires client, bucket" + if ! check_param_count "list_object" "client, bucket" 2 $#; then return 1 fi local output - local result=0 + local list_objects_result=0 if [[ $1 == 's3' ]]; then - output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$? + output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || list_objects_result=$? elif [[ $1 == 's3api' ]]; then - list_objects_s3api "$2" || result=$? - return $result + list_objects_s3api "$2" || list_objects_result=$? + return $list_objects_result elif [[ $1 == 's3cmd' ]]; then - output=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$? + output=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || list_objects_result=$? elif [[ $1 == 'mc' ]]; then - output=$(send_command mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$? + output=$(send_command mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || list_objects_result=$? elif [[ $1 == 'rest' ]]; then - list_objects_rest "$2" || result=$? - return $result + list_objects_rest "$2" || list_objects_result=$? + return $list_objects_result else fail "invalid command type $1" return 1 @@ -63,8 +62,7 @@ list_objects() { # fail if unable to list list_objects_s3api() { log 6 "list_objects_s3api" - if [ $# -ne 1 ]; then - log 2 "'list_objects_s3api' requires bucket" + if ! check_param_count "list_objects_s3api" "bucket" 1 $#; then return 1 fi if ! output=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then @@ -107,8 +105,7 @@ list_objects_s3api_v1() { } list_objects_with_prefix() { - if [ $# -ne 3 ]; then - log 2 "'list_objects_with_prefix' command requires, client, bucket, prefix" + if ! check_param_count "list_objects_with_prefix" "client, bucket, prefix" 3 $#; then return 1 fi local result=0 @@ -134,8 +131,7 @@ list_objects_with_prefix() { } list_objects_rest() { - if [ $# -ne 1 ]; then - log 2 "'list_objects_rest' requires bucket name" + if ! check_param_count "list_objects_rest" "bucket" 1 $#; then return 1 fi log 5 "bucket name: $1" diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh index 741b7ed..eb44afc 100644 --- a/tests/commands/put_bucket_policy.sh +++ b/tests/commands/put_bucket_policy.sh @@ -14,10 +14,12 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/drivers.sh + put_bucket_policy() { + log 6 "put_bucket_policy '$1' '$2' '$3'" record_command "put-bucket-policy" "client:$1" - if [[ $# -ne 3 ]]; then - log 2 "'put bucket policy' command requires command type, bucket, policy file" + if ! check_param_count "put_bucket_policy" "command type, bucket, policy file" 3 $#; then return 1 fi local put_policy_result=0 @@ -27,6 +29,9 @@ put_bucket_policy() { policy=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$? elif [[ $1 == 'mc' ]]; then policy=$(send_command mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$? + elif [ "$1" == 'rest' ]; then + put_bucket_policy_rest "$2" "$3" || put_policy_result=$? + return $put_policy_result else log 2 "command 'put bucket policy' not implemented for '$1'" return 1 @@ -46,8 +51,7 @@ put_bucket_policy() { put_bucket_policy_with_user() { record_command "put-bucket-policy" "client:s3api" - if [[ $# -ne 4 ]]; then - log 2 "'put bucket policy with user' command requires bucket, policy file, username, password" + if ! check_param_count "put_bucket_policy_with_user" "bucket, policy file, username, password" 4 $#; then return 1 fi if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then @@ -58,3 +62,18 @@ put_bucket_policy_with_user() { fi return 0 } + +put_bucket_policy_rest() { + if ! check_param_count "put_bucket_policy_rest" "bucket, policy file" 2 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" POLICY_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_policy.sh); then + log 2 "error putting bucket policy: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + return 0 +} diff --git a/tests/commands/put_object_legal_hold.sh b/tests/commands/put_object_legal_hold.sh index 0c9f253..f2fec64 100644 --- a/tests/commands/put_object_legal_hold.sh +++ b/tests/commands/put_object_legal_hold.sh @@ -16,12 +16,33 @@ put_object_legal_hold() { record_command "put-object-legal-hold" "client:s3api" - if [[ $# -ne 3 ]]; then - log 2 "'put object legal hold' command requires bucket, key, hold status ('ON' or 'OFF')" + if ! check_param_count "put_object_legal_hold" "client, bucket, key, hold status ('ON' or 'OFF')" 4 $#; then return 1 fi - if ! error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1); then - log 2 "error putting object legal hold: $error" + if [ "$1" == "rest" ]; then + if ! put_object_legal_hold_rest "$2" "$3" "$4"; then + log 2 "error updating legal hold status w/REST" + return 1 + fi + else + if ! error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$2" --key "$3" --legal-hold "{\"Status\": \"$4\"}" 2>&1); then + log 2 "error putting object legal hold: $error" + return 1 + fi + fi + return 0 +} + +put_object_legal_hold_rest() { + if ! check_param_count "put_object_legal_hold_rest" "bucket, key, hold status ('ON' or 'OFF')" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" STATUS="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh 2>&1); then + log 2 "error putting object legal hold: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi return 0 @@ -29,8 +50,7 @@ put_object_legal_hold() { put_object_legal_hold_version_id() { record_command "put-object-legal-hold" "client:s3api" - if [[ $# -ne 4 ]]; then - log 2 "'put_object_legal_hold_version_id' command requires bucket, key, version ID, hold status ('ON' or 'OFF')" + if ! check_param_count "put_object_legal_hold_version_id" "bucket, key, version ID, hold status ('ON' or 'OFF')" 4 $#; then return 1 fi local error="" diff --git a/tests/rest_scripts/delete_object.sh b/tests/rest_scripts/delete_object.sh new file mode 100755 index 0000000..d51f3e0 --- /dev/null +++ b/tests/rest_scripts/delete_object.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2154 +key=$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")') + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +#x-amz-object-attributes:ETag +canonical_request_data+=("DELETE" "/$bucket_name/$key" "" "host:$host") +canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time") + +build_canonical_request "${canonical_request_data[@]}" + +# shellcheck disable=SC2119 +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") +curl_command+=("${header_fields[@]}") +curl_command+=(-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/put_object_legal_hold.sh b/tests/rest_scripts/put_object_legal_hold.sh index 1886243..9081888 100755 --- a/tests/rest_scripts/put_object_legal_hold.sh +++ b/tests/rest_scripts/put_object_legal_hold.sh @@ -21,7 +21,7 @@ source ./tests/rest_scripts/rest.sh # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" # shellcheck disable=SC2153 -key="$OBJECT_KEY" +key="$(echo -n "$OBJECT_KEY" | jq -sRr @uri)" # shellcheck disable=SC2153 status="$STATUS" # shellcheck disable=SC2153 diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 4e3851f..2cf7e26 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -96,7 +96,7 @@ export RUN_USERS=true run check_verify_object_tags "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$expected_tag_key" "$expected_tag_val" assert_success - run put_object_legal_hold "$BUCKET_ONE_NAME" "$bucket_file" "OFF" + run put_object_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF" assert_success run get_and_check_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF" diff --git a/tests/test_s3api_root_inner.sh b/tests/test_s3api_root_inner.sh index df971b6..8767967 100755 --- a/tests/test_s3api_root_inner.sh +++ b/tests/test_s3api_root_inner.sh @@ -117,7 +117,7 @@ test_get_put_object_legal_hold_s3api_root() { run get_check_object_lock_config_enabled "$BUCKET_ONE_NAME" assert_success - run put_object_legal_hold "$BUCKET_ONE_NAME" "$bucket_file" "ON" + run put_object_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "ON" assert_success run get_and_check_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "ON" @@ -133,7 +133,7 @@ test_get_put_object_legal_hold_s3api_root() { # shellcheck disable=SC2154 assert_output --partial "Object is WORM protected and cannot be overwritten" - run put_object_legal_hold "$BUCKET_ONE_NAME" "$bucket_file" "OFF" + run put_object_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF" assert_success run delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$username" "$password" diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 16daed6..bc59af3 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -27,8 +27,7 @@ delete_bucket_recursive() { # return 0 on success, 1 on error clear_bucket_s3api() { log 6 "clear_bucket_s3api" - if [ $# -ne 1 ]; then - log 2 "'clear_bucket_s3api' requires bucket name" + if ! check_param_count "clear_bucket_s3api" "bucket" 1 $#; then return 1 fi @@ -51,11 +50,21 @@ clear_bucket_s3api() { return 1 fi + if ! abort_all_multipart_uploads "$1"; then + log 2 "error aborting all multipart uploads" + return 1 + fi + if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$1"; then log 2 "error checking ownership rule and resetting acl" return 1 fi + if ! delete_bucket_policy "s3api" "$1"; then + log 2 "error deleting bucket policy" + return 1 + fi + # shellcheck disable=SC2154 if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then log 2 "error disabling object lock config" @@ -72,8 +81,7 @@ clear_bucket_s3api() { # return 0 if able to delete recursively, 1 if not delete_bucket_recursive_s3api() { log 6 "delete_bucket_recursive_s3api" - if [ $# -ne 1 ]; then - log 2 "'delete_bucket_recursive_s3api' requires bucket name" + if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then return 1 fi @@ -112,7 +120,8 @@ bucket_exists() { return 2 fi local exists=0 - head_bucket "s3api" "$1" || exists=$? + head_bucket "rest" "$1" || exists=$? + log 5 "bucket exists response code: $exists" # shellcheck disable=SC2181 if [ $exists -eq 2 ]; then log 2 "unexpected error checking if bucket exists" @@ -125,8 +134,7 @@ bucket_exists() { } direct_wait_for_bucket() { - if [ $# -ne 1 ]; then - log 2 "'direct_wait_for_bucket' requires bucket name" + if ! check_param_count "direct_wait_for_bucket" "bucket" 1 $#; then return 1 fi bucket_verification_start_time=$(date +%s) @@ -154,25 +162,20 @@ bucket_cleanup() { return 1 fi - if ! delete_bucket_policy "s3api" "$1"; then - log 2 "error deleting bucket policy" - return 1 - fi + #if ! delete_bucket_policy "s3api" "$1"; then + # log 2 "error deleting bucket policy" + # return 1 + #fi - if ! get_object_ownership_rule_and_update_acl "$1"; then - log 2 "error getting object ownership rule and updating ACL" - return 1 - fi + #if ! get_object_ownership_rule_and_update_acl "$1"; then + # log 2 "error getting object ownership rule and updating ACL" + # return 1 + #fi - if ! abort_all_multipart_uploads "$1"; then - log 2 "error aborting all multipart uploads" - return 1 - fi - - if [ "$RUN_USERS" == "true" ] && ! reset_bucket_owner "$1"; then - log 2 "error resetting bucket owner" - return 1 - fi + #if [ "$RUN_USERS" == "true" ] && ! reset_bucket_owner "$1"; then + # log 2 "error resetting bucket owner" + # return 1 + #fi log 5 "bucket contents, policy, ACL deletion success" return 0 @@ -193,7 +196,12 @@ bucket_cleanup_if_bucket_exists() { return 1 fi - if [ "$2" == "true" ] || bucket_exists "$1"; then + if [ "$2" == "false" ]; then + log 5 "skipping cleanup, since bucket doesn't exist" + return 0 + fi + + if bucket_exists "$1"; then if ! bucket_cleanup "$1"; then log 2 "error deleting bucket and/or contents" return 1 @@ -270,8 +278,7 @@ setup_bucket() { # param: path of bucket or folder # return 0 for yes, 1 for no, 2 for error bucket_is_accessible() { - if [ $# -ne 1 ]; then - log 2 "bucket accessibility check missing bucket name" + if ! check_param_count "bucket_is_accessible" "bucket" 1 $#; then return 2 fi local exit_code=0 @@ -288,9 +295,8 @@ bucket_is_accessible() { } check_for_empty_region() { - if [ $# -ne 1 ]; then - log 2 "'check_for_empty_region' requires bucket name" - return 1 + if ! check_param_count "check_for_empty_region" "bucket" 1 $#; then + return 2 fi if ! head_bucket "s3api" "$BUCKET_ONE_NAME"; then log 2 "error getting bucket info" diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index eae84fc..9d32194 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -18,18 +18,24 @@ source ./tests/util/util_xml.sh # under the License. parse_objects_list_rest() { - # shellcheck disable=SC2154 - object_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Key"]/text()' -) object_array=() + # shellcheck disable=SC2154 + log 5 "reply: $reply" + if ! object_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Key"]/text()' - 2>&1); then + if [[ "$object_list" == *"XPath set is empty"* ]]; then + return 0 + fi + log 2 "error getting object list: $object_list" + return 1 + fi while read -r object; do - object_array+=("$object") + object_array+=("$(echo -n "$object" | xmlstarlet unesc)") done <<< "$object_list" log 5 "object array: ${object_array[*]}" } list_check_objects_v1() { - if [ $# -ne 5 ]; then - log 2 "'list_check_objects_v1' requires bucket, expected key one, expected size one, expected key two, expected size two" + if ! check_param_count "list_check_objects_v1" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then return 1 fi if ! list_objects_s3api_v1 "$1"; then @@ -44,8 +50,7 @@ list_check_objects_v1() { } check_listed_objects() { - if [ $# -ne 4 ]; then - log 2 "'check_listed_objects' requires expected key one, expected size one, expected key two, expected size two" + if ! check_param_count "check_listed_objects" "expected key one, expected size one, expected key two, expected size two" 4 $#; then return 1 fi # shellcheck disable=SC2154 @@ -84,8 +89,7 @@ check_listed_objects() { } list_check_objects_v2() { - if [ $# -ne 5 ]; then - log 2 "'list_check_objects_v1' requires bucket, expected key one, expected size one, expected key two, expected size two" + if ! check_param_count "list_check_objects_v2" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then return 1 fi if ! list_objects_v2 "$1"; then @@ -100,8 +104,7 @@ list_check_objects_v2() { } list_check_objects_rest() { - if [ $# -ne 1 ]; then - log 2 "'list_check_objects_rest' requires bucket name" + if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then return 1 fi list_objects "rest" "$1" @@ -122,8 +125,7 @@ list_check_objects_rest() { } list_check_objects_common() { - if [ $# -ne 4 ]; then - log 2 "'list_check_objects_common' requires client, bucket, object one, object two" + if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then return 1 fi if ! list_objects "$1" "$2"; then @@ -149,8 +151,7 @@ list_check_objects_common() { } list_objects_check_file_count() { - if [ $# -ne 3 ]; then - log 2 "'list_objects_check_file_count' requires client, bucket, count" + if ! check_param_count "list_objects_check_file_count" "client, bucket, count" 3 $#; then return 1 fi if ! list_objects "$1" "$2"; then @@ -169,8 +170,7 @@ list_objects_check_file_count() { } check_object_listing_with_prefixes() { - if [ $# -ne 3 ]; then - log 2 "'check_object_listing_with_prefixes' requires bucket name, folder name, object name" + if ! check_param_count "check_object_listing_with_prefixes" "bucket, folder name, object name" 3 $#; then return 1 fi if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"; then @@ -201,8 +201,7 @@ check_object_listing_with_prefixes() { } list_objects_with_user_rest_verify_access_denied() { - if [ $# -ne 3 ]; then - log 2 "list_objects_with_user_rest_verify_access_denied' requires bucket, username, password" + if ! check_param_count "list_objects_with_user_rest_verify_access_denied" "bucket, username, password" 3 $#; then return 1 fi if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then @@ -222,8 +221,7 @@ list_objects_with_user_rest_verify_access_denied() { } list_objects_with_user_rest_verify_success() { - if [ $# -ne 4 ]; then - log 2 "list_objects_with_user_rest_verify_access_denied' requires bucket, username, password, expected object" + if ! check_param_count "list_objects_with_user_rest_verify_success" "bucket, username, password, expected object" 4 $#; then return 1 fi if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then @@ -246,8 +244,7 @@ list_objects_with_user_rest_verify_success() { } list_objects_check_params_get_token() { - if [ $# -ne 4 ]; then - log 2 "'list_objects_check_params_get_token' requires bucket name, files, version two" + if ! check_param_count "list_objects_check_params_get_token" "bucket, files, version two" 4 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then @@ -284,8 +281,7 @@ list_objects_check_params_get_token() { } list_objects_check_continuation_error() { - if [ $# -ne 2 ]; then - log 2 "'list_objects_check_continuation_error' requires bucket name, continuation token" + if ! check_param_count "list_objects_check_continuation_error" "bucket, continuation token" 2 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then @@ -303,8 +299,7 @@ list_objects_check_continuation_error() { } list_objects_v1_check_nextmarker_empty() { - if [ $# -ne 1 ]; then - log 2 "'get_next_objects_v1' requires bucket name" + if ! check_param_count "list_objects_v1_check_nextmarker_empty" "bucket" 1 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then @@ -330,8 +325,7 @@ list_objects_v1_check_nextmarker_empty() { } get_delete_marker_and_verify_405() { - if [ $# -ne 2 ]; then - log 2 "'get_delete_marker_and_verify_405' requires bucket name, file name" + if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then return 1 fi if ! list_object_versions_rest "$1"; then diff --git a/tests/util/util_lock_config.sh b/tests/util/util_lock_config.sh index 0fc94cc..79bce7b 100644 --- a/tests/util/util_lock_config.sh +++ b/tests/util/util_lock_config.sh @@ -17,12 +17,10 @@ # params: bucket name, expected enabled value, expected governance mode, expected days # return 0 for success, 1 for failure get_and_check_object_lock_config() { - if [ $# -ne 4 ]; then - log 2 "'get_and_check_lock_config' requires bucket name, expected enabled value, expected governance mode, expected days" + if ! check_param_count "get_and_check_object_lock_config" "bucket, expected enabled value, expected governance mode, expected days" 4 $#; then return 1 fi - - if ! get_object_lock_configuration "$1"; then + if ! get_object_lock_configuration "s3api" "$1"; then log 2 "error getting object lock config" return 1 fi @@ -64,11 +62,10 @@ get_and_check_object_lock_config() { } get_check_object_lock_config_enabled() { - if [ $# -ne 1 ]; then - log 2 "'get_check_object_lock_config_enabled' requires bucket name" + if ! check_param_count "get_check_object_lock_config_enabled" "bucket" 1 $#; then return 1 fi - if ! get_object_lock_configuration "$1"; then + if ! get_object_lock_configuration "s3api" "$1"; then log 2 "error getting lock configuration" return 1 fi @@ -86,8 +83,7 @@ get_check_object_lock_config_enabled() { } check_no_object_lock_config_rest() { - if [ $# -ne 1 ]; then - log 2 "'get_check_object_lock_config_rest' requires bucket name" + if ! check_param_count "check_no_object_lock_config_rest" "bucket" 1 $#; then return 1 fi if get_object_lock_configuration_rest "$1"; then @@ -112,6 +108,9 @@ check_no_object_lock_config_rest() { } check_object_lock_config_enabled_rest() { + if ! check_param_count "check_object_lock_config_enabled_rest" "bucket" 1 $#; then + return 1 + fi if [ $# -ne 1 ]; then log 2 "'get_check_object_lock_config_rest' requires bucket name" return 1 diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index 185784c..9cac37b 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -15,8 +15,7 @@ # under the License. multipart_upload_s3api_complete_from_bucket() { - if [ $# -ne 3 ]; then - log 2 "'multipart_upload_s3api_complete_from_bucket' requires bucket, copy source, part count" + if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count" 3 $#; then return 1 fi parts="[" @@ -42,8 +41,7 @@ multipart_upload_s3api_complete_from_bucket() { } multipart_upload_from_bucket() { - if [ $# -ne 4 ]; then - log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count" + if ! check_param_count "multipart_upload_from_bucket" "bucket, copy source, key, part count" 4 $#; then return 1 fi @@ -73,8 +71,7 @@ multipart_upload_from_bucket() { } split_and_put_file() { - if [ $# -ne 4 ]; then - log 2 "'split_and_put_file' requires bucket, key, copy source, part count" + if ! check_param_count "split_and_put_file" "bucket, key, copy source, part count" 4 $#; then return 1 fi if ! split_file "$3" "$4"; then @@ -92,8 +89,7 @@ split_and_put_file() { } multipart_upload_from_bucket_range() { - if [ $# -ne 5 ]; then - log 2 "multipart upload from bucket with range command requires bucket, copy source, key, part count, and range" + if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then return 1 fi if ! split_file "$3" "$4"; then @@ -133,8 +129,7 @@ multipart_upload_from_bucket_range() { } multipart_upload_custom() { - if [ $# -lt 4 ]; then - log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional additional params" + if ! check_param_count_gt "multipart_upload_custom" "bucket, key, file, part count, optional additional parameters" 4 $$; then return 1 fi @@ -153,8 +148,7 @@ multipart_upload_custom() { } multipart_upload() { - if [ $# -ne 4 ]; then - log 2 "multipart upload command missing bucket, key, file, and/or part count" + if ! check_param_count "multipart_upload" "bucket, key, file, part count" 4 $#; then return 1 fi @@ -174,8 +168,7 @@ multipart_upload() { # params: bucket, key, source file location, number of parts # return 0 for success, 1 for failure multipart_upload_with_params() { - if [ $# -ne 10 ]; then - log 2 "multipart upload command requires bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" + if ! check_param_count "multipart_upload_with_params" "bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" 10 $#; then return 1 fi log 5 "1: $1, 2: $2, 3: $3, 4: $4, 5: $5, 6: $6, 7: $7, 8: $8, 9: $9, 10: ${10}" @@ -194,8 +187,7 @@ multipart_upload_with_params() { } run_and_verify_multipart_upload_with_valid_range() { - if [ $# -ne 3 ]; then - log 2 "'run_and_verify_multipart_upload_with_valid_range' requires bucket, key, 5MB file" + if ! check_param_count "run_and_verify_multipart_upload_with_valid_range" "bucket, key, 5MB file" 3 $#; then return 1 fi range_max=$((5*1024*1024-1)) @@ -220,8 +212,7 @@ run_and_verify_multipart_upload_with_valid_range() { } create_upload_part_copy_rest() { - if [ $# -ne 3 ]; then - log 2 "'create_upload_part_copy_rest' requires bucket, key, >20MB file" + if ! check_param_count "create_upload_part_copy_rest" "bucket, key, >20MB file" 3 $#; then return 1 fi if ! split_and_put_file "$1" "$2" "$3" 4; then @@ -263,8 +254,7 @@ create_upload_part_copy_rest() { } create_upload_finish_wrong_etag() { - if [ $# -ne 2 ]; then - log 2 "'create_upload_finish_wrong_etag' requires bucket, key" + if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then return 1 fi @@ -307,8 +297,7 @@ create_upload_finish_wrong_etag() { } setup_multipart_upload_with_params() { - if [ $# -ne 2 ]; then - log 2 "'setup_multipart_upload_with_params' requires bucket name, file name" + if ! check_param_count "setup_multipart_upload_with_params" "bucket, key" 2 $#; then return 1 fi os_name="$(uname)" diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 02a1fba..6932437 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -50,11 +50,11 @@ source ./tests/util/util_users.sh # param: bucket name # return 0 for success, 1 for failure list_and_delete_objects() { - if [ $# -ne 1 ]; then - log 2 "'list_and_delete_objects' missing bucket name" + log 6 "list_and_delete_objects" + if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then return 1 fi - if ! list_objects 's3api' "$1"; then + if ! list_objects 'rest' "$1"; then log 2 "error getting object list" return 1 fi @@ -75,12 +75,12 @@ list_and_delete_objects() { } check_object_lock_config() { - if [ $# -ne 1 ]; then - log 2 "'check_object_lock_config' requires bucket name" + log 6 "check_object_lock_config" + if ! check_param_count "check_object_lock_config" "bucket" 1 $#; then return 1 fi lock_config_exists=true - if ! get_object_lock_configuration "$1"; then + if ! get_object_lock_configuration "rest" "$1"; then # shellcheck disable=SC2154 if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then # shellcheck disable=SC2034 @@ -97,11 +97,10 @@ check_object_lock_config() { # return 0 for success, 1 for error clear_object_in_bucket() { log 6 "clear_object_in_bucket" - if [ $# -ne 2 ]; then - log 2 "'clear_object_in_bucket' requires bucket, object name" + if ! check_param_count "clear_object_in_bucket" "bucket, key" 2 $#; then return 1 fi - if ! delete_object 's3api' "$1" "$2"; then + if ! delete_object 'rest' "$1" "$2"; then # shellcheck disable=SC2154 log 2 "error deleting object $2: $delete_object_error" if ! check_for_and_remove_worm_protection "$1" "$2" "$delete_object_error"; then @@ -116,9 +115,9 @@ clear_object_in_bucket() { # param: command, object path # return 0 for true, 1 for false, 2 for error object_exists() { - if [ $# -ne 3 ]; then - log 2 "object exists check missing command, bucket name, object name" - return 2 + log 6 "object_exists" + if ! check_param_count "object_exists" "command type, bucket, key" 3 $#; then + return 1 fi head_object "$1" "$2" "$3" || local head_object_result=$? if [[ $head_object_result -eq 2 ]]; then @@ -130,8 +129,8 @@ object_exists() { } put_object_with_metadata() { - if [ $# -ne 6 ]; then - log 2 "put object command requires command type, source, destination, key, metadata key, metadata value" + log 6 "put_object_with_metadata" + if ! check_param_count "put_object_with_metadata" "command type, source, destination, key, metadata key, metadata value" 6 $#; then return 1 fi @@ -152,8 +151,8 @@ put_object_with_metadata() { } get_object_metadata() { - if [ $# -ne 3 ]; then - log 2 "get object metadata command requires command type, bucket, key" + log 6 "get_object_metadata" + if ! check_param_count "get_object_metadata" "command type, bucket, key" 3 $#; then return 1 fi @@ -179,8 +178,8 @@ get_object_metadata() { # params: source file, destination copy location # return 0 for success or already exists, 1 for failure check_and_put_object() { - if [ $# -ne 3 ]; then - log 2 "check and put object function requires source, bucket, destination" + log 6 "check_and_put_object" + if ! check_param_count "check_and_put_object" "source, bucket, destination" 3 $#; then return 1 fi object_exists "s3api" "$2" "$3" || local exists_result=$? @@ -202,9 +201,9 @@ check_and_put_object() { # param: path of object # return 0 for yes, 1 for no, 2 for error object_is_accessible() { - if [ $# -ne 2 ]; then - log 2 "object accessibility check missing bucket and/or key" - return 2 + log 6 "object_is_accessible" + if ! check_param_count "object_is_accessible" "bucket, key" 2 $#; then + return 1 fi local exit_code=0 object_data=$(aws --no-verify-ssl s3api head-object --bucket "$1" --key "$2" 2>&1) || exit_code="$?" @@ -223,8 +222,8 @@ object_is_accessible() { # params: source, destination # return 0 for success, 1 for failure copy_file() { - if [ $# -ne 2 ]; then - log 2 "copy file command requires src and dest" + log 6 "copy_file" + if ! check_param_count "copy_file" "source, destination" 2 $#; then return 1 fi @@ -238,9 +237,8 @@ copy_file() { } list_and_check_directory_obj() { - #assert [ $# -eq 2 ] - if [ $# -ne 2 ]; then - log 2 "'list_and_check_directory_obj' requires client, file name" + log 6 "list_and_check_directory_obj" + if ! check_param_count "list_and_check_directory_obj" "client, file name" 2 $#; then return 1 fi if ! list_objects_with_prefix "$1" "$BUCKET_ONE_NAME" "$2/"; then @@ -269,8 +267,8 @@ list_and_check_directory_obj() { } check_checksum_invalid_or_incorrect() { - if [ $# -ne 6 ]; then - log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum type, checksum, expected error" + log 6 "check_checksum_invalid_or_incorrect" + if ! check_param_count "check_checksum_invalid_or_incorrect" "data file, bucket name, key, checksum type, checksum, expected error" 6 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" CHECKSUM="$5" ./tests/rest_scripts/put_object.sh 2>&1); then @@ -289,8 +287,8 @@ check_checksum_invalid_or_incorrect() { } put_object_rest_checksum() { - if [ $# -ne 4 ]; then - log 2 "'put_object_rest_checksum' requires data file, bucket name, key, checksum type" + log 6 "put_object_rest_checksum" + if ! check_param_count "put_object_rest_checksum" "data file, bucket name, key, checksum type" 4 $#; then return 1 fi # shellcheck disable=SC2097,SC2098 @@ -306,8 +304,8 @@ put_object_rest_checksum() { } check_checksum_rest_invalid() { - if [ $# -ne 1 ]; then - log 2 "'check_checksum_rest_invalid' requires checksum type" + log 6 "check_checksum_rest_invalid" + if ! check_param_count "check_checksum_rest_invalid" "checksum type" 1 $#; then return 1 fi test_file="test_file" @@ -323,8 +321,7 @@ check_checksum_rest_invalid() { } check_checksum_rest_incorrect() { - if [ $# -ne 1 ]; then - log 2 "'check_checksum_rest_incorrect' requires checksum type" + if ! check_param_count "check_checksum_rest_incorrect" "checksum type" 1 $#; then return 1 fi test_file="test_file" @@ -346,8 +343,7 @@ check_checksum_rest_incorrect() { } calculate_incorrect_checksum() { - if [ $# -ne 2 ]; then - log 2 "'calculate_incorrect_checksum' requires checksum type, data" + if ! check_param_count "calculate_incorrect_checksum" "checksum type, data" 2 $#; then return 1 fi case "$1" in @@ -381,8 +377,7 @@ calculate_incorrect_checksum() { } put_object_rest_chunked_payload_type_without_content_length() { - if [ $# -ne 3 ]; then - log 2 "'put_object_rest_diff_payload_type' requires data file, bucket name, key" + if ! check_param_count "put_object_rest_chunked_payload_type_without_content_length" "data file, bucket name, key" 3 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" PAYLOAD="STREAMING-AWS4-HMAC-SHA256-PAYLOAD" ./tests/rest_scripts/put_object.sh 2>&1); then @@ -397,8 +392,7 @@ put_object_rest_chunked_payload_type_without_content_length() { } add_correct_checksum() { - if [ $# -ne 1 ]; then - log 2 "'add_correct_checksum' requires checksum type" + if ! check_param_count "add_correct_checksum" "checksum type" 1 $#; then return 1 fi test_file="test_file" @@ -415,8 +409,7 @@ add_correct_checksum() { } check_invalid_checksum_type() { - if [ $# -ne 3 ]; then - log 2 "'check_invalid_checksum_type' requires data file, bucket name, file" + if ! check_param_count "check_invalid_checksum_type" "data file, bucket name, file" 3 $#; then return 1 fi error_message='The algorithm type you specified in x-amz-checksum- header is invalid.' @@ -427,8 +420,7 @@ check_invalid_checksum_type() { } put_object_rest_check_expires_header() { - if [ $# -ne 3 ]; then - log 2 "'put_object-put_object_rest_check_expires_header' requires data file, bucket, key" + if ! check_param_count "put_object_rest_check_expires_header" "data file, bucket name, key" 3 $#; then return 1 fi expiry_date="Tue, 11 Mar 2025 16:00:00 GMT" diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index 9760961..ba5049c 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -15,8 +15,7 @@ # under the License. check_for_empty_policy() { - if [[ $# -ne 2 ]]; then - log 2 "check for empty policy command requires command type, bucket name" + if ! check_param_count "check_for_empty_policy" "command type, bucket name" 2 $#; then return 1 fi @@ -52,8 +51,7 @@ add_direct_user_to_principal() { get_modified_principal() { log 6 "get_modified_principal" - if [ $# -ne 1 ]; then - log 2 "'get_modified_principal' requires principal string" + if ! check_param_count "get_modified_principal" "principal string" 1 $#; then return 1 fi IFS=',' read -r -a principals <<< "$1" @@ -87,8 +85,7 @@ get_modified_principal() { get_modified_action() { log 6 "get_modified_action" - if [ $# -ne 1 ]; then - log 2 "'get_modified_action' requires action" + if ! check_param_count "get_modified_action" "action" 1 $#; then return 1 fi local first_char="${1:0:1}" @@ -104,8 +101,7 @@ get_modified_action() { # fail on error setup_policy_with_single_statement() { log 6 "setup_policy_with_single_statement" - if [ $# -ne 6 ]; then - log 2 "'setup_policy_with_single_statement' requires policy file, version, effect, principal, action, resource" + if ! check_param_count "setup_policy_with_single_statement" "policy file, version, effect, principal, action, resource" 6 $#; then return 1 fi log 5 "policy file: $1" @@ -137,8 +133,7 @@ setup_policy_with_single_statement() { # return 0 on success, 1 on error setup_policy_with_double_statement() { log 6 "setup_policy_with_double_statement" - if [ $# -ne 10 ]; then - log 2 "invalid number of parameters" + if ! check_param_count "setup_policy_with_double_statement" "policy file, version, one set of: 'effect, principal, action, resource', another set" 10 $#; then return 1 fi if ! get_modified_principal "$4"; then @@ -175,8 +170,7 @@ EOF" } get_and_check_policy() { - if [ $# -ne 6 ]; then - log 2 "'get_and_check_policy' requires client, bucket, expected effect, principal, action, resource" + if ! check_param_count "get_and_check_policy" "client, bucket, expected effect, principal, action, resource" 6 $#; then return 1 fi if ! get_bucket_policy "$1" "$BUCKET_ONE_NAME"; then @@ -194,8 +188,7 @@ get_and_check_policy() { } check_policy() { - if [ $# -ne 5 ]; then - log 2 "'check_policy' requires policy, expected effect, policy, action, resource" + if ! check_param_count "check_policy" "policy, expected effect, principal, action, resource" 5 $#; then return 1 fi log 5 "policy: $1" @@ -246,8 +239,7 @@ check_policy() { } put_and_check_for_malformed_policy() { - if [ $# -ne 2 ]; then - log 2 "'put_and_check_for_malformed_policy' requires bucket name, policy file" + if ! check_param_count "put_and_check_for_malformed_policy" "bucket, policy file" 2 $#; then return 1 fi if put_bucket_policy "s3api" "$1" "$2"; then @@ -263,8 +255,7 @@ put_and_check_for_malformed_policy() { } get_and_check_no_policy_error() { - if [ $# -ne 1 ]; then - log 2 "'get_and_check_no_policy_error' requires bucket name" + if ! check_param_count "get_and_check_no_policy_error" "bucket" 1 $#; then return 1 fi if ! get_bucket_policy_rest_expect_code "$1" "404"; then @@ -284,8 +275,7 @@ get_and_check_no_policy_error() { } get_and_compare_policy_with_file() { - if [ $# -ne 4 ]; then - log 2 "'get_and_compare_policies' requires bucket, username, password, filename" + if ! check_param_count "get_and_compare_policy_with_file" "bucket, username, password, filename" 4 $#; then return 1 fi # shellcheck disable=SC2002 @@ -313,8 +303,7 @@ get_and_compare_policy_with_file() { } put_and_check_policy_rest() { - if [ $# -ne 6 ]; then - log 2 "'put_policy_rest' requires bucket name, policy file, effect, principal, action, resource" + if ! check_param_count "put_and_check_policy_rest" "bucket, policy file, effect, principal, action, resource" 6 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" POLICY_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_policy.sh); then @@ -343,11 +332,11 @@ put_and_check_policy_rest() { } log_bucket_policy() { - if [ $# -ne 1 ]; then - log 2 "'log_bucket_policy' requires bucket name" - return + log 6 "log_bucket_policy" + if ! check_param_count "log_bucket_policy" "bucket" 1 $#; then + return 1 fi - if ! get_bucket_policy "s3api" "$1"; then + if ! get_bucket_policy "rest" "$1"; then log 2 "error getting bucket policy" return fi diff --git a/tests/util/util_retention.sh b/tests/util/util_retention.sh index 64557c5..ecedbf7 100644 --- a/tests/util/util_retention.sh +++ b/tests/util/util_retention.sh @@ -17,8 +17,8 @@ # params: bucket name # return 0 for success, 1 for error add_governance_bypass_policy() { - if [[ $# -ne 1 ]]; then - log 2 "'add governance bypass policy' command requires bucket name" + log 6 "add_governance_bypass_policy '$1'" + if ! check_param_count "add_governance_bypass_policy" "bucket" 1 $#; then return 1 fi cat < "$TEST_FILE_FOLDER/policy-bypass-governance.txt" @@ -34,7 +34,7 @@ add_governance_bypass_policy() { ] } EOF - if ! put_bucket_policy "s3api" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then + if ! put_bucket_policy "rest" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then log 2 "error putting governance bypass policy" return 1 fi @@ -44,14 +44,14 @@ EOF # params: bucket, object, possible WORM error after deletion attempt # return 0 for success, 1 for no WORM protection, 2 for error check_for_and_remove_worm_protection() { - if [ $# -ne 3 ]; then - log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error" - return 2 + log 6 "check_for_and_remove_worm_protection" + if ! check_param_count "check_for_and_remove_worm_protection" "bucket, key, error" 3 $#; then + return 1 fi if [[ $3 == *"WORM"* ]]; then log 5 "WORM protection found" - if ! put_object_legal_hold "$1" "$2" "OFF"; then + if ! put_object_legal_hold "rest" "$1" "$2" "OFF"; then log 2 "error removing object legal hold" return 2 fi @@ -76,6 +76,10 @@ check_for_and_remove_worm_protection() { # params: bucket name, object log_worm_protection() { + log 5 "log_worm_protection" + if ! check_param_count "log_worm_protection" "bucket, key" 2 $#; then + return 1 + fi if ! get_object_legal_hold "$1" "$2"; then log 2 "error getting object legal hold status" return @@ -94,8 +98,7 @@ log_worm_protection() { } retention_rest_without_request_body() { - if [ $# -ne 2 ]; then - log 2 "'retention_rest_without_request_body' requires bucket name, key" + if ! check_param_count "retention_rest_without_request_body" "bucket, key" 2 $#; then return 1 fi if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OMIT_PAYLOAD="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_retention.sh); then diff --git a/tests/util/util_setup.sh b/tests/util/util_setup.sh index 52de37a..02d2b45 100644 --- a/tests/util/util_setup.sh +++ b/tests/util/util_setup.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash setup_bucket_and_file() { - if [ $# -ne 2 ]; then - log 2 "'setup_bucket_and_file' requires bucket name, file name" + if ! check_param_count "setup_bucket_and_file" "bucket, file name" 2 $#; then return 1 fi if ! setup_bucket_and_files "$1" "$2"; then @@ -13,8 +12,7 @@ setup_bucket_and_file() { } setup_bucket_and_files() { - if [ $# -lt 2 ]; then - log 2 "'setup_bucket_and_files' requires bucket name, file names" + if ! check_param_count_gt "setup_bucket_and_files" "bucket, file name" 2 $#; then return 1 fi if ! setup_bucket "$1"; then @@ -29,8 +27,7 @@ setup_bucket_and_files() { } setup_bucket_and_large_file() { - if [ $# -ne 2 ]; then - log 2 "'setup_bucket_and_large_file' requires bucket name, file name" + if ! check_param_count "setup_bucket_and_large_file" "bucket, file name" 2 $#; then return 1 fi if ! setup_bucket "$1"; then @@ -45,8 +42,7 @@ setup_bucket_and_large_file() { } setup_bucket_and_user() { - if [ $# -ne 4 ]; then - log 2 "'setup_bucket_and_user' requires bucket name, username, password, user type" + if ! check_param_count "setup_bucket_and_user" "bucket, username, password, user type" 4 $#; then return 1 fi if ! setup_bucket "$1"; then @@ -62,8 +58,7 @@ setup_bucket_and_user() { } setup_bucket_file_and_user() { - if [ $# -ne 5 ]; then - log 2 "'setup_bucket_file_and_user' requires bucket name, file, username, password, user type" + if ! check_param_count "setup_bucket_file_and_user" "bucket, file, username, password, user type" 5 $#; then return 1 fi if ! setup_bucket_and_files "$1" "$2"; then @@ -79,8 +74,7 @@ setup_bucket_file_and_user() { } setup_bucket_object_lock_enabled() { - if [ $# -ne 1 ]; then - log 2 "'setup_bucket_object_lock_enabled' requires bucket name" + if ! check_param_count "setup_bucket_object_lock_enabled" "bucket" 1 $#; then return 1 fi if ! bucket_cleanup_if_bucket_exists "$1"; then diff --git a/tests/versity.sh b/tests/versity.sh index 2b38c39..c11b9af 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -17,8 +17,7 @@ source ./tests/util/util_file.sh start_versity_process() { - if [[ $# -ne 1 ]]; then - log 1 "start versity process function requires number" + if ! check_param_count "start_versity_process" "versity app index" 1 $#; then exit 1 fi build_run_and_log_command @@ -67,8 +66,7 @@ build_run_and_log_command() { } run_versity_app_posix() { - if [[ $# -ne 3 ]]; then - log 1 "run versity app w/posix command requires access ID, secret key, process number" + if ! check_param_count "run_versity_app_posix" "access ID, secret key, versityid app index" 3 $#; then exit 1 fi base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION") @@ -96,8 +94,7 @@ run_versity_app_posix() { } run_versity_app_scoutfs() { - if [[ $# -ne 3 ]]; then - log 1 "run versity app w/scoutfs command requires access ID, secret key, process number" + if ! check_param_count "run_versity_app_scoutfs" "access ID, secret key, versityid app index" 3 $#; then exit 1 fi base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER") @@ -115,8 +112,7 @@ run_versity_app_scoutfs() { } run_versity_app_s3() { - if [[ $# -ne 1 ]]; then - log 1 "run versity app w/s3 command requires process number" + if ! check_param_count "run_versity_app_s3" "versityid app index" 1 $#; then exit 1 fi base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY") @@ -161,9 +157,8 @@ run_versity_app() { } stop_single_process() { - if [[ $# -ne 1 ]]; then - log 2 "stop single process function requires process ID" - return 1 + if ! check_param_count "stop_single_process" "versitygw PID" 1 $#; then + exit 1 fi log 5 "stop process with ID: $1" # shellcheck disable=SC2086