diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 4d608562..6e476274 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -128,6 +128,12 @@ jobs: RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + - set: "setup/remove static buckets scripts" + IAM_TYPE: folder + RUN_SET: "setup-remove-static" + RECREATE_BUCKETS: "true" + DELETE_BUCKETS_AFTER_TEST: "true" + BACKEND: "posix" steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -229,7 +235,10 @@ jobs: BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET - name: Time report - run: cat ${{ github.workspace }}/time.log + run: | + if [ -e ${{ github.workspace }}/time.log ]; then + cat ${{ github.workspace }}/time.log + fi - name: Coverage report run: | diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 1509e3ac..1163c200 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -22,6 +22,9 @@ RUN apt-get update && \ bc \ libxml2-utils \ xmlstarlet \ + python3-pip \ + python3-venv \ + xxd \ ca-certificates && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* @@ -63,6 +66,15 @@ RUN git clone https://github.com/bats-core/bats-core.git && \ cd bats-core && \ ./install.sh /home/tester +# Create a shared venv & install Python deps there +ENV VENV=/opt/venv +RUN python3 -m venv "$VENV" \ + && "$VENV/bin/python" -m pip install --upgrade pip \ + && "$VENV/bin/pip" install --no-cache-dir awscrt + +# Make the venv the default Python for all subsequent RUN/CMD/ENTRYPOINT +ENV PATH="$VENV/bin:${PATH}" + USER tester COPY --chown=tester:tester . /home/tester diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh index a1235b8f..18bd67d2 100644 --- a/tests/commands/get_bucket_ownership_controls.sh +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -41,7 +41,7 @@ get_bucket_ownership_controls_rest() { if ! check_param_count "get_bucket_ownership_controls_rest" "bucket" 1 $#; then return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then log 2 "error getting bucket ownership controls: $result" return 1 fi diff --git a/tests/commands/get_object_legal_hold.sh b/tests/commands/get_object_legal_hold.sh index b48e513c..d9cc9769 100644 --- a/tests/commands/get_object_legal_hold.sh +++ b/tests/commands/get_object_legal_hold.sh @@ -66,10 +66,10 @@ get_object_legal_hold_rest_version_id() { log 2 "error getting object legal hold: $result" return 1 fi + legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt") if [ "$result" != "200" ]; then - log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")" + log 2 "get-object-legal-hold returned code $result: $legal_hold)" return 1 fi - legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt") return 0 } diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index c4b836f3..6ecc0be4 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -53,5 +53,6 @@ get_object_lock_configuration_rest() { return 1 fi lock_config="$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" + log 5 "lock config: $lock_config" return 0 } \ No newline at end of file diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index bcac7297..f7d59a46 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -36,14 +36,14 @@ list_and_delete_objects() { fi done - if ! delete_old_versions "$1"; then + if ! delete_old_versions_base64 "$1"; then log 2 "error deleting old version" return 1 fi return 0 } -delete_old_versions() { +delete_old_versions_base64() { if ! check_param_count "delete_old_versions" "bucket" 1 $#; then return 1 fi @@ -53,45 +53,52 @@ delete_old_versions() { fi # shellcheck disable=SC2154 log 5 "versions: $versions" - version_keys=() - version_ids=() - if ! parse_version_data_by_type "rest" "$2"; then + if ! parse_base64_versions_rest; then log 2 "error parsing version data" return 1 fi - log 5 "version keys: ${version_keys[*]}" - log 5 "version IDs: ${version_ids[*]}" - for idx in "${!version_keys[@]}"; do - if ! delete_object_version_with_or_without_retention "$1"; then + log 5 "base64 versions: ${base64_pairs[*]}" + for pair in "${base64_pairs[@]}"; do + log 5 "pair: $pair" + if ! delete_object_version_with_or_without_retention_base64 "$1" "$pair"; then log 2 "error deleting version with or without retention" return 1 fi done } -delete_object_version_with_or_without_retention() { - if ! check_param_count "delete_object_version_with_or_without_retention" "bucket" 1 $#; then +delete_object_version_with_or_without_retention_base64() { + if ! check_param_count_v2 "bucket, key/value pair" 2 $#; then + return 1 + fi + IFS=":" read -ra key_and_id <<< "$2" + log 5 "key and ID: ${key_and_id[*]}" + if ! key=$(printf '%s' "${key_and_id[0]}" | base64 --decode 2>&1); then + log 2 "error decoding key: $key" + return 1 + fi + if ! id=$(printf '%s' "${key_and_id[1]}" | base64 --decode 2>&1); then + log 2 "error decoding ID: $id" return 1 fi - log 5 "idx: $idx" - log 5 "version ID: ${version_ids[$idx]}" # shellcheck disable=SC2154 if [ "$lock_config_exists" == "true" ]; then - if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! check_remove_legal_hold_versions "$1" "$key" "$id"; then log 2 "error checking, removing legal hold versions" + return 1 fi - if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest_bypass_retention "$1" "$key" "$id"; then log 2 "error deleting object version, bypassing retention" return 1 fi else - if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest "$1" "$key" "$id"; then log 2 "error deleting object version" return 1 fi fi - log 5 "successfully deleted version with key '${version_keys[$idx]}', id '${version_ids[$idx]}'" + log 5 "successfully deleted version with key '$key', id '$id'" return 0 } diff --git a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh index 0710bd3f..baf43f27 100644 --- a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +++ b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh @@ -22,6 +22,7 @@ check_object_lock_config() { lock_config_exists=true if ! get_object_lock_configuration "rest" "$1"; then # shellcheck disable=SC2154 + log 5 "lock config error: $get_object_lock_config_err" if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then # shellcheck disable=SC2034 lock_config_exists=false diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh index 52585dc6..479b7203 100644 --- a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh +++ b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -64,7 +64,7 @@ check_ownership_rule_and_reset_acl() { return 1 fi if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then - log 2 "error getting bucket ownership controls" + log 2 "error getting bucket ownership controls: $object_ownership_rule" return 1 fi log 5 "ownership rule: $object_ownership_rule" diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 6750b64d..7e3e14f2 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -158,8 +158,9 @@ send_rest_command_expect_success_callback() { log 2 "error sending command: $result" return 1 fi - if [ "$result" != "$3" ]; then - log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" + response_code="$(echo "$result" | tail -n 1)" + if [ "$response_code" != "$3" ]; then + log 2 "expected '$3', was '$response_code' ($(cat "$TEST_FILE_FOLDER/output.txt"))" return 1 fi if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then diff --git a/tests/remove_static.sh b/tests/remove_static.sh index 9f1fe894..6394de10 100755 --- a/tests/remove_static.sh +++ b/tests/remove_static.sh @@ -18,31 +18,32 @@ source ./tests/setup.sh source ./tests/util/util_object.sh delete_bucket_if_exists() { - if [[ $# -ne 2 ]]; then - log 2 "delete_bucket_if_exists command missing command type, name" + if [[ $# -ne 1 ]]; then + log 2 "delete_bucket_if_exists command missing bucket name" return 1 fi - bucket_exists "$1" "$2" || local exists_result=$? + bucket_exists "$1" || local exists_result=$? if [[ $exists_result -eq 2 ]]; then log 2 "error checking if bucket exists" return 1 fi if [[ $exists_result -eq 1 ]]; then - echo "bucket '$2' doesn't exist, skipping" + echo "bucket '$1' doesn't exist, skipping" return 0 fi - if ! delete_bucket_recursive "$1" "$2"; then + log 5 "attempting to delete bucket '$1'" + if ! delete_bucket_recursive "$1"; then log 2 "error deleting bucket" return 1 fi - echo "bucket '$2' successfully deleted" + echo "bucket '$1' successfully deleted" return 0 } base_setup -if ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then +if ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_ONE_NAME"; then log 2 "error deleting static bucket one" -elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then +elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_TWO_NAME"; then log 2 "error deleting static bucket two" fi if ! stop_versity; then diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh index 7e23f0d9..05b1e69b 100755 --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -21,7 +21,7 @@ source ./tests/rest_scripts/rest.sh # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" # shellcheck disable=SC2154 -key="$OBJECT_KEY" +key="$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")')" # shellcheck disable=SC2153,SC2154 version_id="$VERSION_ID" @@ -48,4 +48,5 @@ curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_ curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 +log_rest 5 "curl command: ${curl_command[*]}" eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/run.sh b/tests/run.sh index f1d0cc1e..6ab3861b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -61,7 +61,7 @@ handle_param() { s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\ rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\ rest-put-object|rest-versioning|rest-bucket|rest-multipart|rest-delete-bucket-ownership-controls|\ - rest-delete-bucket-tagging) + rest-delete-bucket-tagging|setup-remove-static) run_suite "$1" ;; *) # Handle unrecognized options or positional arguments @@ -247,8 +247,13 @@ run_suite() { s3api-user) echo "Running s3api user tests ..." "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$? + ;; + setup-remove-static) + echo "Testing setup/remove static bucket scripts ..." + VERSITYGW_TEST_ENV="$VERSITYGW_TEST_ENV" ./tests/test_setup_remove_static.sh || exit_code=$? + ;; esac - if [ $exit_code -ne 0 ]; then + if [ "$exit_code" -ne 0 ]; then exit 1 fi } diff --git a/tests/test_rest.sh b/tests/test_rest.sh index ad4d31d3..221814c6 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -111,13 +111,8 @@ test_file="test_file" test_key="TestKey" test_value="TestValue" - run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" + run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" assert_success - # in static bucket config, bucket will still exist - if ! bucket_exists "$BUCKET_ONE_NAME"; then - run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" - assert_success - fi run create_test_files "$test_file" assert_success @@ -135,13 +130,30 @@ test_file="test_file" } @test "REST - legal hold, get without config" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "test requires object lock not to be enabled" + fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success - run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" + run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "InvalidRequest" + assert_success +} + +@test "REST - legal hold, object lock enabled w/o specific object lock set" { + run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "NoSuchObjectLockConfiguration" assert_success } diff --git a/tests/test_rest_delete_bucket_ownership_controls.sh b/tests/test_rest_delete_bucket_ownership_controls.sh index 6739f1b6..2b6f19a6 100755 --- a/tests/test_rest_delete_bucket_ownership_controls.sh +++ b/tests/test_rest_delete_bucket_ownership_controls.sh @@ -29,8 +29,9 @@ export RUN_USERS=true fi run setup_bucket_and_user_v2 "$BUCKET_ONE_NAME" "$USERNAME_ONE" "$PASSWORD_ONE" assert_success - username=${lines[2]} - password=${lines[3]} + username=${lines[${#lines[@]}-2]} + password=${lines[${#lines[@]}-1]} + log 5 "output: $output" log 5 "username: $username, password: $password" run send_rest_go_command_expect_error "403" "AccessDenied" "Access Denied" "-awsAccessKeyId" "$username" "-awsSecretAccessKey" "$password" \ diff --git a/tests/test_setup_remove_static.sh b/tests/test_setup_remove_static.sh new file mode 100755 index 00000000..7642d906 --- /dev/null +++ b/tests/test_setup_remove_static.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/logger.sh + +if [ -z "$VERSITYGW_TEST_ENV" ]; then + echo "VERSITYGW_TEST_ENV variable missing" + return 1 +fi +# shellcheck disable=SC1090 +source "$VERSITYGW_TEST_ENV" +if [ "$RECREATE_BUCKETS" != "true" ] || [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ]; then + echo "for this test 'RECREATE_BUCKETS' and 'DELETE_BUCKETS_AFTER_TEST' must be set to true" + exit 1 +fi +# try to remove buckets if already there, don't worry if they're not +./tests/remove_static.sh +echo "AFTER FIRST REMOVAL" +if ! result=$(./tests/setup_static.sh 2>&1); then + echo "error setting up buckets: $result" + exit 1 +fi +echo "RESULT: $result" +if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully created"* ]]; then + echo "error creating bucket '$BUCKET_ONE_NAME'" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully created"* ]]; then + echo "error creating bucket '$BUCKET_TWO_NAME'" + exit 1 +fi +if ! result=$(./tests/remove_static.sh 2>&1); then + echo "error removing buckets: $result" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully deleted"* ]]; then + echo "error deleting bucket '$BUCKET_ONE_NAME'" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully deleted"* ]]; then + echo "error deleting bucket '$BUCKET_TWO_NAME'" + exit 1 +fi +exit 0 \ No newline at end of file diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 3ec9acc6..11b5c165 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -16,6 +16,7 @@ reset_bucket() { if ! check_param_count "reset_bucket" "bucket" 1 $#; then return 1 fi + log 6 "reset bucket '$1'" if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then log 3 "error logging bucket policy" @@ -66,7 +67,7 @@ reset_bucket() { # params: bucket name # return 0 if able to delete recursively, 1 if not delete_bucket_recursive() { - log 6 "delete_bucket_recursive_s3api" + log 6 "delete_bucket_recursive" if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then return 1 fi @@ -108,6 +109,7 @@ direct_wait_for_bucket() { if ! check_param_count "direct_wait_for_bucket" "bucket" 1 $#; then return 1 fi + sleep 5 bucket_verification_start_time=$(date +%s) while ! bucket_exists "$1"; do bucket_verification_end_time=$(date +%s) @@ -120,6 +122,23 @@ direct_wait_for_bucket() { return 0 } +direct_wait_for_bucket_deletion() { + if ! check_param_count "direct_wait_for_bucket" "bucket" 1 $#; then + return 1 + fi + sleep 5 + bucket_verification_start_time=$(date +%s) + while bucket_exists "$1"; do + bucket_verification_end_time=$(date +%s) + if [ $((bucket_verification_end_time-bucket_verification_start_time)) -ge 60 ]; then + log 2 "bucket existence check timeout" + return 1 + fi + sleep 5 + done + return 0 +} + # params: client, bucket name # return 0 for success, 1 for error bucket_cleanup() { @@ -207,6 +226,10 @@ setup_bucket() { log 5 "util.setup_bucket: bucket name: $1" if [[ $RECREATE_BUCKETS == "true" ]]; then + if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket_deletion "$1"; then + log 2 "bucket not successfully deleted" + return 1 + fi if ! create_bucket "s3api" "$1"; then log 2 "error creating bucket" return 1 @@ -217,6 +240,7 @@ setup_bucket() { # bucket creation and resets take longer to propagate in direct mode if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket "$1"; then + log 2 "bucket not found after creation" return 1 fi diff --git a/tests/util/util_legal_hold.sh b/tests/util/util_legal_hold.sh index c55de831..f8681bcb 100644 --- a/tests/util/util_legal_hold.sh +++ b/tests/util/util_legal_hold.sh @@ -37,7 +37,7 @@ get_and_check_legal_hold() { } check_legal_hold_without_lock_enabled() { - if ! check_param_count "check_legal_hold_without_lock_enabled" "bucket, key" 2 $#; then + if ! check_param_count_v2 "bucket, key, expected error" 3 $#; then return 1 fi if get_object_legal_hold_rest "$1" "$2"; then @@ -49,8 +49,8 @@ check_legal_hold_without_lock_enabled() { log 2 "error getting error code: $code" return 1 fi - if [ "$code" != "InvalidRequest" ]; then - log 2 "code mismatch (expected 'InvalidRequest', actual '$code')" + if [ "$code" != "$3" ]; then + log 2 "code mismatch (expected '$3', actual '$code')" return 1 fi return 0 @@ -62,7 +62,8 @@ check_remove_legal_hold_versions() { fi if ! get_object_legal_hold_rest_version_id "$1" "$2" "$3"; then # shellcheck disable=SC2154 - if [[ "$legal_hold" != *"MethodNotAllowed"* ]]; then + log 5 "legal hold: $legal_hold" + if [[ "$legal_hold" != *"MethodNotAllowed"* ]] && [[ "$legal_hold" != *"NoSuchObjectLockConfiguration"* ]]; then log 2 "error getting object legal hold status with version id" return 1 fi diff --git a/tests/util/util_list_parts.sh b/tests/util/util_list_parts.sh index eba139f9..397e0c09 100644 --- a/tests/util/util_list_parts.sh +++ b/tests/util/util_list_parts.sh @@ -64,7 +64,7 @@ check_part_list_rest() { perform_multipart_upload_rest() { if [ $# -ne 6 ]; then - log 2 "'upload_check_parts' requires bucket, key, part list" + log 2 "'upload_check_parts' requires bucket, key, part list (4 parts)" return 1 fi if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then @@ -93,7 +93,7 @@ perform_multipart_upload_rest() { return 1 fi parts_payload+="$etag4" - if ! complete_multipart_upload_rest "$1" "$2" "$3" "$4"; then + if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then log 2 "error completing multipart upload" return 1 fi diff --git a/tests/util/util_setup.sh b/tests/util/util_setup.sh index 0f349b5f..5aa517da 100644 --- a/tests/util/util_setup.sh +++ b/tests/util/util_setup.sh @@ -67,6 +67,10 @@ setup_bucket_object_lock_enabled() { log 2 "error cleaning up bucket" return 1 fi + if [ "$DIRECT" == "true" ] && [ "$RECREATE_BUCKETS" == "true" ] && ! direct_wait_for_bucket_deletion "$1"; then + log 2 "bucket not confirmed as deleted" + return 1 + fi # in static bucket config, bucket will still exist if ! bucket_exists "$1"; then @@ -75,5 +79,9 @@ setup_bucket_object_lock_enabled() { return 1 fi fi + if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket "$1"; then + log 2 "bucket not confirmed as created" + return 1 + fi return 0 } diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index 909f8151..9e1d03d3 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -102,7 +102,7 @@ check_versioning_status_rest() { } echo_versions() { - if ! check_param_count "echo_versions" "'Version' or 'DeleteMarker', 'Key' or 'VersionId'" 2 $#; then + if ! check_param_count_v2 "'Version' or 'DeleteMarker', 'Key' or 'VersionId', file" 3 $#; then return 1 fi if ! keys=$(echo -n "$versions" | xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" - | xmlstarlet unesc 2>&1); then @@ -113,32 +113,64 @@ echo_versions() { return 1 fi log 5 "keys to append: ${keys[*]}" + if ! result=$(truncate -s 0 "$3" 2>&1); then + log 2 "error truncating file: $result" + fi + for key in "${keys[@]}"; do + echo "$key" >> "$3" + done echo "${keys[*]}" } -parse_versions_rest() { - if ! check_param_count "parse_versions_rest" "versions variable" 1 $#; then +get_base64_version_keys_and_ids() { + if ! check_param_count_v2 "'Version' or 'DeleteMarker'" 1 $#; then return 1 fi - if ! keys=$(echo_versions "Version" "Key"); then + while IFS= read -r key && IFS= read -r vid; do + log 5 "key: $key, vid: $vid" + b_key="$(printf '%s' "$key" | base64 -w0)" + b_vid="$(printf '%s' "$vid" | base64 -w0)" + base64_pairs+=("$b_key:$b_vid") + done < <(xmlstarlet sel -t \ + -m '//*[local-name()='"\"$1\""']' \ + -v '*[local-name()="Key"]' -n \ + -v '*[local-name()="VersionId"]' -n \ + <<<"$versions" | xmlstarlet unesc) +} + +parse_base64_versions_rest() { + base64_pairs=() + if ! get_base64_version_keys_and_ids "Version"; then + log 2 "error getting version base64 keys and IDs" + return 1 + fi + if ! get_base64_version_keys_and_ids "DeleteMarker"; then + log 2 "error getting version base64 keys and IDs" + return 1 + fi +} + +parse_versions_rest() { + base64_pairs=() + if ! keys=$(echo_versions "Version" "Key" "$TEST_FILE_FOLDER/version_keys.txt"); then log 2 "error getting Version Key values: $keys" return 1 fi # shellcheck disable=SC2206 version_keys+=($keys) - if ! ids=$(echo_versions "Version" "VersionId"); then + if ! ids=$(echo_versions "Version" "VersionId" "$TEST_FILE_FOLDER/version_ids.txt"); then log 2 "error getting Version VersionId values: $ids" return 1 fi # shellcheck disable=SC2206 version_ids+=($ids) - if ! keys=$(echo_versions "DeleteMarker" "Key"); then + if ! keys=$(echo_versions "DeleteMarker" "Key" "$TEST_FILE_FOLDER/delete_marker_keys.txt"); then log 2 "error getting DeleteMarker Key values: $keys" return 1 fi # shellcheck disable=SC2206 version_keys+=($keys) - if ! ids=$(echo_versions "DeleteMarker" "VersionId"); then + if ! ids=$(echo_versions "DeleteMarker" "VersionId" "$TEST_FILE_FOLDER/delete_marker_ids.txt"); then log 2 "error getting DeleteMarker VersionId values: $ids" return 1 fi @@ -146,6 +178,7 @@ parse_versions_rest() { version_ids+=($ids) log 5 "version keys: ${version_keys[*]}" log 5 "version IDs: ${version_ids[*]}" + log 5 "base64 pairs: ${base64_pairs[*]}" return 0 }