diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 4d60856..6e47627 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -128,6 +128,12 @@ jobs: RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + - set: "setup/remove static buckets scripts" + IAM_TYPE: folder + RUN_SET: "setup-remove-static" + RECREATE_BUCKETS: "true" + DELETE_BUCKETS_AFTER_TEST: "true" + BACKEND: "posix" steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -229,7 +235,10 @@ jobs: BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET - name: Time report - run: cat ${{ github.workspace }}/time.log + run: | + if [ -e ${{ github.workspace }}/time.log ]; then + cat ${{ github.workspace }}/time.log + fi - name: Coverage report run: | diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 1509e3a..1163c20 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -22,6 +22,9 @@ RUN apt-get update && \ bc \ libxml2-utils \ xmlstarlet \ + python3-pip \ + python3-venv \ + xxd \ ca-certificates && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* @@ -63,6 +66,15 @@ RUN git clone https://github.com/bats-core/bats-core.git && \ cd bats-core && \ ./install.sh /home/tester +# Create a shared venv & install Python deps there +ENV VENV=/opt/venv +RUN python3 -m venv "$VENV" \ + && "$VENV/bin/python" -m pip install --upgrade pip \ + && "$VENV/bin/pip" install --no-cache-dir awscrt + +# Make the venv the default Python for all subsequent RUN/CMD/ENTRYPOINT +ENV PATH="$VENV/bin:${PATH}" + USER tester COPY --chown=tester:tester . /home/tester diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh index a1235b8..18bd67d 100644 --- a/tests/commands/get_bucket_ownership_controls.sh +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -41,7 +41,7 @@ get_bucket_ownership_controls_rest() { if ! check_param_count "get_bucket_ownership_controls_rest" "bucket" 1 $#; then return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then log 2 "error getting bucket ownership controls: $result" return 1 fi diff --git a/tests/commands/get_object_legal_hold.sh b/tests/commands/get_object_legal_hold.sh index b48e513..d9cc976 100644 --- a/tests/commands/get_object_legal_hold.sh +++ b/tests/commands/get_object_legal_hold.sh @@ -66,10 +66,10 @@ get_object_legal_hold_rest_version_id() { log 2 "error getting object legal hold: $result" return 1 fi + legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt") if [ "$result" != "200" ]; then - log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")" + log 2 "get-object-legal-hold returned code $result: $legal_hold)" return 1 fi - legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt") return 0 } diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index c4b836f..6ecc0be 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -53,5 +53,6 @@ get_object_lock_configuration_rest() { return 1 fi lock_config="$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" + log 5 "lock config: $lock_config" return 0 } \ No newline at end of file diff --git a/tests/drivers/create_bucket/create_bucket_rest.sh b/tests/drivers/create_bucket/create_bucket_rest.sh index 8060c98..df97ea0 100644 --- a/tests/drivers/create_bucket/create_bucket_rest.sh +++ b/tests/drivers/create_bucket/create_bucket_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/drivers/get_object/get_object_rest.sh source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -111,4 +112,97 @@ create_bucket_and_check_acl() { return 1 fi return 0 -} \ No newline at end of file +} + +get_bucket_prefix() { + if ! check_param_count_v2 "bucket prefix or name" 1 $#; then + return 1 + fi + if [ "$RECREATE_BUCKETS" == "true" ]; then + # remove date/time suffix + prefix="$(echo "$1" | awk '{print substr($0, 1, length($0)-15)}')" + else + prefix="$1" + fi + echo "$prefix" +} + +setup_bucket_v2() { + if ! check_param_count_v2 "bucket prefix or name" 1 $#; then + return 1 + fi + if ! prefix=$(get_bucket_prefix "$1" 2>&1); then + log 2 "error getting prefix: $prefix" + return 1 + fi + if ! bucket_cleanup_if_bucket_exists_v2 "$prefix"; then + log 2 "error cleaning up bucket(s), if it/they exist(s)" + return 1 + fi + if [ "$RECREATE_BUCKETS" == "false" ]; then + return 0 + fi + if ! create_bucket_rest_expect_success "$1" ""; then + log 2 "error creating bucket '$1'" + return 1 + fi + return 0 +} + +get_bucket_name() { + if ! check_param_count_v2 "bucket" 1 $#; then + return 1 + fi + if [ "$RECREATE_BUCKETS" == "false" ]; then + echo "$1" + fi + echo "$1-$(date +%Y%m%d%H%M%S)" +} + +setup_bucket_object_lock_enabled_v2() { + if ! check_param_count_v2 "bucket" 1 $#; then + return 1 + fi + if ! prefix=$(get_bucket_prefix "$1" 2>&1); then + log 2 "error getting prefix: $prefix" + return 1 + fi + if ! bucket_cleanup_if_bucket_exists_v2 "$prefix"; then + log 2 "error cleaning up bucket" + return 1 + fi + if [ "$RECREATE_BUCKETS" == "true" ]; then + if ! create_bucket_object_lock_enabled "$1"; then + log 2 "error creating bucket '$1' with object lock enabled" + return 1 + fi + fi + return 0 +} + +setup_bucket_object_lock_enabled() { + if ! check_param_count "setup_bucket_object_lock_enabled" "bucket" 1 $#; then + return 1 + fi + if ! bucket_cleanup_if_bucket_exists "$1"; then + log 2 "error cleaning up bucket" + return 1 + fi + if [ "$DIRECT" == "true" ] && [ "$RECREATE_BUCKETS" == "true" ]; then + log 2 "bucket not confirmed as deleted" + return 1 + fi + + # in static bucket config, bucket will still exist + if ! bucket_exists "$1"; then + if ! create_bucket_object_lock_enabled "$1"; then + log 2 "error creating bucket with object lock enabled" + return 1 + fi + fi + if [ "$DIRECT" == "true" ]; then + log 2 "bucket not confirmed as created" + return 1 + fi + return 0 +} diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh new file mode 100644 index 0000000..7ca6bd7 --- /dev/null +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/commands/list_buckets.sh + +delete_buckets_with_prefix() { + if ! check_param_count_v2 "bucket prefix" 1 $#; then + return 1 + fi + if [ "$1" == "" ]; then + log 2 "delete_buckets_with_prefix requires non-empty prefix" + return 1 + fi + if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then + log 2 "error listing buckets with prefix" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "buckets: ${bucket_array[*]}" + for bucket in "${bucket_array[@]}"; do + if ! delete_bucket_recursive "$bucket"; then + log 2 "error with recursive bucket delete of bucket '$bucket'" + return 1 + fi + done + return 0 +} + +cleanup_buckets() { + if ! bucket_cleanup_if_bucket_exists_v2 "$BUCKET_ONE_NAME"; then + log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" + fi + if ! bucket_cleanup_if_bucket_exists_v2 "$BUCKET_TWO_NAME"; then + log 3 "error deleting bucket $BUCKET_TWO_NAME or contents" + fi +} + +# params: client, bucket name +# return 0 for success, 1 for error +bucket_cleanup() { + log 6 "bucket_cleanup" + if ! check_param_count "bucket_cleanup" "bucket name" 1 $#; then + return 1 + fi + if [[ $RECREATE_BUCKETS == "false" ]]; then + if ! reset_bucket "$1"; then + log 2 "error deleting bucket contents" + return 1 + fi + + log 5 "bucket contents, policy, ACL deletion success" + return 0 + fi + if ! delete_bucket_recursive "$1"; then + log 2 "error with recursive bucket delete" + return 1 + fi + log 5 "bucket deletion success" + return 0 +} + +# params: client, bucket name +# return 0 for success, 1 for error +bucket_cleanup_if_bucket_exists() { + log 6 "bucket_cleanup_if_bucket_exists" + if ! check_param_count_gt "bucket name, bucket known to exist (optional)" 1 $#; then + return 1 + fi + + if [ "$2" == "false" ]; then + log 5 "skipping cleanup, since bucket doesn't exist" + return 0 + fi + + if [ "$2" == "true" ] || bucket_exists "$1"; then + if ! bucket_cleanup "$1"; then + log 2 "error deleting bucket and/or contents" + return 1 + fi + log 5 "bucket and/or bucket data deletion success" + return 0 + fi + return 0 +} + +bucket_cleanup_if_bucket_exists_v2() { + log 6 "bucket_cleanup_if_bucket_exists_v2" + if ! check_param_count_gt "bucket name or prefix" 1 $#; then + return 1 + fi + if [[ "$RECREATE_BUCKETS" == "false" ]]; then + if ! bucket_exists "$1"; then + log 2 "When RECREATE_BUCKETS isn't set to \"true\", bucket with full env name should be pre-created by user" + return 1 + fi + if ! reset_bucket "$1"; then + log 2 "error resetting bucket before tests" + return 1 + fi + return 0 + else + if ! delete_buckets_with_prefix "$1"; then + log 2 "error deleting buckets with prefix '$1'" + return 1 + fi + fi + return 0 +} diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index bcac729..f7d59a4 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -36,14 +36,14 @@ list_and_delete_objects() { fi done - if ! delete_old_versions "$1"; then + if ! delete_old_versions_base64 "$1"; then log 2 "error deleting old version" return 1 fi return 0 } -delete_old_versions() { +delete_old_versions_base64() { if ! check_param_count "delete_old_versions" "bucket" 1 $#; then return 1 fi @@ -53,45 +53,52 @@ delete_old_versions() { fi # shellcheck disable=SC2154 log 5 "versions: $versions" - version_keys=() - version_ids=() - if ! parse_version_data_by_type "rest" "$2"; then + if ! parse_base64_versions_rest; then log 2 "error parsing version data" return 1 fi - log 5 "version keys: ${version_keys[*]}" - log 5 "version IDs: ${version_ids[*]}" - for idx in "${!version_keys[@]}"; do - if ! delete_object_version_with_or_without_retention "$1"; then + log 5 "base64 versions: ${base64_pairs[*]}" + for pair in "${base64_pairs[@]}"; do + log 5 "pair: $pair" + if ! delete_object_version_with_or_without_retention_base64 "$1" "$pair"; then log 2 "error deleting version with or without retention" return 1 fi done } -delete_object_version_with_or_without_retention() { - if ! check_param_count "delete_object_version_with_or_without_retention" "bucket" 1 $#; then +delete_object_version_with_or_without_retention_base64() { + if ! check_param_count_v2 "bucket, key/value pair" 2 $#; then + return 1 + fi + IFS=":" read -ra key_and_id <<< "$2" + log 5 "key and ID: ${key_and_id[*]}" + if ! key=$(printf '%s' "${key_and_id[0]}" | base64 --decode 2>&1); then + log 2 "error decoding key: $key" + return 1 + fi + if ! id=$(printf '%s' "${key_and_id[1]}" | base64 --decode 2>&1); then + log 2 "error decoding ID: $id" return 1 fi - log 5 "idx: $idx" - log 5 "version ID: ${version_ids[$idx]}" # shellcheck disable=SC2154 if [ "$lock_config_exists" == "true" ]; then - if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! check_remove_legal_hold_versions "$1" "$key" "$id"; then log 2 "error checking, removing legal hold versions" + return 1 fi - if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest_bypass_retention "$1" "$key" "$id"; then log 2 "error deleting object version, bypassing retention" return 1 fi else - if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest "$1" "$key" "$id"; then log 2 "error deleting object version" return 1 fi fi - log 5 "successfully deleted version with key '${version_keys[$idx]}', id '${version_ids[$idx]}'" + log 5 "successfully deleted version with key '$key', id '$id'" return 0 } diff --git a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh index 0710bd3..baf43f2 100644 --- a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +++ b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh @@ -22,6 +22,7 @@ check_object_lock_config() { lock_config_exists=true if ! get_object_lock_configuration "rest" "$1"; then # shellcheck disable=SC2154 + log 5 "lock config error: $get_object_lock_config_err" if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then # shellcheck disable=SC2034 lock_config_exists=false diff --git a/tests/drivers/list_buckets/list_buckets_rest.sh b/tests/drivers/list_buckets/list_buckets_rest.sh index 1104bda..c25d0e4 100644 --- a/tests/drivers/list_buckets/list_buckets_rest.sh +++ b/tests/drivers/list_buckets/list_buckets_rest.sh @@ -64,9 +64,13 @@ parse_bucket_list() { bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1") bucket_array=() while read -r bucket; do - bucket_array+=("$bucket") + if [ -n "$bucket" ]; then + log 5 "reading bucket '$bucket'" + bucket_array+=("$bucket") + fi done <<< "$bucket_list" log 5 "bucket array: ${bucket_array[*]}" + log 5 "bucket array length: ${#bucket_array[@]}" } parse_buckets_and_continuation_token() { diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh index 52585dc..479b720 100644 --- a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh +++ b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -64,7 +64,7 @@ check_ownership_rule_and_reset_acl() { return 1 fi if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then - log 2 "error getting bucket ownership controls" + log 2 "error getting bucket ownership controls: $object_ownership_rule" return 1 fi log 5 "ownership rule: $object_ownership_rule" diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 6750b64..7e3e14f 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -158,8 +158,9 @@ send_rest_command_expect_success_callback() { log 2 "error sending command: $result" return 1 fi - if [ "$result" != "$3" ]; then - log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" + response_code="$(echo "$result" | tail -n 1)" + if [ "$response_code" != "$3" ]; then + log 2 "expected '$3', was '$response_code' ($(cat "$TEST_FILE_FOLDER/output.txt"))" return 1 fi if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then diff --git a/tests/remove_static.sh b/tests/remove_static.sh index 9f1fe89..6394de1 100755 --- a/tests/remove_static.sh +++ b/tests/remove_static.sh @@ -18,31 +18,32 @@ source ./tests/setup.sh source ./tests/util/util_object.sh delete_bucket_if_exists() { - if [[ $# -ne 2 ]]; then - log 2 "delete_bucket_if_exists command missing command type, name" + if [[ $# -ne 1 ]]; then + log 2 "delete_bucket_if_exists command missing bucket name" return 1 fi - bucket_exists "$1" "$2" || local exists_result=$? + bucket_exists "$1" || local exists_result=$? if [[ $exists_result -eq 2 ]]; then log 2 "error checking if bucket exists" return 1 fi if [[ $exists_result -eq 1 ]]; then - echo "bucket '$2' doesn't exist, skipping" + echo "bucket '$1' doesn't exist, skipping" return 0 fi - if ! delete_bucket_recursive "$1" "$2"; then + log 5 "attempting to delete bucket '$1'" + if ! delete_bucket_recursive "$1"; then log 2 "error deleting bucket" return 1 fi - echo "bucket '$2' successfully deleted" + echo "bucket '$1' successfully deleted" return 0 } base_setup -if ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then +if ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_ONE_NAME"; then log 2 "error deleting static bucket one" -elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then +elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_TWO_NAME"; then log 2 "error deleting static bucket two" fi if ! stop_versity; then diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh index 7e23f0d..05b1e69 100755 --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -21,7 +21,7 @@ source ./tests/rest_scripts/rest.sh # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" # shellcheck disable=SC2154 -key="$OBJECT_KEY" +key="$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")')" # shellcheck disable=SC2153,SC2154 version_id="$VERSION_ID" @@ -48,4 +48,5 @@ curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_ curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 +log_rest 5 "curl command: ${curl_command[*]}" eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/run.sh b/tests/run.sh index f1d0cc1..6ab3861 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -61,7 +61,7 @@ handle_param() { s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\ rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\ rest-put-object|rest-versioning|rest-bucket|rest-multipart|rest-delete-bucket-ownership-controls|\ - rest-delete-bucket-tagging) + rest-delete-bucket-tagging|setup-remove-static) run_suite "$1" ;; *) # Handle unrecognized options or positional arguments @@ -247,8 +247,13 @@ run_suite() { s3api-user) echo "Running s3api user tests ..." "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$? + ;; + setup-remove-static) + echo "Testing setup/remove static bucket scripts ..." + VERSITYGW_TEST_ENV="$VERSITYGW_TEST_ENV" ./tests/test_setup_remove_static.sh || exit_code=$? + ;; esac - if [ $exit_code -ne 0 ]; then + if [ "$exit_code" -ne 0 ]; then exit 1 fi } diff --git a/tests/setup.sh b/tests/setup.sh index a4a05e2..c3cafaf 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -17,6 +17,7 @@ source ./tests/env.sh source ./tests/report.sh source ./tests/setup_mc.sh +source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/util/util_object.sh source ./tests/versity.sh @@ -146,14 +147,9 @@ post_versity_cleanup() { teardown() { # shellcheck disable=SC2154 log 4 "********** BEGIN TEARDOWN **********" - if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then - log 5 "deleting or clearing buckets" - if ! bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"; then - log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" - fi - if ! bucket_cleanup_if_bucket_exists "$BUCKET_TWO_NAME"; then - log 3 "error deleting bucket $BUCKET_TWO_NAME or contents" - fi + if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then + log 3 "error cleaning up buckets after test" + return 1 fi if [ "$SKIP_USERS_TESTS" != "true" ]; then if [ -n "$USERNAME_ONE" ]; then diff --git a/tests/test_rest.sh b/tests/test_rest.sh index ad4d31d..2f3bb42 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -29,6 +29,7 @@ source ./tests/commands/put_bucket_versioning.sh source ./tests/commands/put_object.sh source ./tests/commands/put_object_retention.sh source ./tests/commands/put_object_tagging.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/copy_object/copy_object_rest.sh source ./tests/drivers/get_bucket_ownership_controls/get_bucket_ownership_controls_rest.sh source ./tests/drivers/head_object/head_object_rest.sh @@ -111,13 +112,8 @@ test_file="test_file" test_key="TestKey" test_value="TestValue" - run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" + run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" assert_success - # in static bucket config, bucket will still exist - if ! bucket_exists "$BUCKET_ONE_NAME"; then - run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" - assert_success - fi run create_test_files "$test_file" assert_success @@ -135,13 +131,30 @@ test_file="test_file" } @test "REST - legal hold, get without config" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "test requires object lock not to be enabled" + fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success - run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" + run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "InvalidRequest" + assert_success +} + +@test "REST - legal hold, object lock enabled w/o specific object lock set" { + run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "NoSuchObjectLockConfiguration" assert_success } diff --git a/tests/test_rest_bucket.sh b/tests/test_rest_bucket.sh index de47468..3957f99 100755 --- a/tests/test_rest_bucket.sh +++ b/tests/test_rest_bucket.sh @@ -37,10 +37,14 @@ source ./tests/util/util_tags.sh export RUN_USERS=true @test "REST - HeadBucket" { - run setup_bucket "$BUCKET_ONE_NAME" + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_v2 "$bucket_name" assert_success - run head_bucket_rest "$BUCKET_ONE_NAME" + run head_bucket_rest "$bucket_name" assert_success } diff --git a/tests/test_rest_delete_bucket_ownership_controls.sh b/tests/test_rest_delete_bucket_ownership_controls.sh index 6739f1b..2b6f19a 100755 --- a/tests/test_rest_delete_bucket_ownership_controls.sh +++ b/tests/test_rest_delete_bucket_ownership_controls.sh @@ -29,8 +29,9 @@ export RUN_USERS=true fi run setup_bucket_and_user_v2 "$BUCKET_ONE_NAME" "$USERNAME_ONE" "$PASSWORD_ONE" assert_success - username=${lines[2]} - password=${lines[3]} + username=${lines[${#lines[@]}-2]} + password=${lines[${#lines[@]}-1]} + log 5 "output: $output" log 5 "username: $username, password: $password" run send_rest_go_command_expect_error "403" "AccessDenied" "Access Denied" "-awsAccessKeyId" "$username" "-awsSecretAccessKey" "$password" \ diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index d51c483..7b7e664 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -20,6 +20,7 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/drivers/rest.sh source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh source ./tests/drivers/upload_part/upload_part_rest.sh source ./tests/util/util_file.sh source ./tests/util/util_list_parts.sh diff --git a/tests/test_rest_versioning.sh b/tests/test_rest_versioning.sh index cb459b2..4566451 100755 --- a/tests/test_rest_versioning.sh +++ b/tests/test_rest_versioning.sh @@ -20,87 +20,103 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh source ./tests/util/util_rest.sh source ./tests/util/util_setup.sh test_file="test_file" @test "REST - check, enable, suspend versioning" { - run setup_bucket "$BUCKET_ONE_NAME" + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_v2 "$bucket_name" assert_success - log 5 "get versioning" - - run check_versioning_status_rest "$BUCKET_ONE_NAME" "" + run check_versioning_status_rest "$bucket_name" "" assert_success - run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled" + run put_bucket_versioning_rest "$bucket_name" "Enabled" assert_success - run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled" + run check_versioning_status_rest "$bucket_name" "Enabled" assert_success - run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended" + run put_bucket_versioning_rest "$bucket_name" "Suspended" assert_success - run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended" + run check_versioning_status_rest "$bucket_name" "Suspended" assert_success } @test "test_rest_versioning" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" assert_success - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true" + run get_and_check_versions_rest "$bucket_name" "$test_file" "1" "true" "true" assert_success - run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + run put_bucket_versioning_rest "$bucket_name" "Enabled" assert_success - run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true" + run get_and_check_versions_rest "$bucket_name" "$test_file" "1" "true" "true" assert_success - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true" + run get_and_check_versions_rest "$bucket_name" "$test_file" "2" "true" "false" "false" "true" assert_success } @test "versioning - add version, then delete and check for marker" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" assert_success - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + run put_bucket_versioning_rest "$bucket_name" "Enabled" assert_success - run delete_object_rest "$BUCKET_ONE_NAME" "$test_file" + run delete_object_rest "$bucket_name" "$test_file" assert_success - run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file" + run check_versions_after_file_deletion "$bucket_name" "$test_file" assert_success } @test "versioning - retrieve after delete" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + run put_bucket_versioning_rest "$bucket_name" "Enabled" assert_success - run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file" + run delete_object "s3api" "$bucket_name" "$test_file" assert_success - run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run get_object "s3api" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_failure } @@ -108,25 +124,23 @@ test_file="test_file" if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then skip "test isn't valid for this configuration" fi - run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" + run get_bucket_name "$BUCKET_ONE_NAME" assert_success + bucket_name="$output" - # in static bucket config, bucket will still exist - if ! bucket_exists "$BUCKET_ONE_NAME"; then - run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" - assert_success - fi + run setup_bucket_object_lock_enabled_v2 "$bucket_name" + assert_success run create_test_files "$test_file" assert_success - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file" + run delete_object "s3api" "$bucket_name" "$test_file" assert_success - run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file" + run verify_object_not_found "$bucket_name" "$test_file" assert_success } @@ -134,24 +148,22 @@ test_file="test_file" if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then skip "test isn't valid for this configuration" fi - run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" + run get_bucket_name "$BUCKET_ONE_NAME" assert_success + bucket_name="$output" - # in static bucket config, bucket will still exist - if ! bucket_exists "$BUCKET_ONE_NAME"; then - run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" - assert_success - fi + run setup_bucket_object_lock_enabled_v2 "$bucket_name" + assert_success run create_test_files "$test_file" assert_success - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file" + run delete_object "s3api" "$bucket_name" "$test_file" assert_success - run get_delete_marker_and_verify_405 "$BUCKET_ONE_NAME" "$test_file" + run get_delete_marker_and_verify_405 "$bucket_name" "$test_file" assert_success } diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index 19e4a93..175718d 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -50,6 +50,7 @@ source ./tests/commands/put_object_retention.sh source ./tests/commands/put_public_access_block.sh source ./tests/commands/select_object_content.sh source ./tests/drivers/copy_object/copy_object_rest.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh export RUN_USERS=true diff --git a/tests/test_setup_remove_static.sh b/tests/test_setup_remove_static.sh new file mode 100755 index 0000000..7642d90 --- /dev/null +++ b/tests/test_setup_remove_static.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/logger.sh + +if [ -z "$VERSITYGW_TEST_ENV" ]; then + echo "VERSITYGW_TEST_ENV variable missing" + return 1 +fi +# shellcheck disable=SC1090 +source "$VERSITYGW_TEST_ENV" +if [ "$RECREATE_BUCKETS" != "true" ] || [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ]; then + echo "for this test 'RECREATE_BUCKETS' and 'DELETE_BUCKETS_AFTER_TEST' must be set to true" + exit 1 +fi +# try to remove buckets if already there, don't worry if they're not +./tests/remove_static.sh +echo "AFTER FIRST REMOVAL" +if ! result=$(./tests/setup_static.sh 2>&1); then + echo "error setting up buckets: $result" + exit 1 +fi +echo "RESULT: $result" +if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully created"* ]]; then + echo "error creating bucket '$BUCKET_ONE_NAME'" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully created"* ]]; then + echo "error creating bucket '$BUCKET_TWO_NAME'" + exit 1 +fi +if ! result=$(./tests/remove_static.sh 2>&1); then + echo "error removing buckets: $result" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully deleted"* ]]; then + echo "error deleting bucket '$BUCKET_ONE_NAME'" + exit 1 +fi +if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully deleted"* ]]; then + echo "error deleting bucket '$BUCKET_TWO_NAME'" + exit 1 +fi +exit 0 \ No newline at end of file diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 3ec9acc..b464dc4 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -16,6 +16,7 @@ reset_bucket() { if ! check_param_count "reset_bucket" "bucket" 1 $#; then return 1 fi + log 6 "reset bucket '$1'" if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then log 3 "error logging bucket policy" @@ -66,7 +67,7 @@ reset_bucket() { # params: bucket name # return 0 if able to delete recursively, 1 if not delete_bucket_recursive() { - log 6 "delete_bucket_recursive_s3api" + log 6 "delete_bucket_recursive '$1'" if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then return 1 fi @@ -104,70 +105,6 @@ bucket_exists() { return 1 } -direct_wait_for_bucket() { - if ! check_param_count "direct_wait_for_bucket" "bucket" 1 $#; then - return 1 - fi - bucket_verification_start_time=$(date +%s) - while ! bucket_exists "$1"; do - bucket_verification_end_time=$(date +%s) - if [ $((bucket_verification_end_time-bucket_verification_start_time)) -ge 60 ]; then - log 2 "bucket existence check timeout" - return 1 - fi - sleep 5 - done - return 0 -} - -# params: client, bucket name -# return 0 for success, 1 for error -bucket_cleanup() { - log 6 "bucket_cleanup" - if ! check_param_count "bucket_cleanup" "bucket name" 1 $#; then - return 1 - fi - if [[ $RECREATE_BUCKETS == "false" ]]; then - if ! reset_bucket "$1"; then - log 2 "error deleting bucket contents" - return 1 - fi - - log 5 "bucket contents, policy, ACL deletion success" - return 0 - fi - if ! delete_bucket_recursive "$1"; then - log 2 "error with recursive bucket delete" - return 1 - fi - log 5 "bucket deletion success" - return 0 -} - -# params: client, bucket name -# return 0 for success, 1 for error -bucket_cleanup_if_bucket_exists() { - log 6 "bucket_cleanup_if_bucket_exists" - if ! check_param_count_gt "bucket name, bucket known to exist (optional)" 1 $#; then - return 1 - fi - - if [ "$2" == "false" ]; then - log 5 "skipping cleanup, since bucket doesn't exist" - return 0 - fi - - if [ "$2" == "true" ] || bucket_exists "$1"; then - if ! bucket_cleanup "$1"; then - log 2 "error deleting bucket and/or contents" - return 1 - fi - log 5 "bucket and/or bucket data deletion success" - return 0 - fi - return 0 -} - # params: client, bucket name(s) # return 0 for success, 1 for failure setup_buckets() { @@ -207,6 +144,10 @@ setup_bucket() { log 5 "util.setup_bucket: bucket name: $1" if [[ $RECREATE_BUCKETS == "true" ]]; then + if [ "$DIRECT" == "true" ]; then + log 2 "bucket not successfully deleted" + return 1 + fi if ! create_bucket "s3api" "$1"; then log 2 "error creating bucket" return 1 @@ -216,7 +157,8 @@ setup_bucket() { fi # bucket creation and resets take longer to propagate in direct mode - if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket "$1"; then + if [ "$DIRECT" == "true" ]; then + log 2 "bucket not found after creation" return 1 fi diff --git a/tests/util/util_legal_hold.sh b/tests/util/util_legal_hold.sh index c55de83..f8681bc 100644 --- a/tests/util/util_legal_hold.sh +++ b/tests/util/util_legal_hold.sh @@ -37,7 +37,7 @@ get_and_check_legal_hold() { } check_legal_hold_without_lock_enabled() { - if ! check_param_count "check_legal_hold_without_lock_enabled" "bucket, key" 2 $#; then + if ! check_param_count_v2 "bucket, key, expected error" 3 $#; then return 1 fi if get_object_legal_hold_rest "$1" "$2"; then @@ -49,8 +49,8 @@ check_legal_hold_without_lock_enabled() { log 2 "error getting error code: $code" return 1 fi - if [ "$code" != "InvalidRequest" ]; then - log 2 "code mismatch (expected 'InvalidRequest', actual '$code')" + if [ "$code" != "$3" ]; then + log 2 "code mismatch (expected '$3', actual '$code')" return 1 fi return 0 @@ -62,7 +62,8 @@ check_remove_legal_hold_versions() { fi if ! get_object_legal_hold_rest_version_id "$1" "$2" "$3"; then # shellcheck disable=SC2154 - if [[ "$legal_hold" != *"MethodNotAllowed"* ]]; then + log 5 "legal hold: $legal_hold" + if [[ "$legal_hold" != *"MethodNotAllowed"* ]] && [[ "$legal_hold" != *"NoSuchObjectLockConfiguration"* ]]; then log 2 "error getting object legal hold status with version id" return 1 fi diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index 344925e..1dcf456 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -350,7 +350,7 @@ get_delete_marker_and_verify_405() { fi log 5 "xml val: $version_id" - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then + if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then log 2 "error getting result: $result" return 1 fi diff --git a/tests/util/util_list_parts.sh b/tests/util/util_list_parts.sh index eba139f..397e0c0 100644 --- a/tests/util/util_list_parts.sh +++ b/tests/util/util_list_parts.sh @@ -64,7 +64,7 @@ check_part_list_rest() { perform_multipart_upload_rest() { if [ $# -ne 6 ]; then - log 2 "'upload_check_parts' requires bucket, key, part list" + log 2 "'upload_check_parts' requires bucket, key, part list (4 parts)" return 1 fi if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then @@ -93,7 +93,7 @@ perform_multipart_upload_rest() { return 1 fi parts_payload+="$etag4" - if ! complete_multipart_upload_rest "$1" "$2" "$3" "$4"; then + if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then log 2 "error completing multipart upload" return 1 fi diff --git a/tests/util/util_setup.sh b/tests/util/util_setup.sh index 0f349b5..b19dc01 100644 --- a/tests/util/util_setup.sh +++ b/tests/util/util_setup.sh @@ -3,10 +3,32 @@ source ./tests/drivers/params.sh setup_bucket_and_file() { - if ! check_param_count "setup_bucket_and_file" "bucket, file name" 2 $#; then + if ! check_param_count_v2 "bucket, file name" 2 $#; then return 1 fi - if ! setup_bucket_and_files "$1" "$2"; then + if ! setup_bucket_and_file_base "$1" "setup_bucket_and_files" "$2"; then + log 2 "error setting up bucket and file" + return 1 + fi + return 0 +} + +setup_bucket_and_file_v2() { + if ! check_param_count_v2 "bucket, file name" 2 $#; then + return 1 + fi + if ! setup_bucket_and_file_base "$1" "setup_bucket_and_files_v2" "$2"; then + log 2 "error setting up bucket and files" + return 1 + fi + return 0 +} + +setup_bucket_and_file_base() { + if ! check_param_count_v2 "bucket, function, file name" 3 $#; then + return 1 + fi + if ! "$2" "$1" "$3"; then log 2 "error setting up bucket and file" return 1 fi @@ -17,11 +39,33 @@ setup_bucket_and_files() { if ! check_param_count_gt "bucket, file name" 2 $#; then return 1 fi - if ! setup_bucket "$1"; then + if ! setup_bucket_and_files_base "$1" "setup_bucket" "${@:2}"; then + log 2 "error setting up bucket and files" + return 1 + fi + return 0 +} + +setup_bucket_and_files_v2() { + if ! check_param_count_gt "bucket, file name" 2 $#; then + return 1 + fi + if ! setup_bucket_and_files_base "$1" "setup_bucket_v2" "${@:2}"; then + log 2 "error setting up bucket and files" + return 1 + fi + return 0 +} + +setup_bucket_and_files_base() { + if ! check_param_count_gt "bucket, setup bucket function, file name" 3 $#; then + return 1 + fi + if ! "$2" "$1"; then log 2 "error setting up bucket" return 1 fi - if ! create_test_files "${@:2}"; then + if ! create_test_files "${@:3}"; then log 2 "error creating test files" return 1 fi @@ -58,22 +102,3 @@ setup_bucket_file_and_user() { echo "$result" return 0 } - -setup_bucket_object_lock_enabled() { - if ! check_param_count "setup_bucket_object_lock_enabled" "bucket" 1 $#; then - return 1 - fi - if ! bucket_cleanup_if_bucket_exists "$1"; then - log 2 "error cleaning up bucket" - return 1 - fi - - # in static bucket config, bucket will still exist - if ! bucket_exists "$1"; then - if ! create_bucket_object_lock_enabled "$1"; then - log 2 "error creating bucket with object lock enabled" - return 1 - fi - fi - return 0 -} diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index 909f815..81660d0 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -74,7 +74,7 @@ check_versioning_status_rest() { if ! check_param_count "check_versioning_status_rest" "bucket, expected value" 2 $#; then return 1 fi - if ! get_bucket_versioning_rest "$BUCKET_ONE_NAME"; then + if ! get_bucket_versioning_rest "$1"; then log 2 "error getting bucket versioning" return 1 fi @@ -102,7 +102,7 @@ check_versioning_status_rest() { } echo_versions() { - if ! check_param_count "echo_versions" "'Version' or 'DeleteMarker', 'Key' or 'VersionId'" 2 $#; then + if ! check_param_count_v2 "'Version' or 'DeleteMarker', 'Key' or 'VersionId', file" 3 $#; then return 1 fi if ! keys=$(echo -n "$versions" | xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" - | xmlstarlet unesc 2>&1); then @@ -113,32 +113,64 @@ echo_versions() { return 1 fi log 5 "keys to append: ${keys[*]}" + if ! result=$(truncate -s 0 "$3" 2>&1); then + log 2 "error truncating file: $result" + fi + for key in "${keys[@]}"; do + echo "$key" >> "$3" + done echo "${keys[*]}" } -parse_versions_rest() { - if ! check_param_count "parse_versions_rest" "versions variable" 1 $#; then +get_base64_version_keys_and_ids() { + if ! check_param_count_v2 "'Version' or 'DeleteMarker'" 1 $#; then return 1 fi - if ! keys=$(echo_versions "Version" "Key"); then + while IFS= read -r key && IFS= read -r vid; do + log 5 "key: $key, vid: $vid" + b_key="$(printf '%s' "$key" | base64 -w0)" + b_vid="$(printf '%s' "$vid" | base64 -w0)" + base64_pairs+=("$b_key:$b_vid") + done < <(xmlstarlet sel -t \ + -m '//*[local-name()='"\"$1\""']' \ + -v '*[local-name()="Key"]' -n \ + -v '*[local-name()="VersionId"]' -n \ + <<<"$versions" | xmlstarlet unesc) +} + +parse_base64_versions_rest() { + base64_pairs=() + if ! get_base64_version_keys_and_ids "Version"; then + log 2 "error getting version base64 keys and IDs" + return 1 + fi + if ! get_base64_version_keys_and_ids "DeleteMarker"; then + log 2 "error getting version base64 keys and IDs" + return 1 + fi +} + +parse_versions_rest() { + base64_pairs=() + if ! keys=$(echo_versions "Version" "Key" "$TEST_FILE_FOLDER/version_keys.txt"); then log 2 "error getting Version Key values: $keys" return 1 fi # shellcheck disable=SC2206 version_keys+=($keys) - if ! ids=$(echo_versions "Version" "VersionId"); then + if ! ids=$(echo_versions "Version" "VersionId" "$TEST_FILE_FOLDER/version_ids.txt"); then log 2 "error getting Version VersionId values: $ids" return 1 fi # shellcheck disable=SC2206 version_ids+=($ids) - if ! keys=$(echo_versions "DeleteMarker" "Key"); then + if ! keys=$(echo_versions "DeleteMarker" "Key" "$TEST_FILE_FOLDER/delete_marker_keys.txt"); then log 2 "error getting DeleteMarker Key values: $keys" return 1 fi # shellcheck disable=SC2206 version_keys+=($keys) - if ! ids=$(echo_versions "DeleteMarker" "VersionId"); then + if ! ids=$(echo_versions "DeleteMarker" "VersionId" "$TEST_FILE_FOLDER/delete_marker_ids.txt"); then log 2 "error getting DeleteMarker VersionId values: $ids" return 1 fi @@ -146,6 +178,7 @@ parse_versions_rest() { version_ids+=($ids) log 5 "version keys: ${version_keys[*]}" log 5 "version IDs: ${version_ids[*]}" + log 5 "base64 pairs: ${base64_pairs[*]}" return 0 }