mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-06 11:46:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: more dockerfile/direct updates, testing, fixes

Browse Source
This commit is contained in:
Luke McCrone
2025-09-18 13:02:38 -03:00
parent 62ca9b6ff3
commit 5e53c91090
29 changed files with 546 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/system.yml vendored
View File

@@ -128,6 +128,12 @@ jobs:
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "setup/remove static buckets scripts"
IAM_TYPE: folder
RUN_SET: "setup-remove-static"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
@@ -229,7 +235,10 @@ jobs:
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
- name: Time report
run: cat ${{ github.workspace }}/time.log
run: |
if [ -e ${{ github.workspace }}/time.log ]; then
cat ${{ github.workspace }}/time.log
fi
- name: Coverage report
run: |

12
tests/Dockerfile_test_bats
View File

@@ -22,6 +22,9 @@ RUN apt-get update && \
bc \
libxml2-utils \
xmlstarlet \
python3-pip \
python3-venv \
xxd \
ca-certificates && \
update-ca-certificates && \
rm -rf /var/lib/apt/lists/*
@@ -63,6 +66,15 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
cd bats-core && \
./install.sh /home/tester
# Create a shared venv & install Python deps there
ENV VENV=/opt/venv
RUN python3 -m venv "$VENV" \
&& "$VENV/bin/python" -m pip install --upgrade pip \
&& "$VENV/bin/pip" install --no-cache-dir awscrt
# Make the venv the default Python for all subsequent RUN/CMD/ENTRYPOINT
ENV PATH="$VENV/bin:${PATH}"
USER tester
COPY --chown=tester:tester . /home/tester

2
tests/commands/get_bucket_ownership_controls.sh
View File

@@ -41,7 +41,7 @@ get_bucket_ownership_controls_rest() {
if ! check_param_count "get_bucket_ownership_controls_rest" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then
log 2 "error getting bucket ownership controls: $result"
return 1
fi

4
tests/commands/get_object_legal_hold.sh
View File

@@ -66,10 +66,10 @@ get_object_legal_hold_rest_version_id() {
log 2 "error getting object legal hold: $result"
return 1
fi
legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt")
if [ "$result" != "200" ]; then
log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
log 2 "get-object-legal-hold returned code $result: $legal_hold)"
return 1
fi
legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt")
return 0
}

1
tests/commands/get_object_lock_configuration.sh
View File

@@ -53,5 +53,6 @@ get_object_lock_configuration_rest() {
return 1
fi
lock_config="$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
log 5 "lock config: $lock_config"
return 0
}

96
tests/drivers/create_bucket/create_bucket_rest.sh
View File

@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/delete_bucket/delete_bucket_rest.sh
source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh
source ./tests/drivers/get_object/get_object_rest.sh
source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh
@@ -111,4 +112,97 @@ create_bucket_and_check_acl() {
return 1
fi
return 0
}
}
get_bucket_prefix() {
if ! check_param_count_v2 "bucket prefix or name" 1 $#; then
return 1
fi
if [ "$RECREATE_BUCKETS" == "true" ]; then
# remove date/time suffix
prefix="$(echo "$1" | awk '{print substr($0, 1, length($0)-15)}')"
else
prefix="$1"
fi
echo "$prefix"
}
setup_bucket_v2() {
if ! check_param_count_v2 "bucket prefix or name" 1 $#; then
return 1
fi
if ! prefix=$(get_bucket_prefix "$1" 2>&1); then
log 2 "error getting prefix: $prefix"
return 1
fi
if ! bucket_cleanup_if_bucket_exists_v2 "$prefix"; then
log 2 "error cleaning up bucket(s), if it/they exist(s)"
return 1
fi
if [ "$RECREATE_BUCKETS" == "false" ]; then
return 0
fi
if ! create_bucket_rest_expect_success "$1" ""; then
log 2 "error creating bucket '$1'"
return 1
fi
return 0
}
get_bucket_name() {
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
fi
if [ "$RECREATE_BUCKETS" == "false" ]; then
echo "$1"
fi
echo "$1-$(date +%Y%m%d%H%M%S)"
}
setup_bucket_object_lock_enabled_v2() {
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
fi
if ! prefix=$(get_bucket_prefix "$1" 2>&1); then
log 2 "error getting prefix: $prefix"
return 1
fi
if ! bucket_cleanup_if_bucket_exists_v2 "$prefix"; then
log 2 "error cleaning up bucket"
return 1
fi
if [ "$RECREATE_BUCKETS" == "true" ]; then
if ! create_bucket_object_lock_enabled "$1"; then
log 2 "error creating bucket '$1' with object lock enabled"
return 1
fi
fi
return 0
}
setup_bucket_object_lock_enabled() {
if ! check_param_count "setup_bucket_object_lock_enabled" "bucket" 1 $#; then
return 1
fi
if ! bucket_cleanup_if_bucket_exists "$1"; then
log 2 "error cleaning up bucket"
return 1
fi
if [ "$DIRECT" == "true" ] && [ "$RECREATE_BUCKETS" == "true" ]; then
log 2 "bucket not confirmed as deleted"
return 1
fi
# in static bucket config, bucket will still exist
if ! bucket_exists "$1"; then
if ! create_bucket_object_lock_enabled "$1"; then
log 2 "error creating bucket with object lock enabled"
return 1
fi
fi
if [ "$DIRECT" == "true" ]; then
log 2 "bucket not confirmed as created"
return 1
fi
return 0
}

121
tests/drivers/delete_bucket/delete_bucket_rest.sh Normal file
View File

@@ -0,0 +1,121 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/list_buckets.sh
delete_buckets_with_prefix() {
if ! check_param_count_v2 "bucket prefix" 1 $#; then
return 1
fi
if [ "$1" == "" ]; then
log 2 "delete_buckets_with_prefix requires non-empty prefix"
return 1
fi
if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then
log 2 "error listing buckets with prefix"
return 1
fi
# shellcheck disable=SC2154
log 5 "buckets: ${bucket_array[*]}"
for bucket in "${bucket_array[@]}"; do
if ! delete_bucket_recursive "$bucket"; then
log 2 "error with recursive bucket delete of bucket '$bucket'"
return 1
fi
done
return 0
}
cleanup_buckets() {
if ! bucket_cleanup_if_bucket_exists_v2 "$BUCKET_ONE_NAME"; then
log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
fi
if ! bucket_cleanup_if_bucket_exists_v2 "$BUCKET_TWO_NAME"; then
log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
fi
}
# params: client, bucket name
# return 0 for success, 1 for error
bucket_cleanup() {
log 6 "bucket_cleanup"
if ! check_param_count "bucket_cleanup" "bucket name" 1 $#; then
return 1
fi
if [[ $RECREATE_BUCKETS == "false" ]]; then
if ! reset_bucket "$1"; then
log 2 "error deleting bucket contents"
return 1
fi
log 5 "bucket contents, policy, ACL deletion success"
return 0
fi
if ! delete_bucket_recursive "$1"; then
log 2 "error with recursive bucket delete"
return 1
fi
log 5 "bucket deletion success"
return 0
}
# params: client, bucket name
# return 0 for success, 1 for error
bucket_cleanup_if_bucket_exists() {
log 6 "bucket_cleanup_if_bucket_exists"
if ! check_param_count_gt "bucket name, bucket known to exist (optional)" 1 $#; then
return 1
fi
if [ "$2" == "false" ]; then
log 5 "skipping cleanup, since bucket doesn't exist"
return 0
fi
if [ "$2" == "true" ] || bucket_exists "$1"; then
if ! bucket_cleanup "$1"; then
log 2 "error deleting bucket and/or contents"
return 1
fi
log 5 "bucket and/or bucket data deletion success"
return 0
fi
return 0
}
bucket_cleanup_if_bucket_exists_v2() {
log 6 "bucket_cleanup_if_bucket_exists_v2"
if ! check_param_count_gt "bucket name or prefix" 1 $#; then
return 1
fi
if [[ "$RECREATE_BUCKETS" == "false" ]]; then
if ! bucket_exists "$1"; then
log 2 "When RECREATE_BUCKETS isn't set to \"true\", bucket with full env name should be pre-created by user"
return 1
fi
if ! reset_bucket "$1"; then
log 2 "error resetting bucket before tests"
return 1
fi
return 0
else
if ! delete_buckets_with_prefix "$1"; then
log 2 "error deleting buckets with prefix '$1'"
return 1
fi
fi
return 0
}

41
tests/drivers/delete_object/delete_object_rest.sh
View File

@@ -36,14 +36,14 @@ list_and_delete_objects() {
fi
done
if ! delete_old_versions "$1"; then
if ! delete_old_versions_base64 "$1"; then
log 2 "error deleting old version"
return 1
fi
return 0
}
delete_old_versions() {
delete_old_versions_base64() {
if ! check_param_count "delete_old_versions" "bucket" 1 $#; then
return 1
fi
@@ -53,45 +53,52 @@ delete_old_versions() {
fi
# shellcheck disable=SC2154
log 5 "versions: $versions"
version_keys=()
version_ids=()
if ! parse_version_data_by_type "rest" "$2"; then
if ! parse_base64_versions_rest; then
log 2 "error parsing version data"
return 1
fi
log 5 "version keys: ${version_keys[*]}"
log 5 "version IDs: ${version_ids[*]}"
for idx in "${!version_keys[@]}"; do
if ! delete_object_version_with_or_without_retention "$1"; then
log 5 "base64 versions: ${base64_pairs[*]}"
for pair in "${base64_pairs[@]}"; do
log 5 "pair: $pair"
if ! delete_object_version_with_or_without_retention_base64 "$1" "$pair"; then
log 2 "error deleting version with or without retention"
return 1
fi
done
}
delete_object_version_with_or_without_retention() {
if ! check_param_count "delete_object_version_with_or_without_retention" "bucket" 1 $#; then
delete_object_version_with_or_without_retention_base64() {
if ! check_param_count_v2 "bucket, key/value pair" 2 $#; then
return 1
fi
IFS=":" read -ra key_and_id <<< "$2"
log 5 "key and ID: ${key_and_id[*]}"
if ! key=$(printf '%s' "${key_and_id[0]}" | base64 --decode 2>&1); then
log 2 "error decoding key: $key"
return 1
fi
if ! id=$(printf '%s' "${key_and_id[1]}" | base64 --decode 2>&1); then
log 2 "error decoding ID: $id"
return 1
fi
log 5 "idx: $idx"
log 5 "version ID: ${version_ids[$idx]}"
# shellcheck disable=SC2154
if [ "$lock_config_exists" == "true" ]; then
if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
if ! check_remove_legal_hold_versions "$1" "$key" "$id"; then
log 2 "error checking, removing legal hold versions"
return 1
fi
if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
if ! delete_object_version_rest_bypass_retention "$1" "$key" "$id"; then
log 2 "error deleting object version, bypassing retention"
return 1
fi
else
if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
if ! delete_object_version_rest "$1" "$key" "$id"; then
log 2 "error deleting object version"
return 1
fi
fi
log 5 "successfully deleted version with key '${version_keys[$idx]}', id '${version_ids[$idx]}'"
log 5 "successfully deleted version with key '$key', id '$id'"
return 0
}

1
tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh
View File

@@ -22,6 +22,7 @@ check_object_lock_config() {
lock_config_exists=true
if ! get_object_lock_configuration "rest" "$1"; then
# shellcheck disable=SC2154
log 5 "lock config error: $get_object_lock_config_err"
if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then
# shellcheck disable=SC2034
lock_config_exists=false

6
tests/drivers/list_buckets/list_buckets_rest.sh
View File

@@ -64,9 +64,13 @@ parse_bucket_list() {
bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1")
bucket_array=()
while read -r bucket; do
bucket_array+=("$bucket")
if [ -n "$bucket" ]; then
log 5 "reading bucket '$bucket'"
bucket_array+=("$bucket")
fi
done <<< "$bucket_list"
log 5 "bucket array: ${bucket_array[*]}"
log 5 "bucket array length: ${#bucket_array[@]}"
}
parse_buckets_and_continuation_token() {

2
tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh
View File

@@ -64,7 +64,7 @@ check_ownership_rule_and_reset_acl() {
return 1
fi
if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then
log 2 "error getting bucket ownership controls"
log 2 "error getting bucket ownership controls: $object_ownership_rule"
return 1
fi
log 5 "ownership rule: $object_ownership_rule"

5
tests/drivers/rest.sh
View File

@@ -158,8 +158,9 @@ send_rest_command_expect_success_callback() {
log 2 "error sending command: $result"
return 1
fi
if [ "$result" != "$3" ]; then
log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))"
response_code="$(echo "$result" | tail -n 1)"
if [ "$response_code" != "$3" ]; then
log 2 "expected '$3', was '$response_code' ($(cat "$TEST_FILE_FOLDER/output.txt"))"
return 1
fi
if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then

17
tests/remove_static.sh
View File

@@ -18,31 +18,32 @@ source ./tests/setup.sh
source ./tests/util/util_object.sh
delete_bucket_if_exists() {
if [[ $# -ne 2 ]]; then
log 2 "delete_bucket_if_exists command missing command type, name"
if [[ $# -ne 1 ]]; then
log 2 "delete_bucket_if_exists command missing bucket name"
return 1
fi
bucket_exists "$1" "$2" || local exists_result=$?
bucket_exists "$1" || local exists_result=$?
if [[ $exists_result -eq 2 ]]; then
log 2 "error checking if bucket exists"
return 1
fi
if [[ $exists_result -eq 1 ]]; then
echo "bucket '$2' doesn't exist, skipping"
echo "bucket '$1' doesn't exist, skipping"
return 0
fi
if ! delete_bucket_recursive "$1" "$2"; then
log 5 "attempting to delete bucket '$1'"
if ! delete_bucket_recursive "$1"; then
log 2 "error deleting bucket"
return 1
fi
echo "bucket '$2' successfully deleted"
echo "bucket '$1' successfully deleted"
return 0
}
base_setup
if ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then
if ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_ONE_NAME"; then
log 2 "error deleting static bucket one"
elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then
elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "$BUCKET_TWO_NAME"; then
log 2 "error deleting static bucket two"
fi
if ! stop_versity; then

3
tests/rest_scripts/get_object_legal_hold.sh
View File

@@ -21,7 +21,7 @@ source ./tests/rest_scripts/rest.sh
# shellcheck disable=SC2153
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2154
key="$OBJECT_KEY"
key="$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")')"
# shellcheck disable=SC2153,SC2154
version_id="$VERSION_ID"
@@ -48,4 +48,5 @@ curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_
curl_command+=("${header_fields[@]}")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
log_rest 5 "curl command: ${curl_command[*]}"
eval "${curl_command[*]}" 2>&1

9
tests/run.sh
View File

@@ -61,7 +61,7 @@ handle_param() {
s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\
rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\
rest-put-object|rest-versioning|rest-bucket|rest-multipart|rest-delete-bucket-ownership-controls|\
rest-delete-bucket-tagging)
rest-delete-bucket-tagging|setup-remove-static)
run_suite "$1"
;;
*) # Handle unrecognized options or positional arguments
@@ -247,8 +247,13 @@ run_suite() {
s3api-user)
echo "Running s3api user tests ..."
"$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
;;
setup-remove-static)
echo "Testing setup/remove static bucket scripts ..."
VERSITYGW_TEST_ENV="$VERSITYGW_TEST_ENV" ./tests/test_setup_remove_static.sh || exit_code=$?
;;
esac
if [ $exit_code -ne 0 ]; then
if [ "$exit_code" -ne 0 ]; then
exit 1
fi
}

12
tests/setup.sh
View File

@@ -17,6 +17,7 @@
source ./tests/env.sh
source ./tests/report.sh
source ./tests/setup_mc.sh
source ./tests/drivers/delete_bucket/delete_bucket_rest.sh
source ./tests/util/util_object.sh
source ./tests/versity.sh
@@ -146,14 +147,9 @@ post_versity_cleanup() {
teardown() {
# shellcheck disable=SC2154
log 4 "********** BEGIN TEARDOWN **********"
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then
log 5 "deleting or clearing buckets"
if ! bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"; then
log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
fi
if ! bucket_cleanup_if_bucket_exists "$BUCKET_TWO_NAME"; then
log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
fi
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
log 3 "error cleaning up buckets after test"
return 1
fi
if [ "$SKIP_USERS_TESTS" != "true" ]; then
if [ -n "$USERNAME_ONE" ]; then

27
tests/test_rest.sh
View File

@@ -29,6 +29,7 @@ source ./tests/commands/put_bucket_versioning.sh
source ./tests/commands/put_object.sh
source ./tests/commands/put_object_retention.sh
source ./tests/commands/put_object_tagging.sh
source ./tests/drivers/create_bucket/create_bucket_rest.sh
source ./tests/drivers/copy_object/copy_object_rest.sh
source ./tests/drivers/get_bucket_ownership_controls/get_bucket_ownership_controls_rest.sh
source ./tests/drivers/head_object/head_object_rest.sh
@@ -111,13 +112,8 @@ test_file="test_file"
test_key="TestKey"
test_value="TestValue"
run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
# in static bucket config, bucket will still exist
if ! bucket_exists "$BUCKET_ONE_NAME"; then
run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
fi
run create_test_files "$test_file"
assert_success
@@ -135,13 +131,30 @@ test_file="test_file"
}
@test "REST - legal hold, get without config" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "test requires object lock not to be enabled"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file"
run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "InvalidRequest"
assert_success
}
@test "REST - legal hold, object lock enabled w/o specific object lock set" {
run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" "NoSuchObjectLockConfiguration"
assert_success
}

8
tests/test_rest_bucket.sh
View File

@@ -37,10 +37,14 @@ source ./tests/util/util_tags.sh
export RUN_USERS=true
@test "REST - HeadBucket" {
run setup_bucket "$BUCKET_ONE_NAME"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run setup_bucket_v2 "$bucket_name"
assert_success
run head_bucket_rest "$BUCKET_ONE_NAME"
run head_bucket_rest "$bucket_name"
assert_success
}

5
tests/test_rest_delete_bucket_ownership_controls.sh
View File

@@ -29,8 +29,9 @@ export RUN_USERS=true
fi
run setup_bucket_and_user_v2 "$BUCKET_ONE_NAME" "$USERNAME_ONE" "$PASSWORD_ONE"
assert_success
username=${lines[2]}
password=${lines[3]}
username=${lines[${#lines[@]}-2]}
password=${lines[${#lines[@]}-1]}
log 5 "output: $output"
log 5 "username: $username, password: $password"
run send_rest_go_command_expect_error "403" "AccessDenied" "Access Denied" "-awsAccessKeyId" "$username" "-awsSecretAccessKey" "$password" \

1
tests/test_rest_multipart.sh
View File

@@ -20,6 +20,7 @@ load ./bats-assert/load
source ./tests/setup.sh
source ./tests/drivers/rest.sh
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
source ./tests/drivers/list_buckets/list_buckets_rest.sh
source ./tests/drivers/upload_part/upload_part_rest.sh
source ./tests/util/util_file.sh
source ./tests/util/util_list_parts.sh

98
tests/test_rest_versioning.sh
View File

@@ -20,87 +20,103 @@ load ./bats-assert/load
source ./tests/setup.sh
source ./tests/commands/get_object.sh
source ./tests/commands/put_object.sh
source ./tests/drivers/create_bucket/create_bucket_rest.sh
source ./tests/drivers/list_buckets/list_buckets_rest.sh
source ./tests/util/util_rest.sh
source ./tests/util/util_setup.sh
test_file="test_file"
@test "REST - check, enable, suspend versioning" {
run setup_bucket "$BUCKET_ONE_NAME"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run setup_bucket_v2 "$bucket_name"
assert_success
log 5 "get versioning"
run check_versioning_status_rest "$BUCKET_ONE_NAME" ""
run check_versioning_status_rest "$bucket_name" ""
assert_success
run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled"
run put_bucket_versioning_rest "$bucket_name" "Enabled"
assert_success
run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled"
run check_versioning_status_rest "$bucket_name" "Enabled"
assert_success
run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended"
run put_bucket_versioning_rest "$bucket_name" "Suspended"
assert_success
run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended"
run check_versioning_status_rest "$bucket_name" "Suspended"
assert_success
}
@test "test_rest_versioning" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
run get_and_check_versions_rest "$bucket_name" "$test_file" "1" "true" "true"
assert_success
run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
run put_bucket_versioning_rest "$bucket_name" "Enabled"
assert_success
run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
run get_and_check_versions_rest "$bucket_name" "$test_file" "1" "true" "true"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true"
run get_and_check_versions_rest "$bucket_name" "$test_file" "2" "true" "false" "false" "true"
assert_success
}
@test "versioning - add version, then delete and check for marker" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
run put_bucket_versioning_rest "$bucket_name" "Enabled"
assert_success
run delete_object_rest "$BUCKET_ONE_NAME" "$test_file"
run delete_object_rest "$bucket_name" "$test_file"
assert_success
run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file"
run check_versions_after_file_deletion "$bucket_name" "$test_file"
assert_success
}
@test "versioning - retrieve after delete" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
run put_bucket_versioning_rest "$bucket_name" "Enabled"
assert_success
run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
run delete_object "s3api" "$bucket_name" "$test_file"
assert_success
run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run get_object "s3api" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_failure
}
@@ -108,25 +124,23 @@ test_file="test_file"
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
skip "test isn't valid for this configuration"
fi
run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
# in static bucket config, bucket will still exist
if ! bucket_exists "$BUCKET_ONE_NAME"; then
run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
fi
run setup_bucket_object_lock_enabled_v2 "$bucket_name"
assert_success
run create_test_files "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
run delete_object "s3api" "$bucket_name" "$test_file"
assert_success
run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file"
run verify_object_not_found "$bucket_name" "$test_file"
assert_success
}
@@ -134,24 +148,22 @@ test_file="test_file"
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
skip "test isn't valid for this configuration"
fi
run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
# in static bucket config, bucket will still exist
if ! bucket_exists "$BUCKET_ONE_NAME"; then
run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
fi
run setup_bucket_object_lock_enabled_v2 "$bucket_name"
assert_success
run create_test_files "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
run delete_object "s3api" "$bucket_name" "$test_file"
assert_success
run get_delete_marker_and_verify_405 "$BUCKET_ONE_NAME" "$test_file"
run get_delete_marker_and_verify_405 "$bucket_name" "$test_file"
assert_success
}

1
tests/test_s3api_object.sh
View File

@@ -50,6 +50,7 @@ source ./tests/commands/put_object_retention.sh
source ./tests/commands/put_public_access_block.sh
source ./tests/commands/select_object_content.sh
source ./tests/drivers/copy_object/copy_object_rest.sh
source ./tests/drivers/list_buckets/list_buckets_rest.sh
export RUN_USERS=true

57
tests/test_setup_remove_static.sh Executable file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/logger.sh
if [ -z "$VERSITYGW_TEST_ENV" ]; then
echo "VERSITYGW_TEST_ENV variable missing"
return 1
fi
# shellcheck disable=SC1090
source "$VERSITYGW_TEST_ENV"
if [ "$RECREATE_BUCKETS" != "true" ] || [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ]; then
echo "for this test 'RECREATE_BUCKETS' and 'DELETE_BUCKETS_AFTER_TEST' must be set to true"
exit 1
fi
# try to remove buckets if already there, don't worry if they're not
./tests/remove_static.sh
echo "AFTER FIRST REMOVAL"
if ! result=$(./tests/setup_static.sh 2>&1); then
echo "error setting up buckets: $result"
exit 1
fi
echo "RESULT: $result"
if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully created"* ]]; then
echo "error creating bucket '$BUCKET_ONE_NAME'"
exit 1
fi
if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully created"* ]]; then
echo "error creating bucket '$BUCKET_TWO_NAME'"
exit 1
fi
if ! result=$(./tests/remove_static.sh 2>&1); then
echo "error removing buckets: $result"
exit 1
fi
if [[ "$result" != *"bucket '$BUCKET_ONE_NAME' successfully deleted"* ]]; then
echo "error deleting bucket '$BUCKET_ONE_NAME'"
exit 1
fi
if [[ "$result" != *"bucket '$BUCKET_TWO_NAME' successfully deleted"* ]]; then
echo "error deleting bucket '$BUCKET_TWO_NAME'"
exit 1
fi
exit 0

74
tests/util/util_bucket.sh
View File

@@ -16,6 +16,7 @@ reset_bucket() {
if ! check_param_count "reset_bucket" "bucket" 1 $#; then
return 1
fi
log 6 "reset bucket '$1'"
if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then
log 3 "error logging bucket policy"
@@ -66,7 +67,7 @@ reset_bucket() {
# params: bucket name
# return 0 if able to delete recursively, 1 if not
delete_bucket_recursive() {
log 6 "delete_bucket_recursive_s3api"
log 6 "delete_bucket_recursive '$1'"
if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then
return 1
fi
@@ -104,70 +105,6 @@ bucket_exists() {
return 1
}
direct_wait_for_bucket() {
if ! check_param_count "direct_wait_for_bucket" "bucket" 1 $#; then
return 1
fi
bucket_verification_start_time=$(date +%s)
while ! bucket_exists "$1"; do
bucket_verification_end_time=$(date +%s)
if [ $((bucket_verification_end_time-bucket_verification_start_time)) -ge 60 ]; then
log 2 "bucket existence check timeout"
return 1
fi
sleep 5
done
return 0
}
# params: client, bucket name
# return 0 for success, 1 for error
bucket_cleanup() {
log 6 "bucket_cleanup"
if ! check_param_count "bucket_cleanup" "bucket name" 1 $#; then
return 1
fi
if [[ $RECREATE_BUCKETS == "false" ]]; then
if ! reset_bucket "$1"; then
log 2 "error deleting bucket contents"
return 1
fi
log 5 "bucket contents, policy, ACL deletion success"
return 0
fi
if ! delete_bucket_recursive "$1"; then
log 2 "error with recursive bucket delete"
return 1
fi
log 5 "bucket deletion success"
return 0
}
# params: client, bucket name
# return 0 for success, 1 for error
bucket_cleanup_if_bucket_exists() {
log 6 "bucket_cleanup_if_bucket_exists"
if ! check_param_count_gt "bucket name, bucket known to exist (optional)" 1 $#; then
return 1
fi
if [ "$2" == "false" ]; then
log 5 "skipping cleanup, since bucket doesn't exist"
return 0
fi
if [ "$2" == "true" ] || bucket_exists "$1"; then
if ! bucket_cleanup "$1"; then
log 2 "error deleting bucket and/or contents"
return 1
fi
log 5 "bucket and/or bucket data deletion success"
return 0
fi
return 0
}
# params: client, bucket name(s)
# return 0 for success, 1 for failure
setup_buckets() {
@@ -207,6 +144,10 @@ setup_bucket() {
log 5 "util.setup_bucket: bucket name: $1"
if [[ $RECREATE_BUCKETS == "true" ]]; then
if [ "$DIRECT" == "true" ]; then
log 2 "bucket not successfully deleted"
return 1
fi
if ! create_bucket "s3api" "$1"; then
log 2 "error creating bucket"
return 1
@@ -216,7 +157,8 @@ setup_bucket() {
fi
# bucket creation and resets take longer to propagate in direct mode
if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket "$1"; then
if [ "$DIRECT" == "true" ]; then
log 2 "bucket not found after creation"
return 1
fi

9
tests/util/util_legal_hold.sh
View File

@@ -37,7 +37,7 @@ get_and_check_legal_hold() {
}
check_legal_hold_without_lock_enabled() {
if ! check_param_count "check_legal_hold_without_lock_enabled" "bucket, key" 2 $#; then
if ! check_param_count_v2 "bucket, key, expected error" 3 $#; then
return 1
fi
if get_object_legal_hold_rest "$1" "$2"; then
@@ -49,8 +49,8 @@ check_legal_hold_without_lock_enabled() {
log 2 "error getting error code: $code"
return 1
fi
if [ "$code" != "InvalidRequest" ]; then
log 2 "code mismatch (expected 'InvalidRequest', actual '$code')"
if [ "$code" != "$3" ]; then
log 2 "code mismatch (expected '$3', actual '$code')"
return 1
fi
return 0
@@ -62,7 +62,8 @@ check_remove_legal_hold_versions() {
fi
if ! get_object_legal_hold_rest_version_id "$1" "$2" "$3"; then
# shellcheck disable=SC2154
if [[ "$legal_hold" != *"MethodNotAllowed"* ]]; then
log 5 "legal hold: $legal_hold"
if [[ "$legal_hold" != *"MethodNotAllowed"* ]] && [[ "$legal_hold" != *"NoSuchObjectLockConfiguration"* ]]; then
log 2 "error getting object legal hold status with version id"
return 1
fi

2
tests/util/util_list_objects.sh
View File

@@ -350,7 +350,7 @@ get_delete_marker_and_verify_405() {
fi
log 5 "xml val: $version_id"
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
return 1
fi

4
tests/util/util_list_parts.sh
View File

@@ -64,7 +64,7 @@ check_part_list_rest() {
perform_multipart_upload_rest() {
if [ $# -ne 6 ]; then
log 2 "'upload_check_parts' requires bucket, key, part list"
log 2 "'upload_check_parts' requires bucket, key, part list (4 parts)"
return 1
fi
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
@@ -93,7 +93,7 @@ perform_multipart_upload_rest() {
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>4</PartNumber></Part>"
if ! complete_multipart_upload_rest "$1" "$2" "$3" "$4"; then
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
log 2 "error completing multipart upload"
return 1
fi

71
tests/util/util_setup.sh
View File

@@ -3,10 +3,32 @@
source ./tests/drivers/params.sh
setup_bucket_and_file() {
if ! check_param_count "setup_bucket_and_file" "bucket, file name" 2 $#; then
if ! check_param_count_v2 "bucket, file name" 2 $#; then
return 1
fi
if ! setup_bucket_and_files "$1" "$2"; then
if ! setup_bucket_and_file_base "$1" "setup_bucket_and_files" "$2"; then
log 2 "error setting up bucket and file"
return 1
fi
return 0
}
setup_bucket_and_file_v2() {
if ! check_param_count_v2 "bucket, file name" 2 $#; then
return 1
fi
if ! setup_bucket_and_file_base "$1" "setup_bucket_and_files_v2" "$2"; then
log 2 "error setting up bucket and files"
return 1
fi
return 0
}
setup_bucket_and_file_base() {
if ! check_param_count_v2 "bucket, function, file name" 3 $#; then
return 1
fi
if ! "$2" "$1" "$3"; then
log 2 "error setting up bucket and file"
return 1
fi
@@ -17,11 +39,33 @@ setup_bucket_and_files() {
if ! check_param_count_gt "bucket, file name" 2 $#; then
return 1
fi
if ! setup_bucket "$1"; then
if ! setup_bucket_and_files_base "$1" "setup_bucket" "${@:2}"; then
log 2 "error setting up bucket and files"
return 1
fi
return 0
}
setup_bucket_and_files_v2() {
if ! check_param_count_gt "bucket, file name" 2 $#; then
return 1
fi
if ! setup_bucket_and_files_base "$1" "setup_bucket_v2" "${@:2}"; then
log 2 "error setting up bucket and files"
return 1
fi
return 0
}
setup_bucket_and_files_base() {
if ! check_param_count_gt "bucket, setup bucket function, file name" 3 $#; then
return 1
fi
if ! "$2" "$1"; then
log 2 "error setting up bucket"
return 1
fi
if ! create_test_files "${@:2}"; then
if ! create_test_files "${@:3}"; then
log 2 "error creating test files"
return 1
fi
@@ -58,22 +102,3 @@ setup_bucket_file_and_user() {
echo "$result"
return 0
}
setup_bucket_object_lock_enabled() {
if ! check_param_count "setup_bucket_object_lock_enabled" "bucket" 1 $#; then
return 1
fi
if ! bucket_cleanup_if_bucket_exists "$1"; then
log 2 "error cleaning up bucket"
return 1
fi
# in static bucket config, bucket will still exist
if ! bucket_exists "$1"; then
if ! create_bucket_object_lock_enabled "$1"; then
log 2 "error creating bucket with object lock enabled"
return 1
fi
fi
return 0
}

49
tests/util/util_versioning.sh
View File

@@ -74,7 +74,7 @@ check_versioning_status_rest() {
if ! check_param_count "check_versioning_status_rest" "bucket, expected value" 2 $#; then
return 1
fi
if ! get_bucket_versioning_rest "$BUCKET_ONE_NAME"; then
if ! get_bucket_versioning_rest "$1"; then
log 2 "error getting bucket versioning"
return 1
fi
@@ -102,7 +102,7 @@ check_versioning_status_rest() {
}
echo_versions() {
if ! check_param_count "echo_versions" "'Version' or 'DeleteMarker', 'Key' or 'VersionId'" 2 $#; then
if ! check_param_count_v2 "'Version' or 'DeleteMarker', 'Key' or 'VersionId', file" 3 $#; then
return 1
fi
if ! keys=$(echo -n "$versions" | xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" - | xmlstarlet unesc 2>&1); then
@@ -113,32 +113,64 @@ echo_versions() {
return 1
fi
log 5 "keys to append: ${keys[*]}"
if ! result=$(truncate -s 0 "$3" 2>&1); then
log 2 "error truncating file: $result"
fi
for key in "${keys[@]}"; do
echo "$key" >> "$3"
done
echo "${keys[*]}"
}
parse_versions_rest() {
if ! check_param_count "parse_versions_rest" "versions variable" 1 $#; then
get_base64_version_keys_and_ids() {
if ! check_param_count_v2 "'Version' or 'DeleteMarker'" 1 $#; then
return 1
fi
if ! keys=$(echo_versions "Version" "Key"); then
while IFS= read -r key && IFS= read -r vid; do
log 5 "key: $key, vid: $vid"
b_key="$(printf '%s' "$key" | base64 -w0)"
b_vid="$(printf '%s' "$vid" | base64 -w0)"
base64_pairs+=("$b_key:$b_vid")
done < <(xmlstarlet sel -t \
-m '//*[local-name()='"\"$1\""']' \
-v '*[local-name()="Key"]' -n \
-v '*[local-name()="VersionId"]' -n \
<<<"$versions" | xmlstarlet unesc)
}
parse_base64_versions_rest() {
base64_pairs=()
if ! get_base64_version_keys_and_ids "Version"; then
log 2 "error getting version base64 keys and IDs"
return 1
fi
if ! get_base64_version_keys_and_ids "DeleteMarker"; then
log 2 "error getting version base64 keys and IDs"
return 1
fi
}
parse_versions_rest() {
base64_pairs=()
if ! keys=$(echo_versions "Version" "Key" "$TEST_FILE_FOLDER/version_keys.txt"); then
log 2 "error getting Version Key values: $keys"
return 1
fi
# shellcheck disable=SC2206
version_keys+=($keys)
if ! ids=$(echo_versions "Version" "VersionId"); then
if ! ids=$(echo_versions "Version" "VersionId" "$TEST_FILE_FOLDER/version_ids.txt"); then
log 2 "error getting Version VersionId values: $ids"
return 1
fi
# shellcheck disable=SC2206
version_ids+=($ids)
if ! keys=$(echo_versions "DeleteMarker" "Key"); then
if ! keys=$(echo_versions "DeleteMarker" "Key" "$TEST_FILE_FOLDER/delete_marker_keys.txt"); then
log 2 "error getting DeleteMarker Key values: $keys"
return 1
fi
# shellcheck disable=SC2206
version_keys+=($keys)
if ! ids=$(echo_versions "DeleteMarker" "VersionId"); then
if ! ids=$(echo_versions "DeleteMarker" "VersionId" "$TEST_FILE_FOLDER/delete_marker_ids.txt"); then
log 2 "error getting DeleteMarker VersionId values: $ids"
return 1
fi
@@ -146,6 +178,7 @@ parse_versions_rest() {
version_ids+=($ids)
log 5 "version keys: ${version_keys[*]}"
log 5 "version IDs: ${version_ids[*]}"
log 5 "base64 pairs: ${base64_pairs[*]}"
return 0
}