mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-28 20:06:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: allow versitygw command customization, website testing, s3 config fix

Browse Source
This commit is contained in:
Luke McCrone
2026-08-26 19:28:27 -03:00
parent 12c8d52e22
commit 7811ba8337
23 changed files with 565 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/system.yml
+12 -4
View File
@@ -69,6 +69,7 @@ jobs:
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: /home/tester/test-files/versioning
COMMAND_LOG: /home/tester/test-files/command.log
TEST_LOG_FILE: /home/tester/test-files/test.log
TIME_LOG: /home/tester/test-files/time.log
PYTHON_ENV_FOLDER: /home/tester/test-files/env
AUTOGENERATE_USERS: true
@@ -76,6 +77,11 @@ jobs:
AWS_REGION: ${{ matrix.AWS_REGION }}
COVERAGE_LOG: /home/tester/test-files/coverage.log
TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml
QUICK_COMPARE_SIZE: 1048576
GENERATE_COMMAND_EXECUTABLE: /home/tester/test-files/generateCommand
WEBSITE_DOMAIN: localhost
WEBSITE: :8071
VERSITY_LOG_FILE: /home/tester/test-files/versity.log
run: |
cat > /home/tester/test-files/.env <<EOF
IAM_TYPE=$IAM_TYPE
@@ -107,6 +113,7 @@ jobs:
REMOVE_TEST_FILE_FOLDER=$REMOVE_TEST_FILE_FOLDER
VERSIONING_DIR=$VERSIONING_DIR
COMMAND_LOG=$COMMAND_LOG
TEST_LOG_FILE=$TEST_LOG_FILE
TIME_LOG=$TIME_LOG
PYTHON_ENV_FOLDER=$PYTHON_ENV_FOLDER
AUTOGENERATE_USERS=$AUTOGENERATE_USERS
@@ -114,10 +121,11 @@ jobs:
AWS_REGION=$AWS_REGION
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
GENERATE_COMMAND_EXECUTABLE=/home/tester/test-files/generateCommand
WEBSITE_DOMAIN=localhost
WEBSITE=:7071
QUICK_COMPARE_SIZE=$QUICK_COMPARE_SIZE
GENERATE_COMMAND_EXECUTABLE=$GENERATE_COMMAND_EXECUTABLE
WEBSITE_DOMAIN=$WEBSITE_DOMAIN
WEBSITE=$WEBSITE
VERSITY_LOG_FILE=$VERSITY_LOG_FILE
EOF
sudo chown tester:tester /home/tester/test-files/.env
sudo chmod 600 /home/tester/test-files/.env
tests/commands/head_bucket.sh
+1 -1
View File
@@ -64,7 +64,7 @@ head_bucket_rest() {
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/head_bucket.sh 2>&1); then
log 2 "error getting head bucket: $result"
log 2 "error with HeadBucket command: $result"
return 2
fi
local callback_code=0
tests/commands/put_bucket_policy.sh
+34
View File
@@ -90,3 +90,37 @@ put_bucket_policy_rest_200_or_204() {
fi
return 0
}
create_website_with_random_string_and_add_permissions() {
if ! check_param_count_v2 "bucket name" 1 $#; then
return 1
fi
local bucket_name="$1"
local response random_string policy_file
if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then
log 2 "error creating website: $response"
return 1
fi
read -r _ random_string <<< "$response"
if [ "$DIRECT" == "true" ]; then
if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"; then
log 2 "error putting public access block"
return 1
fi
fi
if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then
log 2 "error setting up policy: $response"
return 1
fi
policy_file="$response"
if ! put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then
log 2 '"error putting bucket policy'
return 1
fi
printf '%s\n' "$random_string"
return 0
}
tests/drivers/create_bucket/create_bucket_rest.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/create_bucket.sh
source ./tests/drivers/delete_bucket/delete_bucket_rest.sh
source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh
source ./tests/drivers/get_object/get_object_rest.sh
tests/drivers/delete_bucket/delete_bucket_rest.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/delete_bucket.sh
source ./tests/commands/get_bucket_location.sh
source ./tests/commands/list_buckets.sh
source ./tests/drivers/get_bucket_location/get_bucket_location_rest.sh
tests/drivers/delete_object/delete_object_rest.sh
+3
View File
@@ -14,10 +14,13 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/list_object_versions.sh
source ./tests/commands/list_objects.sh
source ./tests/commands/list_objects_v2.sh
source ./tests/drivers/list_object_versions/list_object_versions_rest.sh
source ./tests/drivers/xml.sh
source ./tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh
source ./tests/util/util_object.sh
list_and_delete_objects() {
if ! check_param_count_v2 "bucket, lock config exists" 2 $#; then
tests/drivers/get_bucket_website/get_bucket_website_rest.sh
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
compare_website_suffixes() {
if ! check_param_count_v2 "data file, expected suffix" 2 $#; then
return 1
fi
local data_file="$1" expected_suffix="$2"
if ! check_xml_element "$data_file" "$expected_suffix" "WebsiteConfiguration" "IndexDocument" "Suffix"; then
log 2 "error checking xml element"
return 1
fi
return 0
}
check_index_document_suffix() {
if ! check_param_count_v2 "bucket name, expected suffix" 2 $#; then
return 1
fi
local bucket_name="$1" expected_suffix="$2"
if ! send_rest_go_command_callback "200" "compare_website_suffixes" "-bucketName" "$bucket_name" "-query" "website" "--" "$expected_suffix"; then
log 2 "error sending get website command or comparing suffix"
return 1
fi
return 0
}
tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh
+2
View File
@@ -14,6 +14,8 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/get_object_lock_configuration.sh
check_object_lock_config() {
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh
+2
View File
@@ -14,6 +14,8 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/get_bucket_ownership_controls.sh
put_bucket_acl_success_or_access_denied() {
if ! check_param_count_v2 "bucket, acl file, username, password, expect success" 5 $#; then
return 1
tests/drivers/put_bucket_website/put_bucket_website_rest.sh
+4 -16
View File
@@ -24,29 +24,17 @@ create_website_with_random_string() {
local bucket_name="$1"
local response file_name test_string
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! response=$(generate_random_string 8 10 2>&1); then
log 2 "error generating random string: $response"
return 1
fi
test_string="$response"
echo "$test_string" > "$TEST_FILE_FOLDER/$file_name"
if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then
log 2 "error putting random string file: $response"
if ! response=$(put_object_with_random_alnum_string "$bucket_name" 2>&1); then
log 2 "error putting object with random alnum string: $response"
return 1
fi
read -r file_name test_string <<< "$response"
if ! response=$(send_rest_go_command "200" "-commandType" "putBucketWebsiteConfiguration" "-bucketName" "$bucket_name" \
"-websiteConfiguration" "{\"IndexDocument\":{\"Suffix\":\"$file_name\"}}" 2>&1); then
log 2 "error putting website configuration: $response"
return 1
fi
printf '%s\n' "$test_string"
printf '%s\n' "$file_name $test_string"
return 0
}
tests/drivers/put_object/put_object_rest.sh
+29
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/put_object.sh
source ./tests/drivers/rest.sh
source ./tests/drivers/openssl.sh
@@ -525,3 +526,31 @@ put_objects() {
done
return 0
}
put_object_with_random_alnum_string() {
if ! check_param_count_v2 "bucket name" 1 $#; then
return 1
fi
local bucket_name="$1"
local response file_name test_string
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! response=$(generate_random_string 8 10 2>&1); then
log 2 "error generating random string: $response"
return 1
fi
test_string="$response"
echo "$test_string" > "$TEST_FILE_FOLDER/$file_name"
if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then
log 2 "error putting random string file: $response"
return 1
fi
echo "$file_name" "$test_string"
return 0
}
tests/drivers/rest.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/command.sh
source ./tests/drivers/list_objects/list_objects_rest.sh
source ./tests/drivers/xml.sh
tests/drivers/user.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/delete_bucket_policy.sh
source ./tests/drivers/delete_object/delete_object_rest.sh
source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh
source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh
tests/env.sh
+1 -1
View File
@@ -123,7 +123,7 @@ source_config_file() {
if [ -r tests/.env ]; then
source tests/.env
else
echo "Warning: no .env file found in tests folder" > /dev/stderr
echo "Warning: no .env file found in tests folder" >&2
fi
else
# shellcheck source=./tests/.env.default
tests/setup.sh
+9 -101
View File
@@ -16,6 +16,7 @@
source ./tests/env.sh
source ./tests/report.sh
source ./tests/setup_common.sh
source ./tests/setup_mc.sh
source ./tests/drivers/delete_bucket/delete_bucket_rest.sh
source ./tests/util/util_object.sh
@@ -64,118 +65,25 @@ static_user_versitygw_setup() {
# bats setup function
setup() {
TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME
export TEST_ID
local response
source_config_file
if ! setup_test_log_file; then
log 1 "error creating test log file"
if ! setup_env; then
log 1 "error setting up env"
return 1
fi
if [ -n "$TEST_LOG_FILE" ]; then
printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}"
fi
if ! base_setup; then
log 2 "error checking env vars or running versitygw"
if ! setup_versitygw; then
log 1 "error starting versitygw app: $response"
return 1
fi
if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then
if ! static_user_v1_cleanup; then
log 2 "error cleaning up v1 static users"
return 1
fi
if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
return 1
fi
fi
fi
log 4 "Running test $BATS_TEST_NAME"
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
START_TIME=$(date +%s)
export START_TIME
fi
if [[ $RUN_S3CMD == true ]]; then
S3CMD_OPTS=()
S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
S3CMD_OPTS+=(--region="$AWS_REGION")
export S3CMD_CONFIG S3CMD_OPTS
fi
if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then
log 1 "error checking for or adding mc alias"
if ! setup_clients; then
log 1 "error setting up clients"
return 1
fi
export AWS_PROFILE
log 4 "********** END SETUP **********"
return 0
}
# bats teardown function
teardown() {
local response proc_status_one proc_status_two
response=$(check_versity_process_status)
read -r proc_status_one proc_status_two <<< "$response"
if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then
bucket_and_user_cleanup
fi
if [ "$proc_status_one" == "running" ]; then
stop_versity_process "$VERSITYGW_PID_1"
fi
if [ "$proc_status_two" == "running" ]; then
stop_versity_process "$VERSITYGW_PID_2"
fi
remove_test_file_folder_if_desired
teardown_logs
}
static_user_v1_cleanup() {
if [ -n "$USERNAME_ONE" ]; then
if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
log 2 "error deleting user '$USERNAME_ONE'"
return 1
fi
fi
if [ -n "$USERNAME_TWO" ]; then
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 2 "error deleting user '$USERNAME_TWO'"
return 1
fi
fi
return 0
}
bucket_and_user_cleanup() {
log 4 "********** BEGIN TEARDOWN **********"
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
log 3 "error cleaning up buckets after test"
fi
if [ "$SKIP_USERS_TESTS" != "true" ]; then
if ! static_user_v1_cleanup; then
log 3 "error cleaning up v1 static users"
return 1
fi
fi
if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then
log 3 "error deleting autocreated users"
fi
if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then
if ! delete_tester_created_distributions "$USER_ID_USER_1"; then
log 3 "error deleting test-created S3 distributions"
fi
fi
return 0
teardown_common
}
tests/setup_common.sh
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env bats
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/env.sh
source ./tests/drivers/delete_bucket/delete_bucket_rest.sh
setup_env() {
TEST_ID=$(date +"%Y%m%d-%H%M%S")
if [ -n "$BATS_TEST_FILENAME" ]; then
TEST_ID+=".$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME"
fi
export TEST_ID
source_config_file
if ! setup_test_log_file; then
log 1 "error creating test log file"
return 1
fi
if [ -n "$TEST_LOG_FILE" ]; then
printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}"
fi
if [ -n "$BATS_TEST_NAME" ]; then
log 4 "Running test $BATS_TEST_NAME"
fi
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
START_TIME=$(date +%s)
export START_TIME
fi
if ! check_env_vars; then
log 1 "error checking env vars"
return 1
fi
export AWS_PROFILE
log 4 "********** END ENV SETUP **********"
return 0
}
setup_versitygw() {
local params=("$@")
local response pid
if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then
if ! run_versity_app "${params[@]}"; then
log 1 "error running versitygw app"
return 1
fi
fi
if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then
if ! static_user_v1_cleanup; then
log 2 "error cleaning up v1 static users"
return 1
fi
if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
return 1
fi
fi
fi
log 4 "********** END VERSITYGW SETUP **********"
return 0
}
setup_env_and_versitygw() {
local response
if ! setup_env; then
log 1 "error setting up env"
return 1
fi
if ! setup_versitygw "$@"; then
log 2 "error setting up versitygw"
return 1
fi
return 0
}
setup_clients() {
if [[ $RUN_S3CMD == true ]]; then
S3CMD_OPTS=()
S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
S3CMD_OPTS+=(--region="$AWS_REGION")
export S3CMD_CONFIG S3CMD_OPTS
fi
if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then
log 1 "error checking for or adding mc alias"
return 1
fi
log 4 "********** END CLIENT SETUP **********"
return 0
}
teardown_common() {
local response proc_status_one proc_status_two
response=$(check_versity_process_status)
read -r proc_status_one proc_status_two <<< "$response"
log 5 "proc status one: '$proc_status_one'"
log 5 "proc status two: '$proc_status_two'"
if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then
if ! bucket_and_user_cleanup; then
log 3 "bucket and user cleanup not properly done"
fi
fi
if [ "$proc_status_one" == "running" ]; then
if ! stop_versity_process "$VERSITYGW_PID_1"; then
log 3 "unable to properly stop versitygw process"
fi
fi
if [ "$proc_status_two" == "running" ]; then
if ! stop_versity_process "$VERSITYGW_PID_2"; then
log 3 "unable to properly stop second versity process"
fi
fi
if ! remove_test_file_folder_if_desired; then
log 3 "test file folder cleanup error"
fi
if ! teardown_logs; then
log 3 "log file teardown error"
fi
return 0
}
static_user_v1_cleanup() {
if [ -n "$USERNAME_ONE" ]; then
if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
log 2 "error deleting user '$USERNAME_ONE'"
return 1
fi
fi
if [ -n "$USERNAME_TWO" ]; then
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 2 "error deleting user '$USERNAME_TWO'"
return 1
fi
fi
return 0
}
bucket_and_user_cleanup() {
log 4 "********** BEGIN TEARDOWN **********"
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
log 3 "error cleaning up buckets after test"
fi
if [ "$SKIP_USERS_TESTS" != "true" ] && [ "$RUN_USERS" == "true" ]; then
if ! static_user_v1_cleanup; then
log 3 "error cleaning up v1 static users"
return 1
fi
if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then
log 3 "error deleting autocreated users"
fi
fi
if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then
if ! delete_tester_created_distributions "$USER_ID_USER_1"; then
log 3 "error deleting test-created S3 distributions"
fi
fi
return 0
}
tests/setup_static.sh
+9 -4
View File
@@ -14,7 +14,8 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/env.sh
source ./tests/logger.sh
source ./tests/setup_common.sh
source ./tests/util/util_object.sh
source ./tests/commands/create_bucket.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
@@ -40,7 +41,11 @@ create_bucket_if_not_exists() {
return 0
}
base_setup
if ! setup_env_and_versitygw; then
log 1 "error setting up env and/or versitygw"
exit 1
fi
if ! create_bucket_if_not_exists "$BUCKET_ONE_NAME"; then
log 2 "error creating static bucket one"
elif ! create_bucket_if_not_exists "$BUCKET_TWO_NAME"; then
@@ -49,6 +54,6 @@ fi
# shellcheck disable=SC2034
RECREATE_BUCKETS=false
if ! stop_versity_process "$VERSITYGW_PID_1"; then
log 2 "error stopping versity"
if ! teardown_common; then
log 2 "teardown error"
fi
tests/test_rest_list_buckets_s3_posix.sh
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bats
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
load ./bats-support/load
load ./bats-assert/load
source ./tests/commands/list_buckets.sh
source ./tests/drivers/create_bucket/create_bucket_rest.sh
source ./tests/drivers/list_buckets/list_buckets_rest.sh
source ./tests/logger.sh
source ./tests/setup.sh
setup() {
if ! setup_env; then
log 1 "error setting up env"
return 1
fi
if ! BACKEND=s3ToPosix setup_versitygw; then
log 1 "error setting up s3->posix versitygw config"
return 1
fi
return 0
}
teardown() {
BACKEND=s3ToPosix teardown_common
}
@test "ListBuckets - s3 -> posix" {
skip "https://github.com/versity/versitygw/issues/2309"
local bucket_name
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run list_check_buckets_rest "$bucket_name"
assert_success
}
tests/test_rest_website.sh
+79 -29
View File
@@ -19,14 +19,40 @@ load ./bats-assert/load
source ./tests/commands/put_public_access_block.sh
source ./tests/drivers/create_bucket/create_bucket_rest.sh
source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh
source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh
source ./tests/drivers/cloudfront.sh
source ./tests/drivers/string.sh
source ./tests/setup.sh
source ./tests/setup_common.sh
setup() {
if ! setup_env; then
log 1 "error setting up env"
return 1
fi
}
setup_versitygw_for_website() {
optional_params=("$@")
run setup_versitygw "${optional_params[@]}"
assert_success
read -r process_id process_id_two <<< "$output"
export VERSITYGW_PID_1="$process_id"
if [ -n "$process_id_two" ]; then
export VERSITYGW_PID_2="$process_id_two"
fi
}
teardown() {
teardown_common
}
@test "PutBucketWebsite - empty payload" {
local bucket_name
setup_versitygw_for_website
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
@@ -41,6 +67,8 @@ source ./tests/setup.sh
fi
local bucket_name
setup_versitygw_for_website
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
@@ -56,27 +84,17 @@ source ./tests/setup.sh
fi
distribution_created=false
local bucket_name policy_file distribution_domain http_domain
setup_versitygw_for_website
local bucket_name distribution_domain http_domain
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run create_website_with_random_string "$bucket_name"
assert_success
random_string="$output"
if [ "$DIRECT" == "true" ]; then
run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"
assert_success
fi
run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*"
assert_success
policy_file="$output"
run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"
run create_website_with_random_string_and_add_permissions "$bucket_name"
assert_success
read -r random_string <<< "$output"
if [ "$DIRECT" == "true" ]; then
run create_cloudfront_distribution "$bucket_name" "index.html" "${bucket_name}.s3-website.us-east-1.amazonaws.com"
@@ -95,7 +113,27 @@ source ./tests/setup.sh
}
@test "PutBucketWebsite - IndexDocument suffix" {
local bucket_name policy_file random_string
local bucket_name random_string
setup_versitygw_for_website
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run create_website_with_random_string_and_add_permissions "$bucket_name"
assert_success
read -r random_string <<< "$output"
run curl -ks "https://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}"
assert_success
assert_output "$random_string"
}
@test "REST - GetBucketWebsite - IndexDocument Suffix, DeleteBucketWebsite" {
local bucket_name test_file random_string
setup_versitygw_for_website
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
@@ -103,21 +141,33 @@ source ./tests/setup.sh
run create_website_with_random_string "$bucket_name"
assert_success
random_string="$output"
read -r test_file random_string <<< "$output"
if [ "$DIRECT" == "true" ]; then
run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"
assert_success
fi
run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*"
assert_success
policy_file="$output"
run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"
run check_index_document_suffix "$bucket_name" "$test_file"
assert_success
run curl -ks "https://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}"
run send_rest_go_command "204" "-method" "DELETE" "-query" "website" "-bucketName" "$bucket_name"
assert_success
run send_rest_go_command_expect_error_with_specific_arg_name_value "404" "NoSuchWebsiteConfiguration" \
"does not have a website configuration" "BucketName" "$bucket_name" "-query" "website" "-bucketName" "$bucket_name"
assert_success
}
@test "REST - GetBucketWebsite - no HTTPS" {
local bucket_name random_string
setup_versitygw_for_website "--website-no-tls"
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run create_website_with_random_string_and_add_permissions "$bucket_name"
assert_success
read -r random_string <<< "$output"
run curl -ks "http://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}"
assert_success
assert_output "$random_string"
}
tests/util/util_multipart_abort.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/list_multipart_uploads.sh
source ./tests/util/util_multipart_before_completion.sh
check_abort_access_denied() {
tests/util/util_policy.sh
+2
View File
@@ -14,6 +14,8 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/get_bucket_policy.sh
check_for_empty_policy() {
if ! check_param_count "check_for_empty_policy" "command type, bucket name" 2 $#; then
return 1
tests/util/util_retention.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/put_bucket_policy.sh
source ./tests/drivers/params.sh
# params: bucket name
tests/versity.sh
+94 -39
View File
@@ -29,6 +29,7 @@ start_versity_process() {
process_id="$response"
printf -v "VERSITYGW_PID_$1" '%s' "$process_id"
log 4 "versitygw PID for $1: $process_id"
export VERSITYGW_PID_"$1"
return 0
@@ -38,9 +39,10 @@ build_run_and_log_command() {
if ! check_param_count_gt "versitygw process number (1 or 2), command array" 2 $#; then
return 1
fi
local process_number="$1" command_array=("${@:2}")
local response full_command versitygw_log_file_name="" pid check_result
IFS=' ' read -r -a full_command <<< "${@:2}"
IFS=' ' read -r -a full_command <<< "${command_array[@]}"
log 5 "versity command: ${full_command[*]}"
if [ -n "$COMMAND_LOG" ]; then
if ! response=$(mask_args "${full_command[*]}" 2>&1); then
@@ -51,15 +53,15 @@ build_run_and_log_command() {
echo "$response" >> "$COMMAND_LOG"
fi
if [ -n "$VERSITY_LOG_FILE" ]; then
versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$1
echo "****************************** VERSITYGW $1 LOG ***********************************" >> "$versitygw_log_file_name"
versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$process_number
printf '****************************** VERSITYGW %s LOG \***********************************\n' "$1" >> "$versitygw_log_file_name"
"${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 &
else
"${full_command[@]}" >/dev/null 2>&1 &
fi
pid="$!"
if ! verify_process_started "$pid" "$versitygw_log_file_name"; then
if ! verify_process_started "$process_number" "$pid" "$versitygw_log_file_name"; then
return 1
fi
@@ -67,20 +69,37 @@ build_run_and_log_command() {
return 0
}
get_app_two_endpoint_url() {
local endpoint="http://localhost"
if [ -n "$PORT_TWO" ]; then
endpoint+="$PORT_TWO"
else
endpoint+=":7071"
fi
printf '%s\n' "$endpoint"
return 0
}
verify_process_started() {
if ! check_param_count_ge_le "pid, log file (if any)" 1 2 $#; then
if ! check_param_count_ge_le "app ID, pid, log file (if any)" 2 3 $#; then
return 1
fi
local check_result process_running="false" proc_state
local app_id="$1" pid="$2" log_file="$3"
local check_result process_running="false" proc_state params endpoint_url
for ((check_num=1; check_num<=3; check_num++)); do
sleep 1
if [ "$process_running" == "false" ] && check_result=$(kill -0 "$1" 2>&1); then
if [ "$process_running" == "false" ] && check_result=$(kill -0 "$pid" 2>&1); then
process_running="true"
fi
if [ "$process_running" == "true" ]; then
proc_state=$(ps -p "$1" -o state= 2>/dev/null | tr -d ' ')
if [ "$proc_state" != "Z" ] && list_buckets_rest "" "" >/dev/null; then
proc_state=$(ps -p "$pid" -o state= 2>/dev/null | tr -d ' ')
if [ "$app_id" == 2 ]; then
endpoint_url="$(get_app_two_endpoint_url)"
params="AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TWO AWS_ENDPOINT_URL=$endpoint_url"
fi
if [ "$proc_state" != "Z" ] && list_buckets_rest "$params" "" >/dev/null; then
return 0
fi
fi
@@ -93,27 +112,39 @@ verify_process_started() {
else
log 1 "process running in state '$proc_state', but not communicating properly"
fi
if [[ -n "$2" ]]; then
log 1 "log data: '$(cat "$2")'"
if [[ -n "$log_file" ]]; then
log 1 "log data: '$(cat "$log_file")'"
fi
return 1
}
run_versity_app_posix() {
if ! check_param_count "run_versity_app_posix" "access ID, secret key, versityid app index" 3 $#; then
if ! check_param_count_gt "access ID, secret key, versitygw app index, optional params" 3 $#; then
return 1
fi
base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
local access_id="$1" secret_key="$2" versitygw_app_index="$3" optional_params=("${@:4}")
local -a base_command
local response process_id
base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION")
if [ -n "$RUN_USERS" ]; then
# shellcheck disable=SC2153
IFS=' ' read -r -a iam_array <<< "$IAM_PARAMS"
fi
base_command+=("${iam_array[@]}")
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
if [ -n "$PORT" ]; then
base_command+=(--port ":$PORT")
if [ "$BACKEND" == "s3ToPosix" ]; then
if [ -n "$PORT_TWO" ]; then
base_command+=(--port ":$PORT_TWO")
else
base_command+=(--port ":7071")
fi
else
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
if [ -n "$PORT" ]; then
base_command+=(--port ":$PORT")
fi
fi
if [ -n "$WEBSITE" ]; then
base_command+=(--website "$WEBSITE")
@@ -121,13 +152,15 @@ run_versity_app_posix() {
if [ -n "$WEBSITE_DOMAIN" ]; then
base_command+=(--website-domain "$WEBSITE_DOMAIN")
fi
base_command+=("${optional_params[@]}")
base_command+=(posix)
if [ -n "$VERSIONING_DIR" ]; then
base_command+=(--versioning-dir "$VERSIONING_DIR")
fi
base_command+=("$LOCAL_FOLDER")
log 5 "base command: ${base_command[*]}"
if ! start_versity_process "$3" "${base_command[@]}"; then
if ! start_versity_process "$versitygw_app_index" "${base_command[@]}"; then
log 1 "error starting versity process"
return 1
fi
@@ -138,7 +171,11 @@ run_versity_app_scoutfs() {
if ! check_param_count "run_versity_app_scoutfs" "access ID, secret key, versityid app index" 3 $#; then
return 1
fi
base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER")
local access_id="$1" secret_key="$2" app_index="$3"
local -a base_command
local response process_id
base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER")
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
@@ -147,29 +184,36 @@ run_versity_app_scoutfs() {
fi
base_command+=(scoutfs "$LOCAL_FOLDER")
if ! start_versity_process "$3" "${base_command[@]}"; then
log 1 "error starting versity process"
if ! start_versity_process "$app_index" "${base_command[@]}"; then
log 1 "error starting versity process: $response"
return 1
fi
return 0
}
run_versity_app_s3() {
if ! check_param_count "run_versity_app_s3" "versityid app index" 1 $#; then
if ! check_param_count "run_versity_app_s3" "server key ID, server access key, dest key ID, dest access key, app index" 5 $#; then
return 1
fi
base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY" --region="$AWS_REGION")
local server_key_id="$1" server_access_key="$2" dest_key_id="$3" dest_access_key="$4" app_index="$5"
local -a base_command
local endpoint_url
base_command=("$VERSITY_EXE" --access="$server_key_id" --secret="$server_access_key" --region="$AWS_REGION")
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
if [ -n "$PORT_TWO" ]; then
base_command+=(--port ":$PORT_TWO")
else
base_command+=(--port ":7071")
if [ -n "$PORT" ]; then
base_command+=(--port ":$PORT")
fi
base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
if [ "$BACKEND" == "s3ToPosix" ]; then
endpoint_url=$(get_app_two_endpoint_url)
else
endpoint_url="$S3_AWS_ENDPOINT_URL"
fi
base_command+=(s3 --access="$dest_key_id" --secret="$dest_access_key" --region="$AWS_REGION" --endpoint="$endpoint_url")
if ! start_versity_process "$1" "${base_command[@]}"; then
if ! start_versity_process "$app_index" "${base_command[@]}"; then
log 2 "error starting versity process"
return 1
fi
@@ -177,29 +221,40 @@ run_versity_app_s3() {
}
run_versity_app() {
local additional_params=("$@")
local response process_id
if [[ $BACKEND == 'posix' ]]; then
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" "${additional_params[@]}"; then
log 2 "error running POSIX versity app: $response"
return 1
fi
elif [[ $BACKEND == 'scoutfs' ]]; then
if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
log 2 "error running scoutfs versity app: $response"
return 1
fi
elif [[ $BACKEND == 's3' ]]; then
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
return 1
elif [[ $BACKEND == 's3' ]] || [ "$BACKEND" == "s3ToPosix" ]; then
# start in reverse order to check with ListBuckets for both
if [ "$BACKEND" == 's3ToPosix' ]; then
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO" "2"; then
log 2 "error running versitygw with posix backend in s3 config"
return 1
fi
fi
if ! run_versity_app_s3 "2"; then
if ! run_versity_app_s3 "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO" "1"; then
log 2 "error running versitygw with s3 backend in s3 config"
return 1
fi
else
log 1 "unrecognized backend type $BACKEND"
return 1
fi
printf '%s %s\n' "$VERSITYGW_PID_1" "$VERSITYGW_PID_2"
if [[ $IAM_TYPE != "s3" ]]; then
return 0
fi
if bucket_exists "$USERS_BUCKET"; then
if bucket_exists "$USERS_BUCKET" >/dev/null; then
return 0
fi
if ! create_bucket "s3api" "$USERS_BUCKET"; then
@@ -230,13 +285,13 @@ check_versity_process_status() {
status_one="none"
status_two="none"
if [ "$RUN_VERSITYGW" == "true" ]; then
if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "$VERSITYGW_PID_1" >/dev/null; then
if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "1" "$VERSITYGW_PID_1" >/dev/null; then
status_one="running"
else
status_one="failed"
fi
if [ "$BACKEND" == "s3" ]; then
if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "$VERSITYGW_PID_2" >/dev/null; then
if [ "$BACKEND" == "s3ToPosix" ]; then
if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "2" "$VERSITYGW_PID_2" >/dev/null; then
status_two="running"
else
status_two="failed"