mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-10 13:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: static bucket config debugging, add to github-actions

Browse Source
This commit is contained in:
Luke McCrone
2024-06-11 19:30:45 -03:00
parent d98ca9b034
commit fdbb2d8f01
9 changed files with 148 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/system.yml vendored
View File

@@ -16,6 +16,7 @@ jobs:
USERS_FOLDER: /tmp/iam1
AWS_ENDPOINT_URL: https://127.0.0.1:7070
RUN_SET: "s3cmd"
RECREATE_BUCKETS: "true"
PORT: 7070
- set: 2
LOCAL_FOLDER: /tmp/gw2
@@ -25,6 +26,7 @@ jobs:
USERS_FOLDER: /tmp/iam2
AWS_ENDPOINT_URL: https://127.0.0.1:7071
RUN_SET: "s3"
RECREATE_BUCKETS: "true"
PORT: 7071
- set: 3
LOCAL_FOLDER: /tmp/gw3
@@ -34,6 +36,7 @@ jobs:
USERS_FOLDER: /tmp/iam3
AWS_ENDPOINT_URL: https://127.0.0.1:7072
RUN_SET: "s3api"
RECREATE_BUCKETS: "true"
PORT: 7072
- set: 4
LOCAL_FOLDER: /tmp/gw4
@@ -43,17 +46,28 @@ jobs:
USERS_FOLDER: /tmp/iam4
AWS_ENDPOINT_URL: https://127.0.0.1:7073
RUN_SET: "mc"
RECREATE_BUCKETS: "true"
PORT: 7073
- set: 5
LOCAL_FOLDER: /tmp/gw4
BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
LOCAL_FOLDER: /tmp/gw5
BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
IAM_TYPE: s3
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7074
RUN_SET: "aws-user"
RECREATE_BUCKETS: "true"
PORT: 7074
- set: 6
LOCAL_FOLDER: /tmp/gw6
BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam6
AWS_ENDPOINT_URL: https://127.0.0.1:7075
RUN_SET: "aws"
RECREATE_BUCKETS: "false"
PORT: 7075
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
@@ -97,7 +111,7 @@ jobs:
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: posix
RECREATE_BUCKETS: true
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
S3CMD_CONFIG: tests/s3cfg.local.default
@@ -117,6 +131,9 @@ jobs:
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir $GOCOVERDIR $USERS_FOLDER
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
#- name: Build and run, s3 backend

16
tests/env.sh
View File

@@ -29,15 +29,17 @@ check_env_vars() {
}
check_universal_vars() {
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
if [[ $BYPASS_ENV_FILE != "true" ]]; then
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
log 3 "Warning: no .env file found in tests folder"
fi
else
log 3 "Warning: no .env file found in tests folder"
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
elif [[ $BYPASS_ENV_FILE != "true" ]]; then
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets

40
tests/remove_static.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/bin/bash
source ./tests/setup.sh
source ./tests/util.sh
source ./tests/commands/delete_bucket.sh
delete_bucket_if_exists() {
if [[ $# -ne 2 ]]; then
log 2 "delete_bucket_if_exists command missing command type, name"
return 1
fi
bucket_exists "$1" "$2" || local exists_result=$?
if [[ $exists_result -eq 2 ]]; then
log 2 "error checking if bucket exists"
return 1
fi
if [[ $exists_result -eq 1 ]]; then
log 5 "bucket '$2' doesn't exist, skipping"
return 0
fi
if ! delete_bucket "$1" "$2"; then
log 2 "error deleting bucket"
return 1
fi
log 5 "bucket '$2' successfully deleted"
return 0
}
if ! setup; then
log 2 "error starting versity to set up static buckets"
exit 1
fi
if ! delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error deleting static bucket one"
elif ! delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then
log 2 "error deleting static bucket two"
fi
if ! teardown; then
log 2 "error stopping versity"
fi

18
tests/run.sh
View File

@@ -19,9 +19,6 @@ handle_param() {
show_help
exit 0
;;
-s|--static)
export RECREATE_BUCKETS=false
;;
s3|s3api|aws|s3cmd|mc|aws-user)
set_command_type "$1"
;;
@@ -41,14 +38,6 @@ set_command_type() {
export command_type
}
if [[ -z $RECREATE_BUCKETS ]]; then
export RECREATE_BUCKETS=true
elif [[ $RECREATE_BUCKETS != true ]] && [[ $RECREATE_BUCKETS != false ]]; then
echo "Invalid RECREATE_BUCKETS value: $RECREATE_BUCKETS"
exit 1
else
export RECREATE_BUCKETS=$RECREATE_BUCKETS
fi
while [[ "$#" -gt 0 ]]; do
handle_param "$1"
shift # past argument or value
@@ -59,13 +48,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
exit 1
fi
if [[ $RECREATE_BUCKETS == false ]]; then
./tests/setup_static.sh || exit_code=$?
if [[ exit_code -ne 0 ]]; then
exit 1
fi
fi
case $command_type in
s3api|aws)
echo "Running aws tests ..."

4
tests/run_all.sh
View File

@@ -5,10 +5,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
exit 1
fi
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
export RECREATE_BUCKETS
if ! ./tests/run.sh aws; then
exit 1
fi

41
tests/setup_static.sh
View File

@@ -1,7 +1,40 @@
#!/bin/bash
source ./tests/setup.sh
setup
aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
teardown
source ./tests/util.sh
source ./tests/commands/create_bucket.sh
create_bucket_if_not_exists() {
if [[ $# -ne 2 ]]; then
log 2 "create_bucket_if_not_exists command missing command type, name"
return 1
fi
bucket_exists "$1" "$2" || local exists_result=$?
if [[ $exists_result -eq 2 ]]; then
log 2 "error checking if bucket exists"
return 1
fi
if [[ $exists_result -eq 0 ]]; then
log 5 "bucket '$2' already exists, skipping"
return 0
fi
if ! create_bucket_object_lock_enabled "$2"; then
log 2 "error creating bucket"
return 1
fi
log 5 "bucket '$2' successfully created"
return 0
}
if ! setup; then
log 2 "error starting versity to set up static buckets"
exit 1
fi
if ! create_bucket_if_not_exists "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error creating static bucket one"
elif ! create_bucket_if_not_exists "s3api" "$BUCKET_TWO_NAME"; then
log 2 "error creating static bucket two"
fi
if ! teardown; then
log 2 "error stopping versity"
fi

51
tests/test_aws.sh
View File

@@ -35,17 +35,16 @@ export RUN_USERS=true
local bucket_file="bucket-file"
bucket_file_data="test file\n"
create_test_files "$bucket_file" || local created=$?
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
create_test_files "$bucket_file" || fail "error creating test files"
printf "%s" "$bucket_file_data" > "$test_file_folder/$bucket_file"
run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
[[ $abort_result -eq 0 ]] || fail "Abort failed"
setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file" || exists=$?
[[ $exists -eq 1 ]] || fail "Upload file exists after abort"
run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "abort failed"
if object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file"; then
fail "Upload file exists after abort"
fi
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
@@ -56,18 +55,15 @@ export RUN_USERS=true
local bucket_file="bucket-file"
bucket_file_data="test file\n"
create_test_files "$bucket_file" || local created=$?
create_test_files "$bucket_file" || fail "error creating test files"
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
[[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'"
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$?
[[ $compare_result -eq 0 ]] || fail "Files do not match"
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload"
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error copying file"
compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || fail "files do not match"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
@@ -105,25 +101,30 @@ export RUN_USERS=true
os_name="$(uname)"
if [[ "$os_name" == "Darwin" ]]; then
now=$(date -u +"%Y-%m-%dT%H:%M:%S")
five_seconds_later=$(date -j -v +10S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
else
now=$(date +"%Y-%m-%dT%H:%M:%S")
five_seconds_later=$(date -d "$now 10 seconds" +"%Y-%m-%dT%H:%M:%S")
later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S")
fi
create_test_files "$bucket_file" || fail "error creating test file"
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
# in static bucket config, bucket will still exist
bucket_exists "s3api" "$BUCKET_ONE_NAME" || local exists_result=$?
[[ $exists_result -ne 2 ]] || fail "error checking for bucket existence"
if [[ $exists_result -eq 1 ]]; then
create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
fi
log 5 "$five_seconds_later"
log 5 "LATER: $later"
multipart_upload_with_params "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 \
"$expected_content_type" \
"{\"$expected_meta_key\": \"$expected_meta_val\"}" \
"$expected_hold_status" \
"$expected_retention_mode" \
"$five_seconds_later" \
"$later" \
"$expected_tag_key=$expected_tag_val" || fail "error performing multipart upload"
head_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting metadata"
@@ -139,7 +140,7 @@ export RUN_USERS=true
retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode")
[[ $retention_mode == "$expected_retention_mode" ]] || fail "retention mode mismatch ($retention_mode, $expected_retention_mode)"
retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate")
[[ $retain_until_date == "$five_seconds_later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"
[[ $retain_until_date == "$later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"
get_object_tagging "aws" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting tagging"
log 5 "tags: $tags"
@@ -154,7 +155,7 @@ export RUN_USERS=true
get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error getting object"
compare_files "$test_file_folder/$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "files not equal"
sleep 10
sleep 15
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file

17
tests/util.sh
View File

@@ -153,6 +153,7 @@ delete_bucket_or_contents() {
echo "error deleting bucket contents"
return 1
fi
log 5 "bucket contents deletion success"
return 0
fi
delete_bucket_recursive "$1" "$2" || local delete_result=$?
@@ -160,6 +161,7 @@ delete_bucket_or_contents() {
echo "Bucket deletion error"
return 1
fi
log 5 "bucket deletion success"
return 0
}
@@ -180,10 +182,8 @@ delete_bucket_if_exists() {
log 2 "error deleting bucket or contents"
return 1
fi
#if [[ $RECREATE_BUCKETS == "false" ]]; then
log 5 "bucket and/or bucket data deletion success"
return 0
#fi
fi
if [[ $RECREATE_BUCKETS == "false" ]]; then
log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
@@ -208,12 +208,15 @@ setup_bucket() {
fi
local create_result
log 5 "util.setup_bucket: command type: $1, bucket name: $2"
create_bucket "$1" "$2" || create_result=$?
if [[ $create_result -ne 0 ]]; then
log 2 "Error creating bucket"
return 1
if [[ $RECREATE_BUCKETS == "true" ]]; then
if ! create_bucket "$1" "$2"; then
log 2 "Error creating bucket"
return 1
fi
log 5 "bucket creation success"
else
log 5 "skipping bucket re-creation"
fi
log 5 "Bucket creation success"
return 0
}

6
tests/versity.sh
View File

@@ -28,8 +28,10 @@ start_versity_process() {
return 1
fi
eval versitygw_pid_"$1"=$!
process_info="Versity process $1, PID $!"
echo "$process_info" >> "$VERSITY_LOG_FILE"
if [ -n "$VERSITY_LOG_FILE" ]; then
process_info="Versity process $1, PID $!"
echo "$process_info" >> "$VERSITY_LOG_FILE"
fi
log 4 "$process_info"
local pid
eval pid=\$versitygw_pid_"$1"