mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-07 04:06:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

another shellcheck fix

Browse Source
This commit is contained in:
Luke McCrone
2024-02-02 17:10:02 -03:00
parent e6440da30a
commit 5d6c0f8b67
8 changed files with 563 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/system.yml vendored
View File

@@ -1,19 +1,19 @@
name: system tests
on: pull_request
#on:
# workflow_dispatch:
# inputs:
# run_workflow:
# description: 'Run command-line tests'
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Install ShellCheck
run: sudo apt-get install shellcheck
- name: Run ShellCheck
run: shellcheck -S warning ./tests/*.sh
- name: Set up Go
uses: actions/setup-go@v4
with:
@@ -38,5 +38,5 @@ jobs:
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
export VERSITY_EXE=./versitygw
mkdir /tmp/gw
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/s3_bucket_tests.sh
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/posix_tests.sh
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/s3_bucket_tests.sh
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/posix_tests.sh

7
tests/.env.default
View File

@@ -1,6 +1,9 @@
AWS_REGION=us-west-2
AWS_REGION=us-east-1
AWS_PROFILE=versity
VERSITY_EXE=./versitygw
BACKEND=posix
LOCAL_FOLDER=/tmp/gw
AWS_ENDPOINT_URL=http://127.0.0.1:7070
AWS_ENDPOINT_URL=http://127.0.0.1:7070
BUCKET_ONE_NAME=versity-gwtest-bucket-one
BUCKET_TWO_NAME=versity-gwtest-bucket-two
RECREATE_BUCKETS=true

5
tests/.env.versitygw
View File

@@ -3,4 +3,7 @@ AWS_PROFILE=versity
VERSITY_EXE=./versitygw
BACKEND=posix
LOCAL_FOLDER=/tmp/gw
AWS_ENDPOINT_URL=http://127.0.0.1:7070
AWS_ENDPOINT_URL=http://127.0.0.1:7070
BUCKET_ONE_NAME=versity-gwtest-bucket-one
BUCKET_TWO_NAME=versity-gwtest-bucket-two
RECREATE_BUCKETS=true

83
tests/posix_tests.sh
View File

@@ -7,88 +7,73 @@ source ./tests/util_posix.sh
# test that changes to local folders and files are reflected on S3
@test "test_local_creation_deletion" {
local bucket_name="versity-gwtest-put-object-test"
if [[ $RECREATE_BUCKETS != "true" ]]; then
return
fi
local object_name="test-object"
bucket_exists_remote_and_local $bucket_name || local bucket_exists=$?
if [[ $bucket_exists -eq 2 ]]; then
fail "Bucket existence check error"
fi
local object="$bucket_name"/"$object_name"
if [[ $bucket_exists -eq 0 ]]; then
object_exists_remote_and_local "$object" || local object_exists=$?
if [[ $object_exists -eq 2 ]]; then
fail "Object existence check error"
fi
if [[ $object_exists -eq 0 ]]; then
delete_object "$object" || local delete_object=$?
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
fi
delete_bucket $bucket_name || local delete_bucket=$?
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
fi
mkdir "$LOCAL_FOLDER"/$bucket_name
touch "$LOCAL_FOLDER"/$object
bucket_exists_remote_and_local $bucket_name || local bucket_exists_two=$?
mkdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
local object="$BUCKET_ONE_NAME"/"$object_name"
touch "$LOCAL_FOLDER"/"$object"
bucket_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_exists_two=$?
[[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
object_exists_remote_and_local $object || local object_exists_two=$?
object_exists_remote_and_local "$object" || local object_exists_two=$?
[[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
rm "$LOCAL_FOLDER"/$object
rm "$LOCAL_FOLDER"/"$object"
sleep 1
object_not_exists_remote_and_local $object || local object_deleted=$?
object_not_exists_remote_and_local "$object" || local object_deleted=$?
[[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
rmdir "$LOCAL_FOLDER"/$bucket_name
rmdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_not_exists_remote_and_local $bucket_name || local bucket_deleted=$?
bucket_not_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_deleted=$?
[[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check"
}
# test head-object command
@test "test_head_object" {
local bucket_name="versity-gwtest-head-object"
local bucket_name=$BUCKET_ONE_NAME
local object_name="object-one"
touch "$object_name"
if [ -e "$LOCAL_FOLDER"/$bucket_name/$object_name ]; then
chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name
create_test_files $object_name
if [ -e "$LOCAL_FOLDER"/"$bucket_name"/$object_name ]; then
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
fi
check_and_create_bucket $bucket_name || local created=$?
setup_bucket "$bucket_name" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating bucket"
put_object "$object_name" "$bucket_name"/"$object_name" || local result="$?"
put_object "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?"
[[ result -eq 0 ]] || fail "Error adding object one"
chmod 000 "$LOCAL_FOLDER"/$bucket_name/$object_name
chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
sleep 1
object_is_accessible $bucket_name $object_name || local accessible=$?
object_is_accessible "$bucket_name" $object_name || local accessible=$?
[[ $accessible -eq 1 ]] || fail "Object should be inaccessible"
chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
sleep 1
object_is_accessible $bucket_name $object_name || local accessible_two=$?
object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
[[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
delete_object $bucket_name/$object_name
delete_bucket $bucket_name
delete_object "$bucket_name"/$object_name
delete_bucket_or_contents "$bucket_name"
delete_test_files $object_name
}
# check info, accessiblity of bucket
@test "test_get_bucket_info" {
local bucket_name="versity-gwtest-get-bucket-info"
if [ -e "$LOCAL_FOLDER"/$bucket_name ]; then
chmod 755 "$LOCAL_FOLDER"/$bucket_name
if [ -e "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME" ]; then
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
else
create_bucket $bucket_name || local created=$?
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating bucket"
fi
chmod 000 "$LOCAL_FOLDER"/$bucket_name
chmod 000 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_is_accessible $bucket_name || local accessible=$?
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible=$?
[[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible"
chmod 755 "$LOCAL_FOLDER"/$bucket_name
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_is_accessible $bucket_name || local accessible_two=$?
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible_two=$?
[[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible"
delete_bucket $bucket_name
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}

277
tests/s3_bucket_tests.sh
View File

@@ -4,122 +4,81 @@ source ./tests/setup.sh
source ./tests/util.sh
# test creation and deletion of bucket on versitygw
@test "create_delete_bucket_test" {
@test "test_create_delete_bucket" {
local bucket_name="versity-gwtest-create-delete-bucket-test"
if [[ $RECREATE_BUCKETS != "true" ]]; then
return
fi
bucket_exists $bucket_name || local exists=$?
if [[ $exists -eq 2 ]]; then
fail "Bucket existence check error"
fi
if [[ $exists -eq 0 ]]; then
delete_bucket $bucket_name || local delete_result=$?
[[ $delete_result -eq 0 ]] || fail "Failed to delete bucket"
bucket_exists $bucket_name || local exists_two=$?
[[ $exists_two -eq 1 ]] || fail "Failed bucket deletion"
fi
create_bucket $bucket_name || local create_result=$?
setup_bucket "$BUCKET_ONE_NAME" || local create_result=$?
[[ $create_result -eq 0 ]] || fail "Failed to create bucket"
bucket_exists $bucket_name || local exists_three=$?
bucket_exists "$BUCKET_ONE_NAME" || local exists_three=$?
[[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
delete_bucket $bucket_name || local delete_result_two=$?
delete_bucket_or_contents "$BUCKET_ONE_NAME" || local delete_result_two=$?
[[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
}
# test adding and removing an object on versitygw
@test "put_object_test" {
@test "test_put_object" {
local bucket_name="versity-gwtest-put-object-test"
local object_name="test-object"
bucket_exists $bucket_name || local bucket_exists=$?
if [[ $bucket_exists -eq 2 ]]; then
fail "Bucket existence check error"
fi
local object="$bucket_name"/"$object_name"
if [[ $bucket_exists -eq 0 ]]; then
object_exists "$object" || local object_exists=$?
if [[ $object_exists -eq 2 ]]; then
fail "Object existence check error"
fi
if [[ $object_exists -eq 0 ]]; then
delete_object "$object" || local delete_object=$?
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
fi
delete_bucket $bucket_name || local delete_bucket=$?
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
fi
touch "$object_name"
create_bucket $bucket_name || local create_bucket=$?
[[ $create_bucket -eq 0 ]] || fail "Failed to create bucket"
put_object "$object_name" "$object" || local put_object=$?
setup_bucket "$BUCKET_ONE_NAME" || local setup_result=$?
[[ $setup_result -eq 0 ]] || fail "error setting up bucket"
create_test_files "$object_name" || local create_result=$?
object="$BUCKET_ONE_NAME"/$object_name
put_object "$test_file_folder"/"$object_name" "$object" || local put_object=$?
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
object_exists "$object" || local object_exists_two=$?
[[ $object_exists_two -eq 0 ]] || fail "Object not added to bucket"
delete_object "$object" || local delete_object_two=$?
[[ $delete_object_two -eq 0 ]] || fail "Failed to delete object"
delete_bucket $bucket_name || local delete_bucket=$?
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
rm "$object_name"
object_exists "$object" || local exists_result_one=$?
[[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
delete_object "$object" || local delete_result=$?
[[ $delete_result -eq 0 ]] || fail "Failed to delete object"
object_exists "$object" || local exists_result_two=$?
[[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$object_name"
}
# test listing buckets on versitygw
@test "test_list_buckets" {
bucket_name_one="versity-gwtest-list-one"
bucket_name_two="versity-gwtest-list-two"
bucket_exists $bucket_name_one || local exists=$?
if [[ $exists -eq 2 ]]; then
fail "Bucket existence check error"
fi
if [[ $exists -eq 1 ]]; then
create_bucket $bucket_name_one || local bucket_create_one=$?
[[ $bucket_create_one -eq 0 ]] || fail "Failed to create bucket"
fi
bucket_exists $bucket_name_two || local exists_two=$?
if [[ $exists_two -eq 2 ]]; then
fail "Bucket existence check error"
fi
if [[ $exists_two -eq 1 ]]; then
create_bucket $bucket_name_two || local bucket_create_two=$?
[[ $bucket_create_two -eq 0 ]] || fail "Failed to create bucket"
fi
setup_bucket "$BUCKET_ONE_NAME" || local setup_result_one=$?
[[ $setup_result_one -eq 0 ]] || fail "Bucket one setup error"
setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$?
[[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
list_buckets
local bucket_one_found=false
local bucket_two_found=false
for bucket in "${bucket_array[@]}"; do
if [ "$bucket" == $bucket_name_one ]; then
if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
bucket_one_found=true
elif [ "$bucket" == $bucket_name_two ]; then
elif [ "$bucket" == "$BUCKET_TWO_NAME" ]; then
bucket_two_found=true
fi
if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
return
fi
done
fail "$bucket_name_one and/or $bucket_name_two not listed (all buckets: ${bucket_array[*]})"
delete_bucket $bucket_name_one || local deleted_one=$?
[[ $deleted_one -eq 0 ]] || fail "Failed to delete bucket one"
delete_bucket $bucket_name_two || local deleted_two=$?
[[ $deleted_two -eq 0 ]] || fail "Failed to delete bucket one"
fail "'$BUCKET_ONE_NAME' and/or '$BUCKET_TWO_NAME' not listed (all buckets: ${bucket_array[*]})"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_bucket_or_contents "$BUCKET_TWO_NAME"
}
# test listing a bucket's objects on versitygw
@test "test_list_objects" {
bucket_name="versity-gwtest-list-object"
object_one="test-file-one"
object_two="test-file-two"
touch $object_one $object_two
check_and_create_bucket $bucket_name || local result_one=$?
create_test_files $object_one $object_two
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
[[ result_one -eq 0 ]] || fail "Error creating bucket"
put_object $object_one "$bucket_name"/"$object_one" || local result_two=$?
put_object "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
[[ result_two -eq 0 ]] || fail "Error adding object one"
put_object $object_two "$bucket_name"/"$object_two" || local result_three=$?
put_object "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
[[ result_three -eq 0 ]] || fail "Error adding object two"
list_objects $bucket_name
list_objects "$BUCKET_ONE_NAME"
local object_one_found=false
local object_two_found=false
for object in "${object_array[@]}"; do
@@ -132,41 +91,38 @@ source ./tests/util.sh
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
fi
delete_object "$bucket_name"/"$object_one"
delete_object "$bucket_name"/"$object_two"
delete_bucket $bucket_name
rm $object_one $object_two
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $object_one $object_two
}
# test ability to retrieve bucket ACLs
@test "test_get_bucket_acl" {
local bucket_name="versity-gwtest-get-bucket-acl"
check_and_create_bucket $bucket_name || local created=$?
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating bucket"
get_bucket_acl $bucket_name || local result=$?
get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Error retrieving acl"
id=$(echo "$acl" | jq '.Owner.ID')
[[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
delete_bucket $bucket_name
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}
# test ability to delete multiple objects from bucket
@test "test_delete_objects" {
local bucket_name="versity-gwtest-delete-objects"
local object_one="test-file-one"
local object_two="test-file-two"
touch "$object_one" "$object_two"
check_and_create_bucket $bucket_name || local result_one=$?
create_test_files "$object_one" "$object_two" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
[[ $result_one -eq 0 ]] || fail "Error creating bucket"
put_object "$object_one" "$bucket_name"/"$object_one" || local result_two=$?
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
[[ $result_two -eq 0 ]] || fail "Error adding object one"
put_object "$object_two" "$bucket_name"/"$object_two" || local result_three=$?
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
[[ $result_three -eq 0 ]] || fail "Error adding object two"
error=$(aws s3api delete-objects --bucket $bucket_name --delete '{
error=$(aws s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{
"Objects": [
{"Key": "test-file-one"},
{"Key": "test-file-two"}
@@ -174,11 +130,142 @@ source ./tests/util.sh
}') || local result=$?
[[ $result -eq 0 ]] || fail "Error deleting objects: $error"
object_exists "$bucket_name"/"$object_one" || local exists_one=$?
object_exists "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
[[ $exists_one -eq 1 ]] || fail "Object one not deleted"
object_exists "$bucket_name"/"$object_two" || local exists_two=$?
object_exists "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
[[ $exists_two -eq 1 ]] || fail "Object two not deleted"
delete_bucket $bucket_name
rm "$object_one" "$object_two"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$object_one" "$object_two"
}
# test abilty to set and retrieve bucket tags
@test "test-set-get-bucket-tags" {
local key="test_key"
local value="test_value"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
[[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
tag_set=$(echo "$tags" | jq '.TagSet')
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
put_bucket_tag "$BUCKET_ONE_NAME" $key $value
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
[[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}
# test v1 s3api list objects command
@test "test-s3api-list-objects-v1" {
local object_one="test-file-one"
local object_two="test-file-two"
local object_two_data="test data\n"
create_test_files "$object_one" "$object_two" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
list_objects_s3api_v1 "$BUCKET_ONE_NAME"
key_one=$(echo "$objects" | jq '.Contents[0].Key')
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
size_one=$(echo "$objects" | jq '.Contents[0].Size')
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
key_two=$(echo "$objects" | jq '.Contents[1].Key')
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
size_two=$(echo "$objects" | jq '.Contents[1].Size')
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$object_one" "$object_two"
}
# test v2 s3api list objects command
@test "test-s3api-list-objects-v2" {
local object_one="test-file-one"
local object_two="test-file-two"
local object_two_data="test data\n"
create_test_files "$object_one" "$object_two" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
list_objects_s3api_v2 "$BUCKET_ONE_NAME"
key_one=$(echo "$objects" | jq '.Contents[0].Key')
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
size_one=$(echo "$objects" | jq '.Contents[0].Size')
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
key_two=$(echo "$objects" | jq '.Contents[1].Key')
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
size_two=$(echo "$objects" | jq '.Contents[1].Size')
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$object_one" "$object_two"
}
# test abilty to set and retrieve object tags
@test "test-set-get-object-tags" {
local bucket_file="bucket-file"
local key="test_key"
local value="test_value"
create_test_files "$bucket_file" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
put_object "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
[[ $get_result -eq 0 ]] || fail "Error getting object tags"
tag_set=$(echo "$tags" | jq '.TagSet')
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
[[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
# test multi-part upload
@test "test-multi-part-upload" {
local bucket_file="bucket-file"
bucket_file_data="test file\n"
create_test_files "$bucket_file" || local created=$?
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
copy_data=$(<"$test_file_folder"/$bucket_file-copy)
[[ $bucket_file_data == "$copy_data" ]] || fail "Data doesn't match"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}

17
tests/setup.sh
View File

@@ -15,7 +15,8 @@ setup() {
fi
else
echo "$VERSITYGW_TEST_ENV"
source $VERSITYGW_TEST_ENV
# shellcheck source=./.env.default
source "$VERSITYGW_TEST_ENV"
fi
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
@@ -42,6 +43,18 @@ setup() {
elif [ -z "$AWS_ENDPOINT_URL" ]; then
echo "No AWS endpoint URL set"
return 1
elif [ -z "$BUCKET_ONE_NAME" ]; then
echo "No bucket one name set"
return 1
elif [ -z "$BUCKET_TWO_NAME" ]; then
echo "No bucket two name set"
return 1
elif [ -z "$RECREATE_BUCKETS" ]; then
echo "No recreate buckets parameter set"
return 1
elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
echo "RECREATE_BUCKETS must be 'true' or 'false'"
return 1
fi
ROOT_ACCESS_KEY="$AWS_ACCESS_KEY_ID" ROOT_SECRET_KEY="$AWS_SECRET_ACCESS_KEY" "$VERSITY_EXE" "$BACKEND" "$LOCAL_FOLDER" &
@@ -50,6 +63,8 @@ setup() {
export AWS_PROFILE
export AWS_ENDPOINT_URL
export LOCAL_FOLDER
export BUCKET_ONE_NAME
export BUCKET_TWO_NAME
versitygw_pid=$!
export versitygw_pid

325
tests/util.sh
View File

@@ -1,3 +1,5 @@
#!/usr/bin/env bats
# create an AWS bucket
# param: bucket name
# return 0 for success, 1 for failure
@@ -38,6 +40,46 @@ delete_bucket() {
return 0
}
# recursively delete an AWS bucket
# param: bucket name
# return 0 for success, 1 for failure
delete_bucket_recursive() {
if [ $# -ne 1 ]; then
echo "delete bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 rb s3://"$1" --force 2>&1) || exit_code="$?"
if [ $exit_code -ne 0 ]; then
if [[ "$error" == *"The specified bucket does not exist"* ]]; then
return 0
else
echo "error deleting bucket: $error"
return 1
fi
fi
return 0
}
# delete contents of a bucket
# param: bucket name
# return 0 for success, 1 for failure
delete_bucket_contents() {
if [ $# -ne 1 ]; then
echo "delete bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 rm s3://"$1" --recursive 2>&1) || exit_code="$?"
if [ $exit_code -ne 0 ]; then
echo "error deleting bucket: $error"
return 1
fi
return 0
}
# check if bucket exists
# param: bucket name
# return 0 for true, 1 for false, 2 for error
@@ -61,10 +103,35 @@ bucket_exists() {
return 0
}
# create bucket if it doesn't exist
# delete buckets or just the contents depending on RECREATE_BUCKETS parameter
# param: bucket name
# return: 0 for success, 1 for failure
delete_bucket_or_contents() {
if [ $# -ne 1 ]; then
echo "delete bucket or contents function requires bucket name"
return 1
fi
if [[ $RECREATE_BUCKETS != "true" ]]; then
delete_bucket_contents "$1" || local delete_result=$?
if [[ $delete_result -ne 0 ]]; then
echo "error deleting bucket contents"
return 1
fi
return 0
fi
delete_bucket_recursive "$1" || local delete_result=$?
if [[ $delete_result -ne 0 ]]; then
echo "Bucket deletion error"
return 1
fi
return 0
}
# if RECREATE_BUCKETS is set to true create bucket, deleting it if it exists to clear state. If not,
# check to see if it exists and return an error if it does not.
# param: bucket name
# return 0 for success, 1 for failure
check_and_create_bucket() {
setup_bucket() {
if [ $# -ne 1 ]; then
echo "bucket creation function requires bucket name"
return 1
@@ -75,14 +142,24 @@ check_and_create_bucket() {
echo "Bucket existence check error"
return 1
fi
local create_result
if [[ $exists_result -eq 1 ]]; then
create_bucket "$1" || create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "Error creating bucket"
if [[ $exists_result -eq 0 ]]; then
delete_bucket_or_contents "$1" || delete_result=$?
if [[ delete_result -ne 0 ]]; then
echo "error deleting bucket or contents"
return 1
fi
fi
if [[ $RECREATE_BUCKETS != "true" ]]; then
echo "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
return 1
fi
local create_result
create_bucket "$1" || create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "Error creating bucket"
return 1
fi
echo "Bucket creation success"
return 0
}
@@ -135,13 +212,13 @@ check_and_put_object() {
return 1
fi
object_exists "$2" || local exists_result=$?
if [ $exists_result -eq 2 ]; then
if [ "$exists_result" -eq 2 ]; then
echo "error checking if object exists"
return 1
fi
if [ $exists_result -eq 1 ]; then
if [ "$exists_result" -eq 1 ]; then
put_object "$1" "$2" || local put_result=$?
if [ $put_result -ne 0 ]; then
if [ "$put_result" -ne 0 ]; then
echo "error adding object"
return 1
fi
@@ -150,7 +227,7 @@ check_and_put_object() {
}
# delete object from versitygw
# param: object location
# param: object path, including bucket name
# return 0 for success, 1 for failure
delete_object() {
if [ $# -ne 1 ]; then
@@ -266,8 +343,232 @@ get_bucket_acl() {
local exit_code=0
acl=$(aws s3api get-bucket-acl --bucket "$1" 2>&1) || exit_code="$?"
if [ $exit_code -ne 0 ]; then
echo "Error: $acl"
echo "Error getting bucket ACLs: $acl"
return 1
fi
export acl
}
# add tags to bucket
# params: bucket, key, value
# return: 0 for success, 1 for error
put_bucket_tag() {
if [ $# -ne 3 ]; then
echo "bucket tag command missing bucket name, key, value"
return 1
fi
local error
local result
error=$(aws s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$?
if [[ $result -ne 0 ]]; then
echo "Error adding bucket tag: $error"
return 1
fi
return 0
}
# get bucket tags
# params: bucket
# export 'tags' on success, return 1 for error
get_bucket_tags() {
if [ $# -ne 1 ]; then
echo "get bucket tag command missing bucket name"
return 1
fi
local result
tags=$(aws s3api get-bucket-tagging --bucket "$1") || result=$?
if [[ $result -ne 0 ]]; then
echo "error getting bucket tags: $tags"
return 1
fi
export tags
}
# add tags to object
# params: object, key, value
# return: 0 for success, 1 for error
put_object_tag() {
if [ $# -ne 4 ]; then
echo "object tag command missing object name, file, key, and/or value"
return 1
fi
local error
local result
error=$(aws s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
if [[ $result -ne 0 ]]; then
echo "Error adding object tag: $error"
return 1
fi
return 0
}
# get object tags
# params: bucket
# export 'tags' on success, return 1 for error
get_object_tags() {
if [ $# -ne 2 ]; then
echo "get object tag command missing bucket and/or key"
return 1
fi
local result
tags=$(aws s3api get-object-tagging --bucket "$1" --key "$2") || result=$?
if [[ $result -ne 0 ]]; then
echo "error getting object tags: $tags"
return 1
fi
export tags
}
# create a test file and export folder. do so in temp folder
# params: filename
# export test file folder on success, return 1 for error
create_test_files() {
if [ $# -lt 1 ]; then
echo "create test files command missing filename"
return 1
fi
test_file_folder=.
if [[ -z "$GITHUB_ACTIONS" ]]; then
test_file_folder=${TMPDIR}versity-gwtest
mkdir -p "$test_file_folder" || local mkdir_result=$?
if [[ $mkdir_result -ne 0 ]]; then
echo "error creating test file folder"
fi
fi
for name in "$@"; do
touch "$test_file_folder"/"$name" || local touch_result=$?
if [[ $touch_result -ne 0 ]]; then
echo "error creating file $name"
fi
done
export test_file_folder
}
# delete a test file
# params: filename
# return: 0 for success, 1 for error
delete_test_files() {
if [ $# -lt 1 ]; then
echo "delete test files command missing filenames"
return 1
fi
if [ -z "$test_file_folder" ]; then
echo "no test file folder defined, not deleting"
return 1
fi
for name in "$@"; do
rm "$test_file_folder"/"$name" || rm_result=$?
if [[ $rm_result -ne 0 ]]; then
echo "error deleting file $name"
fi
done
return 0
}
# list objects in bucket, v1
# param: bucket
# export objects on success, return 1 for failure
list_objects_s3api_v1() {
if [ $# -ne 1 ]; then
echo "list objects command missing bucket"
return 1
fi
objects=$(aws s3api list-objects --bucket "$1") || local result=$?
if [[ $result -ne 0 ]]; then
echo "error listing objects: $objects"
return 1
fi
export objects
}
# list objects in bucket, v2
# param: bucket
# export objects on success, return 1 for failure
list_objects_s3api_v2() {
if [ $# -ne 1 ]; then
echo "list objects command missing bucket and/or path"
return 1
fi
objects=$(aws s3api list-objects-v2 --bucket "$1") || local result=$?
if [[ $result -ne 0 ]]; then
echo "error listing objects: $objects"
return 1
fi
export objects
}
# perform a multi-part upload
# params: bucket, key, source file location, number of parts
# return 0 for success, 1 for failure
multipart_upload() {
if [ $# -ne 4 ]; then
echo "multipart upload command missing bucket, key, file, and/or part count"
return 1
fi
file_size=$(stat -c %s "$3" 2>/dev/null || stat -f %z "$3" 2>/dev/null)
part_size=$((file_size / $4))
remainder=$((file_size % $4))
if [[ remainder -ne 0 ]]; then
part_size=$((part_size+1))
fi
local error
local split_result
error=$(split -a 1 -d -b "$part_size" "$3" "$3"-) || split_result=$?
if [[ $split_result -ne 0 ]]; then
echo "error splitting file: $error"
return 1
fi
local multipart_data
multipart_data=$(aws s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$?
if [[ $created -ne 0 ]]; then
echo "Error creating multipart upload: $upload_id"
return 1
fi
local upload_id
upload_id=$(echo "$multipart_data" | jq '.UploadId')
upload_id="${upload_id//\"/}"
local parts="["
for ((i = 1; i <= $4; i++)); do
local etag_json
etag_json=$(aws s3api upload-part --bucket "$1" --key "$2" --upload-id "$upload_id" --part-number "$i" --body "$3-$((i-1))") || local uploaded=$?
if [[ $uploaded -ne 0 ]]; then
echo "Error uploading part $i: $etag_json"
return 1
fi
local etag
etag=$(echo "$etag_json" | jq '.ETag')
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
if [[ $i -ne $4 ]]; then
parts+=","
fi
done
parts+="]"
error=$(aws s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$?
if [[ $completed -ne 0 ]]; then
echo "Error completing upload: $error"
return 1
fi
return 0
}
# copy a file to/from S3
# params: source, destination
# return 0 for success, 1 for failure
copy_file() {
if [ $# -ne 2 ]; then
echo "copy file command requires src and dest"
return 1
fi
local result
error=$(aws s3 cp "$1" "$2") || result=$?
if [[ $result -ne 0 ]]; then
echo "Error copying file: $error"
return 1
fi
return 0
}

2
tests/util_posix.sh
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env bats
# check if object exists both on S3 and locally
# param: object path