mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-08 12:41:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #400 from versity/test_cmdline_list_parts

Browse Source 
Test cmdline list parts
This commit is contained in:
Ben McClelland
2024-02-12 16:08:30 -08:00
committed by GitHub
parent 5931d713f2 7a56c7e15e
commit cde033811f
3 changed files with 212 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/posix_tests.sh
View File

@@ -16,14 +16,17 @@ source ./tests/util_posix.sh
mkdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
local object="$BUCKET_ONE_NAME"/"$object_name"
touch "$LOCAL_FOLDER"/"$object"
bucket_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_exists_two=$?
[[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
object_exists_remote_and_local "$object" || local object_exists_two=$?
[[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
rm "$LOCAL_FOLDER"/"$object"
sleep 1
object_not_exists_remote_and_local "$object" || local object_deleted=$?
[[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
rmdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_not_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_deleted=$?
@@ -44,14 +47,17 @@ source ./tests/util_posix.sh
[[ $created -eq 0 ]] || fail "Error creating bucket"
put_object "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?"
[[ result -eq 0 ]] || fail "Error adding object one"
chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
sleep 1
object_is_accessible "$bucket_name" $object_name || local accessible=$?
[[ $accessible -eq 1 ]] || fail "Object should be inaccessible"
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
sleep 1
object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
[[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
delete_object "$bucket_name"/$object_name
delete_bucket_or_contents "$bucket_name"
delete_test_files $object_name
@@ -67,13 +73,16 @@ source ./tests/util_posix.sh
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating bucket"
fi
chmod 000 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible=$?
[[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible"
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
sleep 1
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible_two=$?
[[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}

128
tests/s3_bucket_tests.sh
View File

@@ -12,8 +12,10 @@ source ./tests/util.sh
setup_bucket "$BUCKET_ONE_NAME" || local create_result=$?
[[ $create_result -eq 0 ]] || fail "Failed to create bucket"
bucket_exists "$BUCKET_ONE_NAME" || local exists_three=$?
[[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
delete_bucket_or_contents "$BUCKET_ONE_NAME" || local delete_result_two=$?
[[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
}
@@ -25,6 +27,7 @@ source ./tests/util.sh
setup_bucket "$BUCKET_ONE_NAME" || local setup_result=$?
[[ $setup_result -eq 0 ]] || fail "error setting up bucket"
create_test_files "$object_name" || local create_result=$?
object="$BUCKET_ONE_NAME"/$object_name
@@ -32,10 +35,12 @@ source ./tests/util.sh
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
object_exists "$object" || local exists_result_one=$?
[[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
delete_object "$object" || local delete_result=$?
[[ $delete_result -eq 0 ]] || fail "Failed to delete object"
object_exists "$object" || local exists_result_two=$?
[[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$object_name"
}
@@ -47,6 +52,7 @@ source ./tests/util.sh
[[ $setup_result_one -eq 0 ]] || fail "Bucket one setup error"
setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$?
[[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
list_buckets
local bucket_one_found=false
local bucket_two_found=false
@@ -57,12 +63,16 @@ source ./tests/util.sh
bucket_two_found=true
fi
if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
return
break
fi
done
fail "'$BUCKET_ONE_NAME' and/or '$BUCKET_TWO_NAME' not listed (all buckets: ${bucket_array[*]})"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_bucket_or_contents "$BUCKET_TWO_NAME"
if [ $bucket_one_found != true ] || [ $bucket_two_found != true ]; then
fail "'$BUCKET_ONE_NAME' and/or '$BUCKET_TWO_NAME' not listed (all buckets: ${bucket_array[*]})"
fi
}
# test listing a bucket's objects on versitygw
@@ -78,6 +88,7 @@ source ./tests/util.sh
[[ result_two -eq 0 ]] || fail "Error adding object one"
put_object "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
[[ result_three -eq 0 ]] || fail "Error adding object two"
list_objects "$BUCKET_ONE_NAME"
local object_one_found=false
local object_two_found=false
@@ -88,11 +99,13 @@ source ./tests/util.sh
object_two_found=true
fi
done
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $object_one $object_two
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
fi
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $object_one $object_two
}
# test ability to retrieve bucket ACLs
@@ -100,10 +113,13 @@ source ./tests/util.sh
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating bucket"
get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Error retrieving acl"
id=$(echo "$acl" | jq '.Owner.ID')
[[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}
@@ -117,6 +133,7 @@ source ./tests/util.sh
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
[[ $result_one -eq 0 ]] || fail "Error creating bucket"
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
[[ $result_two -eq 0 ]] || fail "Error adding object one"
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
@@ -147,10 +164,12 @@ source ./tests/util.sh
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
[[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
tag_set=$(echo "$tags" | jq '.TagSet')
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
put_bucket_tag "$BUCKET_ONE_NAME" $key $value
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
[[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
@@ -158,6 +177,7 @@ source ./tests/util.sh
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
}
@@ -177,6 +197,7 @@ source ./tests/util.sh
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
list_objects_s3api_v1 "$BUCKET_ONE_NAME"
key_one=$(echo "$objects" | jq '.Contents[0].Key')
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
@@ -207,6 +228,7 @@ source ./tests/util.sh
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
list_objects_s3api_v2 "$BUCKET_ONE_NAME"
key_one=$(echo "$objects" | jq '.Contents[0].Key')
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
@@ -232,14 +254,15 @@ source ./tests/util.sh
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
put_object "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
[[ $get_result -eq 0 ]] || fail "Error getting object tags"
tag_set=$(echo "$tags" | jq '.TagSet')
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
[[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
@@ -247,6 +270,7 @@ source ./tests/util.sh
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
@@ -262,10 +286,14 @@ source ./tests/util.sh
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
[[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
copy_data=$(<"$test_file_folder"/$bucket_file-copy)
[[ $bucket_file_data == "$copy_data" ]] || fail "Data doesn't match"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
@@ -281,9 +309,95 @@ source ./tests/util.sh
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4
abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
[[ $abort_result -eq 0 ]] || fail "Abort failed"
object_exists "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
[[ $exists -eq 1 ]] || fail "Upload file exists after abort"
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
# test multi-part upload list parts command
@test "test-multipart-upload-list-parts" {
local bucket_file="bucket-file"
local bucket_file_data="test file\n"
create_test_files "$bucket_file" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || list_result=$?
[[ list_result -eq 0 ]] || fail "Listing multipart upload parts failed"
declare -a parts_map
for ((i=0;i<$4;i++)) {
local part_number
local etag
part_number=$(echo "$parts" | jq ".[$i].PartNumber")
if [[ $part_number -eq "" ]]; then
echo "error: blank part number"
return 1
fi
etag=$(echo "$parts" | jq ".[$i].ETag")
if [[ $etag == "" ]]; then
echo "error: blank etag"
return 1
fi
parts_map[$part_number]=$etag
}
for ((i=0;i<$4;i++)) {
local part_number
local etag
part_number=$(echo "$listed_parts" | jq ".Parts[$i].PartNumber")
etag=$(echo "$listed_parts" | jq ".Parts[$i].ETag")
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
echo "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
return 1
fi
}
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
# test listing of active uploads
@test "test-multipart-upload-list-uploads" {
local bucket_file_one="bucket-file-one"
local bucket_file_two="bucket-file-two"
create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two"
[[ $? -eq 0 ]] || fail "failed to list multipart uploads"
local key_one
local key_two
key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
key_one=${key_one//\"/}
key_two=${key_two//\"/}
echo "$test_file_folder/${bucket_file_one}abc"
echo "${key_one}abc"
echo "Length of test_file_folder/bucket_file_one: ${#test_file_folder}/${#bucket_file_one}"
echo "Length of key_one: ${#key_one}"
if [[ "$test_file_folder/$bucket_file_one" != *"$key_one" ]]; then
fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
fi
if [[ "$test_file_folder/$bucket_file_two" != *"$key_two" ]]; then
fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
fi
delete_bucket_or_contents "$BUCKET_ONE_NAME"
delete_test_files "$bucket_file_one" "$bucket_file_two"
}

88
tests/util.sh
View File

@@ -8,6 +8,7 @@ create_bucket() {
echo "create bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
@@ -26,6 +27,7 @@ delete_bucket() {
echo "delete bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
@@ -48,6 +50,7 @@ delete_bucket_recursive() {
echo "delete bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 rb s3://"$1" --force 2>&1) || exit_code="$?"
@@ -70,6 +73,7 @@ delete_bucket_contents() {
echo "delete bucket missing bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws s3 rm s3://"$1" --recursive 2>&1) || exit_code="$?"
@@ -88,6 +92,7 @@ bucket_exists() {
echo "bucket exists check missing bucket name"
return 2
fi
local exit_code=0
local error
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
@@ -579,6 +584,7 @@ multipart_upload_before_completion() {
fi
done
parts+="]"
export parts
}
@@ -606,6 +612,26 @@ multipart_upload() {
return 0
}
# run the abort multipart command
# params: bucket, key, upload ID
# return 0 for success, 1 for failure
run_abort_command() {
if [ $# -ne 3 ]; then
echo "command to run abort requires bucket, key, upload ID"
return 1
fi
error=$(aws s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
if [[ $aborted -ne 0 ]]; then
echo "Error aborting upload: $error"
return 1
fi
return 0
}
# run upload, then abort it
# params: bucket, key, local file location, number of parts to split into before uploading
# return 0 for success, 1 for failure
abort_multipart_upload() {
if [ $# -ne 4 ]; then
echo "abort multipart upload command missing bucket, key, file, and/or part count"
@@ -618,12 +644,8 @@ abort_multipart_upload() {
return 1
fi
error=$(aws s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$upload_id") || local aborted=$?
if [[ $aborted -ne 0 ]]; then
echo "Error aborting upload: $error"
return 1
fi
return 0
run_abort_command "$1" "$2" "$upload_id"
return $?
}
# copy a file to/from S3
@@ -634,6 +656,7 @@ copy_file() {
echo "copy file command requires src and dest"
return 1
fi
local result
error=$(aws s3 cp "$1" "$2") || result=$?
if [[ $result -ne 0 ]]; then
@@ -642,3 +665,56 @@ copy_file() {
fi
return 0
}
# list parts of an unfinished multipart upload
# params: bucket, key, local file location, and parts to split into before upload
# export parts on success, return 1 for error
list_parts() {
if [ $# -ne 4 ]; then
echo "list multipart upload parts command missing bucket, key, file, and/or part count"
return 1
fi
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
if [[ $result -ne 0 ]]; then
echo "error performing pre-completion multipart upload"
return 1
fi
listed_parts=$(aws s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id") || local listed=$?
if [[ $listed -ne 0 ]]; then
echo "Error aborting upload: $parts"
return 1
fi
export listed_parts
}
# list unfinished multipart uploads
# params: bucket, key one, key two
# export current two uploads on success, return 1 for error
list_multipart_uploads() {
if [ $# -ne 3 ]; then
echo "list multipart uploads command requires bucket and two keys"
return 1
fi
create_multipart_upload "$1" "$2" || local create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "error creating multpart upload"
return 1
fi
create_multipart_upload "$1" "$3" || local create_result_two=$?
if [[ $create_result_two -ne 0 ]]; then
echo "error creating multpart upload two"
return 1
fi
uploads=$(aws s3api list-multipart-uploads --bucket "$1") || local list_result=$?
if [[ $list_result -ne 0 ]]; then
echo "error listing uploads: $uploads"
return 1
fi
export uploads
}