mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-19 14:34:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: ListObjectsV1 updates, file code changes, multipart updates

Browse Source
This commit is contained in:
Luke McCrone
2026-04-07 15:24:20 -03:00
parent 09413c5f60
commit d7d9179db0
31 changed files with 764 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/util/util_file.sh
-335
View File
@@ -1,335 +0,0 @@
#!/usr/bin/env bats
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/logger.sh
source ./tests/commands/get_object.sh
# create a test file and export folder. do so in temp folder
# params: filenames
# fail if error
create_test_files() {
log 6 "create_test_files"
if [ $# -lt 1 ]; then
log 2 "'create_test_files' requires file names"
return 1
fi
for name in "$@"; do
if ! create_test_file "$name"; then
log 2 "error creating test file"
return 1
fi
done
return 0
}
# params: filename, size (optional, defaults to 10)
create_test_file() {
if [[ ( $# -lt 1 ) || ( $# -gt 2 ) ]]; then
log 2 "'create_test_file' requires filename, size (optional)"
return 1
fi
if [[ -e "$TEST_FILE_FOLDER/$1" ]]; then
if ! error=$(rm "$TEST_FILE_FOLDER/$1" 2>&1); then
log 2 "error removing existing file: $error"
return 1
fi
fi
if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then
log 2 "error creating new file: $error"
return 1
fi
if [ -z "$2" ]; then
file_size=10
else
file_size="$2"
fi
if [ "$file_size" -eq 0 ]; then
return 0
fi
if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$1" bs=1 count="$file_size" 2>&1); then
log 2 "error adding data to file: $error"
return 1
fi
return 0
}
create_file_single_char() {
if [ "$#" -ne 3 ]; then
log 2 "'create_file_single_char' requires filename, size, char"
return 1
fi
if [[ -e "$TEST_FILE_FOLDER/$1" ]]; then
if ! error=$(rm "$TEST_FILE_FOLDER/$1" 2>&1); then
log 2 "error removing existing file: $error"
return 1
fi
fi
if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then
log 2 "error creating new file: $error"
return 1
fi
if ! error=$(dd if=/dev/zero bs=1 count="$2" | tr '\0' "$3" > "$TEST_FILE_FOLDER/$1" 2>&1); then
log 2 "error adding data to file: $error"
return 1
fi
return 0
}
# params: folder name
# fail if error
create_test_folder() {
if [ $# -lt 1 ]; then
log 2 "'create_test_folder' requires folder names"
return 1
fi
for name in "$@"; do
if ! error=$(mkdir -p "$TEST_FILE_FOLDER"/"$name" 2>&1); then
log 2 "error creating folder $name: $error"
return 1
fi
done
return 0
}
# delete a test file
# params: filename
# return: 0 for success, 1 for error
delete_test_files() {
if [ $# -lt 1 ]; then
log 2 "delete test files command missing filenames"
return 1
fi
if [ -z "$TEST_FILE_FOLDER" ]; then
log 2 "no test file folder defined, not deleting"
return 1
fi
for name in "$@"; do
rm -rf "${TEST_FILE_FOLDER:?}"/"${name:?}" || rm_result=$?
if [[ $rm_result -ne 0 ]]; then
log 2 "error deleting file $name"
fi
done
return 0
}
get_file_size() {
if [ $# -ne 1 ]; then
log 2 "'get_file_size' requires file location"
return 1
fi
local file_size=""
if [[ "$OSTYPE" == "darwin"* ]]; then
if ! file_size=$(stat -f %z "$1" 2>&1); then
log 2 "error getting file size: $file_size"
return 1
fi
else
if ! file_size=$(stat -c %s "$1" 2>&1); then
log 2 "error getting file size: $file_size"
return 1
fi
fi
echo "$file_size"
}
# split file into pieces to test multipart upload
# param: file location
# return 0 for success, 1 for error
split_file() {
if [ $# -ne 2 ]; then
log 2 "'split_file' requires file name, number of pieces"
return 1
fi
file_size=$(stat -c %s "$1" 2>/dev/null || stat -f %z "$1" 2>/dev/null)
part_size=$((file_size / $2))
remainder=$((file_size % $2))
if [[ remainder -ne 0 ]]; then
part_size=$((part_size+1))
fi
local error
if ! error=$(split -a 1 -d -b "$part_size" "$1" "$1"- 2>&1); then
log 2 "error splitting file: $error"
return 1
fi
return 0
}
# compare files
# input: two files
# return 0 for same data, 1 for different data, 2 for error
compare_files() {
if [ $# -ne 2 ]; then
log 2 "file comparison requires two files"
return 2
fi
log 5 "comparing files '$1' and '$2'"
os=$(uname)
if [[ $os == "Darwin" ]]; then
if ! file_one_md5=$(md5 -q "$1" 2>&1); then
log 2 "error getting md5 for '$1': $file_one_md5"
return 2
fi
if ! file_two_md5=$(md5 -q "$2" 2>&1); then
log 2 "error getting md5 for '$2': $file_two_md5"
return 2
fi
else
if ! file_one_md5=$(md5sum "$1" | cut -d " " -f 1 2>&1); then
log 2 "error getting md5 for '$1': $file_one_md5"
return 2
fi
if ! file_two_md5=$(md5sum "$2" | cut -d " " -f 1 2>&1); then
log 2 "error getting md5 for '$2': $file_two_md5"
return 2
fi
fi
if [[ "$file_one_md5" == "$file_two_md5" ]]; then
return 0
fi
return 1
}
# generate 160MB file
# input: filename
# fail on error
create_large_file() {
log 6 "create_large_file"
if ! check_param_count_v2 "file name" 1 $#; then
return 1
fi
if ! create_large_file_with_size "$1" 160; then
log 2 "error creating 160MB file"
return 1
fi
return 0
}
create_large_file_v2() {
if ! file_name=$(get_file_name 2>&1); then
log 2 "error getting file name: $file_name"
return 1
fi
if ! create_large_file_with_size "$file_name" 160; then
log 2 "error creating 160MB file with name '$file_name'"
return 1
fi
echo "$file_name"
return 0
}
create_large_file_with_size() {
if ! check_param_count_v2 "file name, size in MB" 2 $#; then
return 1
fi
filesize=$(($2*1024*1024))
if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1024 count=$((filesize/1024)) 2>&1); then
log 2 "error adding data to large file: $error"
return 1
fi
return 0
}
create_and_split_large_file() {
if ! check_param_count_v2 "file name, size in MB, pieces" 3 $#; then
return 1
fi
if ! create_large_file_with_size "$1" "$2"; then
log 2 "error creating large file"
return 1
fi
if ! split_file "$TEST_FILE_FOLDER/$1" "$3"; then
log 2 "error splitting file"
return 1
fi
}
# param: number of files
# fail on error
create_test_file_count() {
if [ $# -ne 1 ]; then
log 2 "'create_test_file_count' requires number of files"
return 1
fi
for ((i=1;i<=$1;i++)) {
if ! error=$(touch "$TEST_FILE_FOLDER/file_$i" 2>&1); then
log 2 "error creating file_$i: $error"
return 1
fi
}
# shellcheck disable=SC2153
if [[ $LOG_LEVEL -ge 5 ]]; then
ls_result=$(ls "$TEST_FILE_FOLDER/file_*")
log 5 "$ls_result"
fi
return 0
}
download_and_compare_file_with_user() {
if ! check_param_count_gt "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then
return 1
fi
if ! download_file_with_user "$5" "$6" "$2" "$3" "$4" "$7"; then
log 2 "error downloading file"
return 1
fi
if ! compare_files "$1" "$4"; then
log 2 "files don't match"
return 1
fi
return 0
}
download_and_compare_file() {
log 6 "download_and_compare_file"
if ! check_param_count_gt "original file, bucket, key, destination, chunk size (optional)" 4 $#; then
return 1
fi
if ! download_and_compare_file_with_user "$1" "$2" "$3" "$4" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$5"; then
log 2 "error downloading and comparing file with user"
return 1
fi
return 0
}
# params: src, dst
# fail if error
copy_file_locally() {
if [ $# -ne 2 ]; then
log 2 "'copy_file_locally' requires src, dst"
return 1
fi
if ! error=$(cp "$1" "$2" 2>&1); then
log 2 "error copying file: $error"
return 1
fi
return 0
}
# params: src, dst
# fail if error
move_file_locally() {
if [ $# -ne 2 ]; then
log 2 "'move_file_locally' requires src, dst"
return 1
fi
if ! error=$(mv "$1" "$2" 2>&1); then
log 2 "error moving file: $error"
return 1
fi
return 0
}
tests/util/util_multipart.sh
+13 -8
View File
@@ -48,14 +48,15 @@ multipart_upload_from_bucket() {
return 1
fi
if ! split_file "$3" "$4"; then
if ! segments=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file"
return 1
fi
read -r -a segment_array <<< "$segments"
for ((i=0;i<$4;i++)) {
log 5 "key: $3"
if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then
if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then
log 2 "error copying object"
return 1
fi
@@ -77,13 +78,15 @@ split_and_put_file() {
if ! check_param_count "split_and_put_file" "bucket, key, copy source, part count" 4 $#; then
return 1
fi
if ! split_file "$3" "$4"; then
log 2 "error splitting file"
if ! file_parts=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file: $file_parts"
return 1
fi
read -r -a part_array <<< "$file_parts"
for ((i=0;i<$4;i++)) {
log 5 "key: $2, file info: $(ls -l "$3"-"$i")"
if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then
log 5 "key: $2, file info: $(ls -l "${part_array[$i]}")"
if ! put_object "s3api" "${part_array[$i]}" "$1" "$2-$i"; then
log 2 "error copying object"
return 1
fi
@@ -95,13 +98,15 @@ multipart_upload_from_bucket_range() {
if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then
return 1
fi
if ! split_file "$3" "$4"; then
if ! segments=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file"
return 1
fi
read -r -a segment_array <<< "$segments"
for ((i=0;i<$4;i++)) {
log 5 "key: $3, file info: $(ls -l "$3"-"$i")"
if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then
if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then
log 2 "error copying object"
return 1
fi
tests/util/util_multipart_before_completion.sh
+11 -9
View File
@@ -233,15 +233,15 @@ multipart_upload_before_completion() {
# params: bucket, key, file to split and upload, number of file parts to upload
# return: 0 for success, 1 for failure
multipart_upload_before_completion_with_user() {
if [ $# -ne 6 ]; then
log 2 "multipart upload pre-completion command missing bucket, key, file, part count, username, password"
if ! check_param_count_v2 "bucket, key, file, part count, username, password" 6 $#; then
return 1
fi
if ! split_file "$3" "$4"; then
log 2 "error splitting file"
if ! segments=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file: $segments"
return 1
fi
read -r -a segment_array <<< "$segments"
if ! create_multipart_upload_s3api_with_user "$1" "$2" "$5" "$6"; then
log 2 "error creating multpart upload"
@@ -251,7 +251,7 @@ multipart_upload_before_completion_with_user() {
parts="["
for ((i = 1; i <= $4; i++)); do
# shellcheck disable=SC2154
if ! upload_part_with_user "$1" "$2" "$upload_id" "$3" "$i" "$5" "$6"; then
if ! upload_part_with_user "$1" "$2" "$upload_id" "${segment_array[$((i-1))]}" "$i" "$5" "$6"; then
log 2 "error uploading part $i"
return 1
fi
@@ -271,10 +271,11 @@ multipart_upload_before_completion_with_params() {
return 1
fi
if ! result=$(split_file "$3" "$4" 2>&1); then
if ! segments=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file: $result"
return 1
fi
read -r -a segment_array <<< "$segments"
if ! create_multipart_upload_s3api_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}"; then
log 2 "error creating multpart upload"
@@ -283,7 +284,7 @@ multipart_upload_before_completion_with_params() {
parts="["
for ((i = 1; i <= $4; i++)); do
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
if ! upload_part "$1" "$2" "$upload_id" "${segment_array[$((i-1))]}" "$i"; then
log 2 "error uploading part $i"
return 1
fi
@@ -303,10 +304,11 @@ multipart_upload_before_completion_custom() {
return 1
fi
if ! result=$(split_file "$3" "$4" 2>&1); then
if ! segments=$(split_file "$3" "$4" 2>&1); then
log 2 "error splitting file"
return 1
fi
read -r -a segment_array <<< "$segments"
# shellcheck disable=SC2086 disable=SC2048
if ! create_multipart_upload_custom "$1" "$2" ${*:5}; then
@@ -317,7 +319,7 @@ multipart_upload_before_completion_custom() {
parts="["
for ((i = 1; i <= $4; i++)); do
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
if ! upload_part "$1" "$2" "$upload_id" "${segment_array[$i]}" "$i"; then
log 2 "error uploading part $i"
return 1
fi
tests/util/util_object.sh
+1
View File
@@ -28,6 +28,7 @@ source ./tests/commands/delete_object.sh
source ./tests/commands/get_bucket_acl.sh
source ./tests/commands/get_bucket_ownership_controls.sh
source ./tests/commands/get_bucket_policy.sh
source ./tests/commands/get_object.sh
source ./tests/commands/get_object_legal_hold.sh
source ./tests/commands/get_object_lock_configuration.sh
source ./tests/commands/head_bucket.sh