mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-03 10:35:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: presign work, s3 backend testing

Browse Source
This commit is contained in:
Luke McCrone
2024-03-19 13:22:25 -03:00
parent 0011ccd80e
commit 9fd22ca8e7
10 changed files with 269 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/system.yml vendored
View File

@@ -38,7 +38,7 @@ jobs:
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
chmod 755 /usr/local/bin/mc
- name: Build and run
- name: Build and run, posix backend
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -54,6 +54,20 @@ jobs:
mkdir /tmp/cover
VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
#- name: Build and run, s3 backend
# run: |
# make testbin
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export AWS_REGION=us-east-1
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
# aws configure set aws_region $AWS_REGION --profile versity_s3
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export WORKSPACE=$GITHUB_WORKSPACE
# VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
- name: Coverage report
run: |
go tool covdata percent -i=/tmp/cover

13
Dockerfile_test_bats
View File

@@ -1,6 +1,9 @@
FROM --platform=linux/arm64 ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
ARG SECRETS_FILE=tests/.secrets
ARG CONFIG_FILE=tests/.env.docker
ENV TZ=Etc/UTC
RUN apt-get update && \
apt-get install -y --no-install-recommends \
@@ -12,6 +15,7 @@ RUN apt-get update && \
tzdata \
s3cmd \
jq \
bc \
ca-certificates && \
update-ca-certificates && \
rm -rf /var/lib/apt/lists/*
@@ -57,11 +61,12 @@ USER tester
COPY --chown=tester:tester . /home/tester
WORKDIR /home/tester
RUN cp tests/.env.docker.default tests/.env.docker
RUN cp ${CONFIG_FILE}.default $CONFIG_FILE
#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
RUN cp tests/s3cfg.local.default tests/s3cfg.local
RUN make
RUN . tests/.secrets && \
RUN . $SECRETS_FILE && \
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
@@ -74,6 +79,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
ENV WORKSPACE=.
ENV VERSITYGW_TEST_ENV=tests/.env.docker
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
CMD ["tests/run_all.sh"]
CMD ["tests/run_all.sh"]

14
tests/.env.s3.default Normal file
View File

@@ -0,0 +1,14 @@
AWS_PROFILE=versity_s3
AWS_ENDPOINT_URL=https://127.0.0.1:7070
VERSITY_EXE=./versitygw
RUN_VERSITYGW=true
BACKEND=s3
LOCAL_FOLDER=/tmp/gw
BUCKET_ONE_NAME=versity-gwtest-bucket-one
BUCKET_TWO_NAME=versity-gwtest-bucket-two
#RECREATE_BUCKETS=true
CERT=$PWD/cert.pem
KEY=$PWD/versitygw.pem
S3CMD_CONFIG=./tests/s3cfg.local.default
SECRETS_FILE=./tests/.secrets.s3
MC_ALIAS=versity

1
tests/s3cfg.local.default
View File

@@ -3,6 +3,7 @@ host_base = 127.0.0.1:7070
host_bucket = 127.0.0.1:7070
bucket_location = us-east-1
use_https = True
signurl_use_https = True
# Enable S3 v4 signature APIs
signature_v2 = False

4
tests/test_aws.sh
View File

@@ -315,3 +315,7 @@ source ./tests/test_common.sh
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
@test "test-presigned-url-utf8-chars" {
test_common_presigned_url_utf8_chars "aws"
}

37
tests/test_common.sh
View File

@@ -256,3 +256,40 @@ test_common_multipart_upload() {
delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
}
test_common_presigned_url_utf8_chars() {
if [[ $# -ne 1 ]]; then
echo "presigned url command missing command type"
return 1
fi
local bucket_file="my-$%^&*;"
local bucket_file_copy="bucket-file-copy"
bucket_file_data="test file\n"
create_test_files "$bucket_file" || local created=$?
printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file"
setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$?
[[ $put_result -eq 0 ]] || fail "Failed to add object $bucket_file"
create_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" || presigned_result=$?
[[ $presigned_result -eq 0 ]] || fail "presigned url creation failure"
error=$(curl -k -v "$presigned_url" -o "$test_file_folder"/"$bucket_file_copy") || curl_result=$?
if [[ $curl_result -ne 0 ]]; then
fail "error downloading file with curl: $error"
fi
compare_files "$test_file_folder"/"$bucket_file" "$test_file_folder"/"$bucket_file_copy" || compare_result=$?
if [[ $compare_result -ne 0 ]]; then
echo "file one: $(cat "$test_file_folder"/"$bucket_file")"
echo "file two: $(cat "$test_file_folder"/"$bucket_file_copy")"
fail "files don't match"
fi
delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
delete_test_files "$bucket_file" "$bucket_file_copy"
}

4
tests/test_mc.sh
View File

@@ -37,3 +37,7 @@ export RUN_MC=true
@test "test_multipart_upload_mc" {
test_common_multipart_upload "mc"
}
@test "test_presigned_url_utf8_chars_mc" {
test_common_presigned_url_utf8_chars "mc"
}

6
tests/test_s3cmd.sh
View File

@@ -31,4 +31,8 @@ export RUN_S3CMD=true
@test "test_multipart_upload_s3cmd" {
test_common_multipart_upload "s3cmd"
}
}
#@test "test_presigned_url_utf8_chars_s3cmd" {
# test_common_presigned_url_utf8_chars "s3cmd"
#}

25
tests/util.sh
View File

@@ -901,3 +901,28 @@ upload_part_copy() {
etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
export etag
}
create_presigned_url() {
if [[ $# -ne 3 ]]; then
echo "create presigned url function requires command type, bucket, and filename"
return 1
fi
local presign_result=0
if [[ $1 == 'aws' ]]; then
presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
elif [[ $1 == 's3cmd' ]]; then
presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
elif [[ $1 == 'mc' ]]; then
presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
presigned_url="${presigned_url_data#*Share: }"
else
echo "unrecognized command type $1"
return 1
fi
if [[ $presign_result -ne 0 ]]; then
echo "error generating presigned url: $presigned_url"
return 1
fi
export presigned_url
}

180
tests/versity.sh
View File

@@ -1,5 +1,30 @@
#!/bin/bash
source ./tests/util_file.sh
check_exe_params_versity() {
if [ -z "$LOCAL_FOLDER" ]; then
echo "No local storage folder set"
return 1
elif [ -z "$VERSITY_EXE" ]; then
echo "No versity executable location set"
return 1
elif [ -z "$BACKEND" ]; then
echo "No backend parameter set (options: 'posix')"
return 1
fi
if [ "$BACKEND" == 's3' ]; then
if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
echo "missing second AWS access key ID for s3 backend"
return 1
fi
if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
echo "missing second AWS secret access key for s3 backend"
return 1
fi
fi
}
check_exe_params() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
echo "No AWS access key set"
@@ -21,14 +46,9 @@ check_exe_params() {
return 1
fi
if [[ $RUN_VERSITYGW == "true" ]]; then
if [ -z "$LOCAL_FOLDER" ]; then
echo "No local storage folder set"
return 1
elif [ -z "$VERSITY_EXE" ]; then
echo "No versity executable location set"
return 1
elif [ -z "$BACKEND" ]; then
echo "No backend parameter set (options: 'posix')"
local check_result
check_exe_params_versity || check_result=$?
if [[ $check_result -ne 0 ]]; then
return 1
fi
fi
@@ -59,32 +79,142 @@ start_versity() {
fi
if [ "$RUN_VERSITYGW" == "true" ]; then
base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY VGW_REGION=$AWS_REGION $VERSITY_EXE"
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=" --cert $CERT --key $KEY"
run_versity_app || run_result=$?
if [[ $run_result -ne 0 ]]; then
echo "error starting versity apps"
return 1
fi
base_command+=" $BACKEND $LOCAL_FOLDER &"
eval "$base_command"
versitygw_pid=$!
export versitygw_pid VERSITY_EXE BACKEND LOCAL_FOLDER
fi
export versitygw_pid AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
}
start_versity_process() {
if [[ $# -ne 1 ]]; then
echo "start versity process function requires number"
return 1
fi
create_test_file_folder || create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "error creating test log folder"
return 1
fi
base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1")
("${base_command[@]}") &
if [[ $? -ne 0 ]]; then
echo "error running versitygw command: $(cat "$test_file_folder/versity_log_$1.txt")"
return 1
fi
eval versitygw_pid_"$1"=$!
local pid
eval pid=\$versitygw_pid_"$1"
sleep 1
local proc_check
check_result=$(kill -0 $pid 2>&1) || proc_check=$?
if [[ $proc_check -ne 0 ]]; then
echo "versitygw failed to start: $check_result"
echo "log data: $(cat "$test_file_folder/versity_log_$1.txt")"
return 1
fi
export versitygw_pid_"$1"
}
run_versity_app_posix() {
if [[ $# -ne 3 ]]; then
echo "run versity app w/posix command requires access ID, secret key, process number"
return 1
fi
base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
base_command+=(posix "$LOCAL_FOLDER")
export base_command
local versity_result
start_versity_process "$3" || versity_result=$?
if [[ $versity_result -ne 0 ]]; then
echo "error starting versity process"
return 1
fi
return 0
}
run_versity_app_s3() {
if [[ $# -ne 1 ]]; then
echo "run versity app w/s3 command requires process number"
return 1
fi
base_command=("$VERSITY_EXE" --port=":7071" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=(--cert "$CERT" --key "$KEY")
fi
base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
export base_command
local versity_result
start_versity_process "$1" || versity_result=$?
if [[ $versity_result -ne 0 ]]; then
echo "error starting versity process"
return 1
fi
return 0
}
run_versity_app() {
if [[ $BACKEND == 'posix' ]]; then
run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
if [[ $result_one -ne 0 ]]; then
echo "error starting versity app"
return 1
fi
elif [[ $BACKEND == 's3' ]]; then
run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
if [[ $result_one -ne 0 ]]; then
echo "error starting versity app"
return 1
fi
run_versity_app_s3 "2" || result_two=$?
if [[ $result_two -ne 0 ]]; then
echo "error starting second versity app"
return 1
fi
else
echo "unrecognized backend type $BACKEND"
return 1
fi
}
stop_single_process() {
if [[ $# -ne 1 ]]; then
echo "stop single process function requires process ID"
return 1
fi
if ps -p "$1" > /dev/null; then
kill "$1"
wait "$1" || true
else
echo "Process with PID $1 does not exist."
fi
}
stop_versity() {
if [ "$RUN_VERSITYGW" == "false" ]; then
return
fi
if [ -n "$versitygw_pid" ]; then
if ps -p "$versitygw_pid" > /dev/null; then
kill "$versitygw_pid"
wait "$versitygw_pid" || true
else
echo "Process with PID $versitygw_pid does not exist."
local result_one
local result_two
# shellcheck disable=SC2154
stop_single_process "$versitygw_pid_1" || result_one=$?
if [[ $result_one -ne 0 ]]; then
echo "error stopping versity process"
fi
if [[ $BACKEND == 's3' ]]; then
# shellcheck disable=SC2154
stop_single_process "$versitygw_pid_2" || result_two=$?
if [[ $result_two -ne 0 ]]; then
echo "error stopping versity process two"
fi
else
echo "versitygw_pid is not set or empty."
fi
}