diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 3f3f21e..84edfd7 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -38,7 +38,7 @@ jobs: curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc chmod 755 /usr/local/bin/mc - - name: Build and run + - name: Build and run, posix backend run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST @@ -54,6 +54,20 @@ jobs: mkdir /tmp/cover VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh + #- name: Build and run, s3 backend + # run: | + # make testbin + # export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST + # export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn + # export AWS_REGION=us-east-1 + # aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3 + # aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3 + # aws configure set aws_region $AWS_REGION --profile versity_s3 + # export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST + # export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn + # export WORKSPACE=$GITHUB_WORKSPACE + # VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh + - name: Coverage report run: | go tool covdata percent -i=/tmp/cover diff --git a/Dockerfile_test_bats b/Dockerfile_test_bats index 5d79911..6730c1a 100644 --- a/Dockerfile_test_bats +++ b/Dockerfile_test_bats @@ -1,6 +1,9 @@ FROM --platform=linux/arm64 ubuntu:latest ARG DEBIAN_FRONTEND=noninteractive +ARG SECRETS_FILE=tests/.secrets +ARG CONFIG_FILE=tests/.env.docker + ENV TZ=Etc/UTC RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -12,6 +15,7 @@ RUN apt-get update && \ tzdata \ s3cmd \ jq \ + bc \ ca-certificates && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* @@ -57,11 +61,12 @@ USER tester COPY --chown=tester:tester . /home/tester WORKDIR /home/tester -RUN cp tests/.env.docker.default tests/.env.docker +RUN cp ${CONFIG_FILE}.default $CONFIG_FILE +#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3 RUN cp tests/s3cfg.local.default tests/s3cfg.local RUN make -RUN . tests/.secrets && \ +RUN . $SECRETS_FILE && \ export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \ aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \ aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \ @@ -74,6 +79,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com" ENV WORKSPACE=. -ENV VERSITYGW_TEST_ENV=tests/.env.docker +ENV VERSITYGW_TEST_ENV=$CONFIG_FILE -CMD ["tests/run_all.sh"] \ No newline at end of file +CMD ["tests/run_all.sh"] diff --git a/tests/.env.s3.default b/tests/.env.s3.default new file mode 100644 index 0000000..231cb67 --- /dev/null +++ b/tests/.env.s3.default @@ -0,0 +1,14 @@ +AWS_PROFILE=versity_s3 +AWS_ENDPOINT_URL=https://127.0.0.1:7070 +VERSITY_EXE=./versitygw +RUN_VERSITYGW=true +BACKEND=s3 +LOCAL_FOLDER=/tmp/gw +BUCKET_ONE_NAME=versity-gwtest-bucket-one +BUCKET_TWO_NAME=versity-gwtest-bucket-two +#RECREATE_BUCKETS=true +CERT=$PWD/cert.pem +KEY=$PWD/versitygw.pem +S3CMD_CONFIG=./tests/s3cfg.local.default +SECRETS_FILE=./tests/.secrets.s3 +MC_ALIAS=versity \ No newline at end of file diff --git a/tests/s3cfg.local.default b/tests/s3cfg.local.default index eab0a6b..d152d57 100644 --- a/tests/s3cfg.local.default +++ b/tests/s3cfg.local.default @@ -3,6 +3,7 @@ host_base = 127.0.0.1:7070 host_bucket = 127.0.0.1:7070 bucket_location = us-east-1 use_https = True +signurl_use_https = True # Enable S3 v4 signature APIs signature_v2 = False diff --git a/tests/test_aws.sh b/tests/test_aws.sh index 6618a6b..dc03203 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -315,3 +315,7 @@ source ./tests/test_common.sh delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" delete_test_files $bucket_file } + +@test "test-presigned-url-utf8-chars" { + test_common_presigned_url_utf8_chars "aws" +} diff --git a/tests/test_common.sh b/tests/test_common.sh index f05c86b..0449fb4 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -256,3 +256,40 @@ test_common_multipart_upload() { delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" delete_test_files $bucket_file } + +test_common_presigned_url_utf8_chars() { + + if [[ $# -ne 1 ]]; then + echo "presigned url command missing command type" + return 1 + fi + + local bucket_file="my-$%^&*;" + local bucket_file_copy="bucket-file-copy" + bucket_file_data="test file\n" + + create_test_files "$bucket_file" || local created=$? + printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file" + setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + + put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$? + [[ $put_result -eq 0 ]] || fail "Failed to add object $bucket_file" + + create_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" || presigned_result=$? + [[ $presigned_result -eq 0 ]] || fail "presigned url creation failure" + + error=$(curl -k -v "$presigned_url" -o "$test_file_folder"/"$bucket_file_copy") || curl_result=$? + if [[ $curl_result -ne 0 ]]; then + fail "error downloading file with curl: $error" + fi + compare_files "$test_file_folder"/"$bucket_file" "$test_file_folder"/"$bucket_file_copy" || compare_result=$? + if [[ $compare_result -ne 0 ]]; then + echo "file one: $(cat "$test_file_folder"/"$bucket_file")" + echo "file two: $(cat "$test_file_folder"/"$bucket_file_copy")" + fail "files don't match" + fi + + delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" + delete_test_files "$bucket_file" "$bucket_file_copy" +} diff --git a/tests/test_mc.sh b/tests/test_mc.sh index 3c95001..e97f6a4 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -37,3 +37,7 @@ export RUN_MC=true @test "test_multipart_upload_mc" { test_common_multipart_upload "mc" } + +@test "test_presigned_url_utf8_chars_mc" { + test_common_presigned_url_utf8_chars "mc" +} diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index f101d4e..a0a1b4b 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -31,4 +31,8 @@ export RUN_S3CMD=true @test "test_multipart_upload_s3cmd" { test_common_multipart_upload "s3cmd" -} \ No newline at end of file +} + +#@test "test_presigned_url_utf8_chars_s3cmd" { +# test_common_presigned_url_utf8_chars "s3cmd" +#} \ No newline at end of file diff --git a/tests/util.sh b/tests/util.sh index 9ad7ed2..5e38a63 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -901,3 +901,28 @@ upload_part_copy() { etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag') export etag } + +create_presigned_url() { + if [[ $# -ne 3 ]]; then + echo "create presigned url function requires command type, bucket, and filename" + return 1 + fi + + local presign_result=0 + if [[ $1 == 'aws' ]]; then + presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$? + elif [[ $1 == 's3cmd' ]]; then + presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$? + elif [[ $1 == 'mc' ]]; then + presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$? + presigned_url="${presigned_url_data#*Share: }" + else + echo "unrecognized command type $1" + return 1 + fi + if [[ $presign_result -ne 0 ]]; then + echo "error generating presigned url: $presigned_url" + return 1 + fi + export presigned_url +} diff --git a/tests/versity.sh b/tests/versity.sh index dbae359..ac57954 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -1,5 +1,30 @@ #!/bin/bash +source ./tests/util_file.sh + +check_exe_params_versity() { + if [ -z "$LOCAL_FOLDER" ]; then + echo "No local storage folder set" + return 1 + elif [ -z "$VERSITY_EXE" ]; then + echo "No versity executable location set" + return 1 + elif [ -z "$BACKEND" ]; then + echo "No backend parameter set (options: 'posix')" + return 1 + fi + if [ "$BACKEND" == 's3' ]; then + if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then + echo "missing second AWS access key ID for s3 backend" + return 1 + fi + if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then + echo "missing second AWS secret access key for s3 backend" + return 1 + fi + fi +} + check_exe_params() { if [ -z "$AWS_ACCESS_KEY_ID" ]; then echo "No AWS access key set" @@ -21,14 +46,9 @@ check_exe_params() { return 1 fi if [[ $RUN_VERSITYGW == "true" ]]; then - if [ -z "$LOCAL_FOLDER" ]; then - echo "No local storage folder set" - return 1 - elif [ -z "$VERSITY_EXE" ]; then - echo "No versity executable location set" - return 1 - elif [ -z "$BACKEND" ]; then - echo "No backend parameter set (options: 'posix')" + local check_result + check_exe_params_versity || check_result=$? + if [[ $check_result -ne 0 ]]; then return 1 fi fi @@ -59,32 +79,142 @@ start_versity() { fi if [ "$RUN_VERSITYGW" == "true" ]; then - base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY VGW_REGION=$AWS_REGION $VERSITY_EXE" - if [ -n "$CERT" ] && [ -n "$KEY" ]; then - base_command+=" --cert $CERT --key $KEY" + run_versity_app || run_result=$? + if [[ $run_result -ne 0 ]]; then + echo "error starting versity apps" + return 1 fi - base_command+=" $BACKEND $LOCAL_FOLDER &" - eval "$base_command" - versitygw_pid=$! - - export versitygw_pid VERSITY_EXE BACKEND LOCAL_FOLDER fi - export versitygw_pid AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL +} + +start_versity_process() { + if [[ $# -ne 1 ]]; then + echo "start versity process function requires number" + return 1 + fi + create_test_file_folder || create_result=$? + if [[ $create_result -ne 0 ]]; then + echo "error creating test log folder" + return 1 + fi + base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1") + ("${base_command[@]}") & + if [[ $? -ne 0 ]]; then + echo "error running versitygw command: $(cat "$test_file_folder/versity_log_$1.txt")" + return 1 + fi + eval versitygw_pid_"$1"=$! + local pid + eval pid=\$versitygw_pid_"$1" + sleep 1 + + local proc_check + check_result=$(kill -0 $pid 2>&1) || proc_check=$? + if [[ $proc_check -ne 0 ]]; then + echo "versitygw failed to start: $check_result" + echo "log data: $(cat "$test_file_folder/versity_log_$1.txt")" + return 1 + fi + export versitygw_pid_"$1" +} + +run_versity_app_posix() { + if [[ $# -ne 3 ]]; then + echo "run versity app w/posix command requires access ID, secret key, process number" + return 1 + fi + base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION") + if [ -n "$CERT" ] && [ -n "$KEY" ]; then + base_command+=(--cert "$CERT" --key "$KEY") + fi + base_command+=(posix "$LOCAL_FOLDER") + export base_command + + local versity_result + start_versity_process "$3" || versity_result=$? + if [[ $versity_result -ne 0 ]]; then + echo "error starting versity process" + return 1 + fi + return 0 +} + +run_versity_app_s3() { + if [[ $# -ne 1 ]]; then + echo "run versity app w/s3 command requires process number" + return 1 + fi + base_command=("$VERSITY_EXE" --port=":7071" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY") + if [ -n "$CERT" ] && [ -n "$KEY" ]; then + base_command+=(--cert "$CERT" --key "$KEY") + fi + base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com) + export base_command + + local versity_result + start_versity_process "$1" || versity_result=$? + if [[ $versity_result -ne 0 ]]; then + echo "error starting versity process" + return 1 + fi + return 0 +} + +run_versity_app() { + if [[ $BACKEND == 'posix' ]]; then + run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$? + if [[ $result_one -ne 0 ]]; then + echo "error starting versity app" + return 1 + fi + elif [[ $BACKEND == 's3' ]]; then + run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$? + if [[ $result_one -ne 0 ]]; then + echo "error starting versity app" + return 1 + fi + run_versity_app_s3 "2" || result_two=$? + if [[ $result_two -ne 0 ]]; then + echo "error starting second versity app" + return 1 + fi + else + echo "unrecognized backend type $BACKEND" + return 1 + fi +} + +stop_single_process() { + if [[ $# -ne 1 ]]; then + echo "stop single process function requires process ID" + return 1 + fi + if ps -p "$1" > /dev/null; then + kill "$1" + wait "$1" || true + else + echo "Process with PID $1 does not exist." + fi } stop_versity() { if [ "$RUN_VERSITYGW" == "false" ]; then return fi - if [ -n "$versitygw_pid" ]; then - if ps -p "$versitygw_pid" > /dev/null; then - kill "$versitygw_pid" - wait "$versitygw_pid" || true - else - echo "Process with PID $versitygw_pid does not exist." + local result_one + local result_two + # shellcheck disable=SC2154 + stop_single_process "$versitygw_pid_1" || result_one=$? + if [[ $result_one -ne 0 ]]; then + echo "error stopping versity process" + fi + if [[ $BACKEND == 's3' ]]; then + # shellcheck disable=SC2154 + stop_single_process "$versitygw_pid_2" || result_two=$? + if [[ $result_two -ne 0 ]]; then + echo "error stopping versity process two" fi - else - echo "versitygw_pid is not set or empty." fi } \ No newline at end of file