diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index b46314a..ec42d79 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -104,6 +104,7 @@ jobs: AUTOGENERATE_USERS: true USER_AUTOGENERATION_PREFIX: github-actions-test- AWS_REGION: ${{ matrix.AWS_REGION }} + COVERAGE_LOG: coverage.log run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST @@ -124,6 +125,7 @@ jobs: BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh fi BYPASS_ENV_FILE=true $HOME/bin/bats ${{ github.workspace }}/$RUN_SET + cat $COVERAGE_LOG - name: Time report run: | diff --git a/tests/commands/command.sh b/tests/commands/command.sh index 888430f..064d745 100644 --- a/tests/commands/command.sh +++ b/tests/commands/command.sh @@ -28,6 +28,7 @@ send_command() { fi # shellcheck disable=SC2154 echo "${masked_args[*]}" >> "$COMMAND_LOG" + log 5 "masked args: ${masked_args[*]}" record_command_v2 "${masked_args[*]}" fi local command_result=0 diff --git a/tests/drivers/openssl.sh b/tests/drivers/openssl.sh index 45698e4..fbe7571 100644 --- a/tests/drivers/openssl.sh +++ b/tests/drivers/openssl.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/drivers/xml.sh +source ./tests/report.sh write_openssl_command_to_command_log() { if ! check_param_count_v2 "command file" 1 $#; then @@ -54,6 +55,7 @@ send_via_openssl() { if [ -n "$COMMAND_LOG" ]; then write_openssl_command_to_command_log "$1" fi + parse_openssl_command "$1" if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then log 2 "error sending openssl command: $result" return 1 diff --git a/tests/env.sh b/tests/env.sh index 4f4ed57..f517767 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -140,6 +140,9 @@ check_universal_vars() { if [ -n "$MAX_FILE_DOWNLOAD_CHUNK_SIZE" ]; then export MAX_FILE_DOWNLOAD_CHUNK_SIZE fi + if [ -n "$COVERAGE_LOG" ]; then + export COVERAGE_LOG + fi check_aws_vars diff --git a/tests/report.sh b/tests/report.sh index f76930b..6b1603c 100644 --- a/tests/report.sh +++ b/tests/report.sh @@ -126,10 +126,6 @@ get_curl_method() { } parse_path_and_get_route() { - -} - -get_route() { if ! check_param_count_v2 "string" 1 $#; then return 1 fi @@ -149,37 +145,39 @@ get_route() { path="${path#/}" path="${path%/}" - # Split path on '/' - IFS='/' read -r -a parts <<< "$path" + get_route "$path" + return 0 +} - if [[ -z "$path" ]]; then +get_route() { + if ! check_param_count_v2 "string" 1 $#; then + return 1 + fi + + if [ "$1" == '/' ]; then + echo "MAIN" + return 0 + fi + + # Split path on '/' + IFS='/' read -r -a parts <<< "$1" + + if [[ -z "$1" ]]; then echo "MAIN" elif [[ "${#parts[@]}" -eq 1 ]]; then echo "BUCKET" else - echo "FILE" + echo "OBJECT" fi return 0 } get_query() { - if ! check_param_count_v2 "string" 1 $#; then - return 1 - fi - url="$(echo "$1" | grep -oE 'https?://[^" ]+' | head -n 1)" - - # Must look like a URL - if [ -z "$url" ]; then - echo "" - return 0 - fi - # Extract query string (everything after '?') local query - query="${url#*\?}" - + query="${1#*\?}" # No query present - if [[ "$query" == "$url" ]]; then + if [[ "$query" == "$1" ]]; then echo "" return 0 fi @@ -203,6 +201,40 @@ get_query() { echo "${keys[*]}" } +check_for_copy_source() { + if ! check_param_count_v2 "'OPENSSL' or 'CURL', string or file" 2 $#; then + return 2 + fi + if [ "$1" == "CURL" ]; then + if [[ "$2" == *"x-amz-copy-source"* ]]; then + return 0 + fi + return 1 + elif [ "$1" == "OPENSSL" ]; then + if grep -qi 'x-amz-copy-source' "$2"; then + return 0 + fi + return 1 + fi + log 2 "invalid type param: $1" + return 2 +} + +parse_path_and_get_query() { + if ! check_param_count_v2 "string" 1 $#; then + return 1 + fi + url="$(echo "$1" | grep -oE 'https?://[^" ]+' | head -n 1)" + + # Must look like a URL + if [ -z "$url" ]; then + echo "" + return 0 + fi + + get_query "$url" +} + parse_curl_rest_command() { if ! check_param_count_v2 "command string" 1 $#; then return 1 @@ -211,15 +243,20 @@ parse_curl_rest_command() { echo "error retrieving method: $method" return 1 fi - if ! route=$(get_route "$1" 2>&1); then + if ! route=$(parse_path_and_get_route "$1" 2>&1); then echo "error retrieving route: $route" return 1 fi - if ! query=$(get_query "$1" 2>&1); then + if ! query=$(parse_path_and_get_query "$1" 2>&1); then echo "error retrieving query: $query" return 1 fi - echo "$method $route $query" + output_string="$method $route $query" + if [[ "$output_string" == "PUT OBJECT"* ]] && check_for_copy_source "CURL" "$1"; then + output_string+=" x-amz-copy-source" + fi + log 5 "output string: $output_string" + echo "$output_string" return 0 } @@ -227,30 +264,42 @@ get_openssl_method_route_queries() { if ! check_param_count_v2 "command file" 1 $#; then return 1 fi - top_line=$(head -n 1 "$1") method=$(awk 'NR==1{print $1}' "$1") - route=$(get_route "$top_line") - query=$(get_query "$top_line") + + route_string=$(awk 'NR==1{print $2}' "$1") + route=$(get_route "$route_string") + query=$(get_query "$route_string") echo "$method $route $query" return 0 } +write_to_coverage_log() { + if ! check_param_count_v2 "string" 1 $#; then + return 1 + fi + echo "$1" >> "$COVERAGE_LOG" + sort "$COVERAGE_LOG" | uniq > "${COVERAGE_LOG}.tmp" + mv "${COVERAGE_LOG}.tmp" "$COVERAGE_LOG" +} + parse_openssl_command() { + if [ -z "$COVERAGE_LOG" ]; then + return 0 + fi if ! check_param_count_v2 "command file" 1 $#; then return 1 fi - -} - -get_client_type() { - if [[ "$1" == *" curl "* ]] || [[ "$1" == "curl "* ]]; then - echo "CURL" - return 0 - elif [[ "$1" == *" s3api "* ]] || [[ "$1" == "s3api "* ]]; then - echo "S3API" - return 0 + if ! command_info=$(get_openssl_method_route_queries "$1" 2>&1); then + log 2 "error getting command info: $command_info" + return 1 fi - echo "UNKNOWN" + if [[ "$command_info" == "PUT OBJECT"* ]] && check_for_copy_source "OPENSSL" "$1"; then + command_info+=" x-amz-copy-source" + fi + if ! write_to_coverage_log "$command_info"; then + return 1 + fi + return 0 } parse_command_info() { @@ -267,43 +316,23 @@ parse_command_info() { fi } -check_and_create_database_v2() { - # Define SQL commands to create a table - SQL_CREATE_TABLE="CREATE TABLE IF NOT EXISTS entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - command TEXT UNIQUE NOT NULL, - count INTEGER DEFAULT 1 - );" - -# Execute the SQL commands to create the database and table -sqlite3 "coverage.db" <&1); then - log 2 "error creating database: $db_result" - return 1 - fi + log 5 "parsing command '$1'" if ! parse_command_info "$1"; then + log 2 "error parsing command info" return 1 fi if [ "$command_info" == "NONE" ]; then return 0 fi - echo "$command_info" >> "commandInfo.txt" - cat "commandInfo.txt" | sort | uniq > "commandInfo.txt.tmp" - mv "commandInfo.txt.tmp" "commandInfo.txt" - if ! error=$(sqlite3 "coverage.db" "INSERT INTO entries (command, count) VALUES(\"$command_info\", 1) ON CONFLICT(command) DO UPDATE SET count = count + 1" 2>&1); then - log 2 "error in sqlite statement: $error" + if ! write_to_coverage_log "$command_info"; then + return 1 fi } diff --git a/tests/test_report.sh b/tests/test_report.sh index d3fc036..8bf67ef 100755 --- a/tests/test_report.sh +++ b/tests/test_report.sh @@ -33,11 +33,11 @@ source ./tests/report.sh @test "reporting - parse curl route" { tests=("http://localhost:7070/bucket_name" "http://localhost:7070/bucket_name/file_name" "http://localhost:7070/" "") - expected_results=("BUCKET" "FILE" "MAIN" "UNKNOWN") + expected_results=("BUCKET" "OBJECT" "MAIN" "UNKNOWN") for ((i=0; i<${#tests[@]}; i++)); do echo "test: ${tests[$i]}, expected result: ${expected_results[$i]}" - run get_curl_route "${tests[$i]}" + run parse_path_and_get_route "${tests[$i]}" assert_output "${expected_results[$i]}" done } @@ -53,17 +53,6 @@ source ./tests/report.sh done } -@test "reporting - get client type" { - tests=("curl -iks https://localhost:7070/versity-gwtest-bucket-one-1-20260127113351?location= -H Authorization: AWS4-HMAC-SHA256 Credential=AKIA6****/20260127/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=68c0b96180a5791be8a10335c10d302d31d358c4bc6028aec94faf502f3a185e -H host: localhost:7070 -H x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -H x-amz-date: 20260127T143355Z" \ - "aws --no-verify-ssl s3api create-bucket --bucket versity-gwtest-bucket-one-1-20260127113351 --object-lock-enabled-for-bucket" "") - expected_results=("CURL" "S3API" "UNKNOWN") - - for ((i=0; i<${#tests[@]}; i++)); do - run get_client_type "${tests[$i]}" - assert_output "${expected_results[$i]}" - done -} - @test "reporting - parse curl rest command" { tests=("curl -iks https://localhost:7070/versity-gwtest-bucket-one-1-20260127113351?location= -H Authorization: AWS4-HMAC-SHA256 Credential=AKIA6****/20260127/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=68c0b96180a5791be8a10335c10d302d31d358c4bc6028aec94faf502f3a185e -H host: localhost:7070 -H x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -H x-amz-date: 20260127T143355Z") expected_command=("GET BUCKET location") @@ -76,9 +65,12 @@ source ./tests/report.sh @test "openssl - get method, route, and queries" { tests=("GET / HTTP/1.1 + Authorization: AWS4-HMAC-SHA256 Credential=AKIAQJVWFRZQNI6LF3W7/20250911/us-east-1/s3/aws4_request,SignedHeaders=x-amz-content-sha256;x-amz-date,Signature=86ffbe2317caddcac569b25aa9b8e8db4a613a639b2a402cf4a9dc0e975ba997 + x-amz-content-sha256:UNSIGNED-PAYLOAD" + "PUT /bucket/file?prefix=dummy HTTP/1.1 Authorization: AWS4-HMAC-SHA256 Credential=AKIAQJVWFRZQNI6LF3W7/20250911/us-east-1/s3/aws4_request,SignedHeaders=x-amz-content-sha256;x-amz-date,Signature=86ffbe2317caddcac569b25aa9b8e8db4a613a639b2a402cf4a9dc0e975ba997 x-amz-content-sha256:UNSIGNED-PAYLOAD") - expected_output=("GET MAIN ") + expected_output=("GET MAIN " "PUT OBJECT prefix") for ((i=0; i<${#tests[@]}; i++)); do file_name="$TMPDIR/openssl-$(uuidgen)" @@ -86,4 +78,35 @@ source ./tests/report.sh run get_openssl_method_route_queries "$file_name" assert_output "${expected_output[$i]}" done +} + +@test "report - check for copy header value" { + test_clients=("OPENSSL" "OPENSSL" "CURL" "CURL" "CUR") + test_data=("GET / HTTP/1.1 + Authorization: AWS4-HMAC-SHA256 Credential=AKIAQJVWFRZQNI6LF3W7/20250911/us-east-1/s3/aws4_request,SignedHeaders=x-amz-content-sha256;x-amz-date,Signature=86ffbe2317caddcac569b25aa9b8e8db4a613a639b2a402cf4a9dc0e975ba997 + x-amz-content-sha256:UNSIGNED-PAYLOAD" + "PUT /bucket/file?prefix=dummy HTTP/1.1 + Authorization: AWS4-HMAC-SHA256 Credential=AKIAQJVWFRZQNI6LF3W7/20250911/us-east-1/s3/aws4_request,SignedHeaders=x-amz-content-sha256;x-amz-date,Signature=86ffbe2317caddcac569b25aa9b8e8db4a613a639b2a402cf4a9dc0e975ba997 + x-amz-copy-source:something" + "curl -ks -w %{http_code} -X PUT https://localhost:7070/versity-gwtest-bucket-one-1-20260129133816/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29-copy -H Authorization: AWS4-HMAC-SHA256 Credential=AKIA6****/20260129/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date,Signature=59091238ab6f297fa79201c90c2e77707177942ef1ba1c78ba31ec735f109477 -H host: localhost:7070 -H x-amz-content-sha256: UNSIGNED-PAYLOAD -H x-amz-copy-source: versity-gwtest-bucket-one-1-20260129133816/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29 -H x-amz-date: 20260129T163817Z -o /Users/lukemccrone/devel/versitygw/versity-gwtest-files/result.txt -T /Users/lukemccrone/devel/versitygw/versity-gwtest-files/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29" + "curl -ks -w %{http_code} -X PUT https://localhost:7070/versity-gwtest-bucket-one-1-20260129133816/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29 -H Authorization: AWS4-HMAC-SHA256 Credential=AKIA6****/20260129/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3f0d83d829b502ed3e5d7c66de109151df10ce76e866def1ccdd46e48bde66ca -H host: localhost:7070 -H x-amz-content-sha256: 778e1535066c2e3def76239d1326c019f5548480d68fd13a1d68942b1eb1b6c5 -H x-amz-date: 20260129T163817Z -T /Users/lukemccrone/devel/versitygw/versity-gwtest-files/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29 -o /Users/lukemccrone/devel/versitygw/versity-gwtest-files/output.txt" + "curl -ks -w %{http_code} -X PUT https://localhost:7070/versity-gwtest-bucket-one-1-20260129133816/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29 -H Authorization: AWS4-HMAC-SHA256 Credential=AKIA6****/20260129/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3f0d83d829b502ed3e5d7c66de109151df10ce76e866def1ccdd46e48bde66ca -H host: localhost:7070 -H x-amz-content-sha256: 778e1535066c2e3def76239d1326c019f5548480d68fd13a1d68942b1eb1b6c5 -H x-amz-date: 20260129T163817Z -T /Users/lukemccrone/devel/versitygw/versity-gwtest-files/test-file-ED302D34-1A3F-47D5-B3B7-78DF01943C29 -o /Users/lukemccrone/devel/versitygw/versity-gwtest-files/output.txt") + expected_responses=(1 0 0 1 2) + + for ((i=0; i<${#test_clients[@]}; i++)); do + echo "test $i" + if [ "${test_clients[$i]}" == "OPENSSL" ]; then + file_name="$TMPDIR/data-$(uuidgen)" + echo "${test_data[$i]}" > "$file_name" + data_param=$file_name + else + data_param=${test_data[$i]} + fi + run check_for_copy_source "${test_clients[$i]}" "$data_param" + if [ "${expected_responses[$i]}" -eq 0 ]; then + assert_success + else + assert_failure "${expected_responses[$i]}" + fi + done } \ No newline at end of file diff --git a/tests/test_rest_copy_object.sh b/tests/test_rest_copy_object.sh index b69b9b0..b437b4a 100755 --- a/tests/test_rest_copy_object.sh +++ b/tests/test_rest_copy_object.sh @@ -57,4 +57,6 @@ source ./tests/setup.sh run copy_object_copy_source_and_payload "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file" assert_success + + return 1 }