From 282ef718675d127f64d0f24dc6807f0896eb2992 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Mon, 3 Jun 2024 18:46:46 -0300 Subject: [PATCH] test: log updates, matrix testing to speed things up --- .github/workflows/system.yml | 66 ++++++++++++++++++++++++++++++--- tests/commands/create_bucket.sh | 2 + tests/logger.sh | 32 +++++++++++----- tests/run.sh | 6 +-- tests/run_all.sh | 4 +- tests/test_aws.sh | 7 +++- tests/util.sh | 1 + tests/util_users.sh | 1 + tests/versity.sh | 6 ++- 9 files changed, 104 insertions(+), 21 deletions(-) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index e391ff0..feec058 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -4,6 +4,43 @@ jobs: build: name: RunTests runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - set: 1 + LOCAL_FOLDER: /tmp/gw1 + BUCKET_ONE_NAME: versity-gwtest-bucket-one-1 + BUCKET_TWO_NAME: versity-gwtest-bucket-two-1 + USERS_FOLDER: /tmp/iam1 + AWS_ENDPOINT_URL: https://127.0.0.1:7070 + RUN_SET: "s3cmd" + PORT: 7070 + - set: 2 + LOCAL_FOLDER: /tmp/gw2 + BUCKET_ONE_NAME: versity-gwtest-bucket-one-2 + BUCKET_TWO_NAME: versity-gwtest-bucket-two-2 + USERS_FOLDER: /tmp/iam2 + AWS_ENDPOINT_URL: https://127.0.0.1:7071 + RUN_SET: "s3" + PORT: 7071 + - set: 3 + LOCAL_FOLDER: /tmp/gw3 + BUCKET_ONE_NAME: versity-gwtest-bucket-one-3 + BUCKET_TWO_NAME: versity-gwtest-bucket-two-3 + USERS_FOLDER: /tmp/iam3 + AWS_ENDPOINT_URL: https://127.0.0.1:7072 + RUN_SET: "s3api" + PORT: 7072 + - set: 4 + LOCAL_FOLDER: /tmp/gw4 + BUCKET_ONE_NAME: versity-gwtest-bucket-one-4 + BUCKET_TWO_NAME: versity-gwtest-bucket-two-4 + USERS_FOLDER: /tmp/iam4 + AWS_ENDPOINT_URL: https://127.0.0.1:7073 + RUN_SET: "mc" + PORT: 7073 + steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -39,6 +76,25 @@ jobs: chmod 755 /usr/local/bin/mc - name: Build and run, posix backend + env: + LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }} + BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }} + BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }} + USERS_FOLDER: ${{ matrix.USERS_FOLDER }} + AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }} + RUN_SET: ${{ matrix.RUN_SET }} + PORT: ${{ matrix.PORT }} + AWS_PROFILE: versity + VERSITY_EXE: ${{ github.workspace }}/versitygw + RUN_VERSITYGW: true + BACKEND: posix + RECREATE_BUCKETS: true + CERT: ${{ github.workspace }}/cert.pem + KEY: ${{ github.workspace }}/versitygw.pem + S3CMD_CONFIG: tests/s3cfg.local.default + MC_ALIAS: versity + LOG_LEVEL: 4 + GOCOVERDIR: ${{ github.workspace }}/cover run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST @@ -47,12 +103,12 @@ jobs: aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity aws configure set aws_region $AWS_REGION --profile versity - mkdir /tmp/gw + mkdir $LOCAL_FOLDER export WORKSPACE=$GITHUB_WORKSPACE - openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048 - openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com" - mkdir cover iam - VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh + openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048 + openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com" + mkdir $GOCOVERDIR $USERS_FOLDER + BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET #- name: Build and run, s3 backend # run: | diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index 9838c6b..54a0b63 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -11,11 +11,13 @@ create_bucket() { local exit_code=0 local error + log 6 "create bucket" if [[ $1 == 's3' ]]; then error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then + log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2" error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "mc" ]]; then error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? diff --git a/tests/logger.sh b/tests/logger.sh index 43fa8fe..f9c616d 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -21,14 +21,11 @@ log() { 6) log_level="TRACE";; *) echo "invalid log level $1"; return 1 esac - if [[ "$2" == *"secret_key"* ]]; then - log_mask $log_level "$2" + if [[ "$2" == *"--secret"* ]]; then + log_mask "$log_level" "$2" return 0 fi - echo "$log_level $2" - if [[ -n "$TEST_LOG_FILE" ]]; then - echo "$log_level $2" >> "$TEST_LOG_FILE" - fi + log_message "$log_level" "$2" } log_mask() { @@ -40,16 +37,33 @@ log_mask() { IFS=' ' read -r -a array <<< "$2" + mask_next=false for arg in "${array[@]}"; do - if [[ "$arg" == --secret_key=* ]]; then + if [[ $mask_next == true ]]; then + masked_args+=("********") + mask_next=false + elif [[ "$arg" == --secret_key=* ]]; then masked_args+=("--secret_key=********") + elif [[ "$arg" == --secret=* ]]; then + masked_args+=("--secret=********") else + if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]]; then + mask_next=true + fi masked_args+=("$arg") fi done + log_message "$log_level" "${masked_args[*]}" +} - echo "$log_level ${masked_args[*]}" +log_message() { + if [[ $# -ne 2 ]]; then + echo "log message requires level, message" + return 1 + fi + now="$(date "+%Y-%m-%d %H:%M:%S")" + echo "$now $1 $2" if [[ -n "$TEST_LOG_FILE" ]]; then - echo "$log_level ${masked_args[*]}" >> "$TEST_LOG_FILE" + echo "$now $1 $2" >> "$TEST_LOG_FILE" fi } diff --git a/tests/run.sh b/tests/run.sh index a95c0c2..542d51c 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -19,7 +19,7 @@ handle_param() { -s|--static) export RECREATE_BUCKETS=false ;; - s3|s3api|aws|s3cmd|mc|user) + s3|s3api|aws|s3cmd|mc) set_command_type "$1" ;; *) # Handle unrecognized options or positional arguments @@ -51,8 +51,8 @@ while [[ "$#" -gt 0 ]]; do shift # past argument or value done -if [[ -z "$VERSITYGW_TEST_ENV" ]]; then - echo "Error: VERSITYGW_TEST_ENV parameter must be set" +if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then + echo "Error: VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true" exit 1 fi diff --git a/tests/run_all.sh b/tests/run_all.sh index 2150012..7815cb2 100755 --- a/tests/run_all.sh +++ b/tests/run_all.sh @@ -1,7 +1,7 @@ #!/bin/bash -if [[ -z "$VERSITYGW_TEST_ENV" ]]; then - echo "Error: VERSITYGW_TEST_ENV parameter must be set" +if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then + echo "Error: VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true" exit 1 fi diff --git a/tests/test_aws.sh b/tests/test_aws.sh index d95ca7d..8ca5f66 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -656,9 +656,14 @@ legal_hold_retention_setup() { # [[ $upload_result -eq 1 ]] || fail "multipart upload with overly large range should have failed" # [[ $upload_part_copy_error == *"Range specified is not valid"* ]] || fail "unexpected error: $upload_part_copy_error" # -# multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-16000000" || local upload_two_result=$? +# range_max=$((5*1024*1024-1)) +# multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-$range_max" || local upload_two_result=$? # [[ $upload_two_result -eq 0 ]] || fail "range should be valid" # +# get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error retrieving object after upload" +# object_size=$(stat -f%z "$test_file_folder/$bucket_file-copy") +# [[ object_size -eq $((range_max*4+4)) ]] || fail "object size mismatch ($object_size, $((range_max*4+4)))" +# # delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" # delete_test_files $bucket_file #} diff --git a/tests/util.sh b/tests/util.sh index 3c98d74..8079da7 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -208,6 +208,7 @@ setup_bucket() { return 1 fi local create_result + log 5 "util.setup_bucket: command type: $1, bucket name: $2" create_bucket "$1" "$2" || create_result=$? if [[ $create_result -ne 0 ]]; then log 2 "Error creating bucket" diff --git a/tests/util_users.sh b/tests/util_users.sh index 01568d8..3ab59db 100644 --- a/tests/util_users.sh +++ b/tests/util_users.sh @@ -76,6 +76,7 @@ delete_user() { echo "delete user command requires user ID" return 1 fi + log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1" error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1") || local delete_result=$? diff --git a/tests/versity.sh b/tests/versity.sh index b5bf246..f361cd4 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -67,7 +67,7 @@ start_versity() { else echo "Warning: no .env file found in tests folder" fi - else + elif [[ $BYPASS_ENV_FILE != "true" ]]; then # shellcheck source=./tests/.env.default source "$VERSITYGW_TEST_ENV" fi @@ -106,6 +106,7 @@ start_versity_process() { return 1 fi base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1") + log 5 "versity command: ${base_command[*]}" ("${base_command[@]}") & # shellcheck disable=SC2181 if [[ $? -ne 0 ]]; then @@ -136,6 +137,9 @@ run_versity_app_posix() { if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi + if [ -n "$PORT" ]; then + base_command+=(--port ":$PORT") + fi base_command+=(posix "$LOCAL_FOLDER") export base_command