Merge pull request #605 from versity/test_cmdline_log_matrix

Test cmdline log matrix
2026-01-04 11:03:57 +00:00 · 2024-06-04 11:53:46 -07:00
parent 1383a27dea 282ef71867
commit 20cef53fd8
9 changed files with 104 additions and 21 deletions
--- a/.github/workflows/system.yml
+++ b/.github/workflows/system.yml
@@ -4,6 +4,43 @@ jobs:
  build:
    name: RunTests
    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - set: 1
+            LOCAL_FOLDER: /tmp/gw1
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+            USERS_FOLDER: /tmp/iam1
+            AWS_ENDPOINT_URL: https://127.0.0.1:7070
+            RUN_SET: "s3cmd"
+            PORT: 7070
+          - set: 2
+            LOCAL_FOLDER: /tmp/gw2
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+            USERS_FOLDER: /tmp/iam2
+            AWS_ENDPOINT_URL: https://127.0.0.1:7071
+            RUN_SET: "s3"
+            PORT: 7071
+          - set: 3
+            LOCAL_FOLDER: /tmp/gw3
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+            USERS_FOLDER: /tmp/iam3
+            AWS_ENDPOINT_URL: https://127.0.0.1:7072
+            RUN_SET: "s3api"
+            PORT: 7072
+          - set: 4
+            LOCAL_FOLDER: /tmp/gw4
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            USERS_FOLDER: /tmp/iam4
+            AWS_ENDPOINT_URL: https://127.0.0.1:7073
+            RUN_SET: "mc"
+            PORT: 7073
+
    steps:
      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
@@ -39,6 +76,25 @@ jobs:
          chmod 755 /usr/local/bin/mc

      - name: Build and run, posix backend
+        env:
+          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
+          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
+          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
+          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
+          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
+          RUN_SET: ${{ matrix.RUN_SET }}
+          PORT: ${{ matrix.PORT }}
+          AWS_PROFILE: versity
+          VERSITY_EXE: ${{ github.workspace }}/versitygw
+          RUN_VERSITYGW: true
+          BACKEND: posix
+          RECREATE_BUCKETS: true
+          CERT: ${{ github.workspace }}/cert.pem
+          KEY: ${{ github.workspace }}/versitygw.pem
+          S3CMD_CONFIG: tests/s3cfg.local.default
+          MC_ALIAS: versity
+          LOG_LEVEL: 4
+          GOCOVERDIR: ${{ github.workspace }}/cover
        run: |
          make testbin
          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -47,12 +103,12 @@ jobs:
          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
          aws configure set aws_region $AWS_REGION --profile versity
-          mkdir /tmp/gw
+          mkdir $LOCAL_FOLDER
          export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir cover iam
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir $GOCOVERDIR $USERS_FOLDER
+          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET

      #- name: Build and run, s3 backend
      #  run: |
--- a/tests/commands/create_bucket.sh
+++ b/tests/commands/create_bucket.sh
@@ -11,11 +11,13 @@ create_bucket() {

  local exit_code=0
  local error
+  log 6 "create bucket"
  if [[ $1 == 's3' ]]; then
    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
  elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
    error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
  elif [[ $1 == "s3cmd" ]]; then
+    log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
  elif [[ $1 == "mc" ]]; then
    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
--- a/tests/logger.sh
+++ b/tests/logger.sh
@@ -21,14 +21,11 @@ log() {
    6) log_level="TRACE";;
    *) echo "invalid log level $1"; return 1
  esac
-  if [[ "$2" == *"secret_key"* ]]; then
-    log_mask $log_level "$2"
+  if [[ "$2" == *"--secret"* ]]; then
+    log_mask "$log_level" "$2"
    return 0
  fi
-  echo "$log_level $2"
-  if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$log_level $2" >> "$TEST_LOG_FILE"
-  fi
+  log_message "$log_level" "$2"
 }

 log_mask() {
@@ -40,16 +37,33 @@ log_mask() {

  IFS=' ' read -r -a array <<< "$2"

+  mask_next=false
  for arg in "${array[@]}"; do
-    if [[ "$arg" == --secret_key=* ]]; then
+    if [[ $mask_next == true ]]; then
+      masked_args+=("********")
+      mask_next=false
+    elif [[ "$arg" == --secret_key=* ]]; then
      masked_args+=("--secret_key=********")
+    elif [[ "$arg" == --secret=* ]]; then
+      masked_args+=("--secret=********")
    else
+      if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]]; then
+        mask_next=true
+      fi
      masked_args+=("$arg")
    fi
  done
+  log_message "$log_level" "${masked_args[*]}"
+}

-  echo "$log_level ${masked_args[*]}"
+log_message() {
+  if [[ $# -ne 2 ]]; then
+    echo "log message requires level, message"
+    return 1
+  fi
+  now="$(date "+%Y-%m-%d %H:%M:%S")"
+  echo "$now $1 $2"
  if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$log_level ${masked_args[*]}" >> "$TEST_LOG_FILE"
+    echo "$now $1 $2" >> "$TEST_LOG_FILE"
  fi
 }
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -19,7 +19,7 @@ handle_param() {
      -s|--static)
          export RECREATE_BUCKETS=false
          ;;
-      s3|s3api|aws|s3cmd|mc|user)
+      s3|s3api|aws|s3cmd|mc)
          set_command_type "$1"
          ;;
      *) # Handle unrecognized options or positional arguments
@@ -51,8 +51,8 @@ while [[ "$#" -gt 0 ]]; do
  shift # past argument or value
 done

-if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
-  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true"
  exit 1
 fi

--- a/tests/run_all.sh
+++ b/tests/run_all.sh
@@ -1,7 +1,7 @@
 #!/bin/bash

-if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
-  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true"
  exit 1
 fi

--- a/tests/test_aws.sh
+++ b/tests/test_aws.sh
@@ -656,9 +656,14 @@ legal_hold_retention_setup() {
 #  [[ $upload_result -eq 1 ]] || fail "multipart upload with overly large range should have failed"
 #  [[ $upload_part_copy_error == *"Range specified is not valid"* ]] || fail "unexpected error: $upload_part_copy_error"
 #
-#  multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-16000000" || local upload_two_result=$?
+#  range_max=$((5*1024*1024-1))
+#  multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-$range_max" || local upload_two_result=$?
 #  [[ $upload_two_result -eq 0 ]] || fail "range should be valid"
 #
+#  get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error retrieving object after upload"
+#  object_size=$(stat -f%z "$test_file_folder/$bucket_file-copy")
+#  [[ object_size -eq $((range_max*4+4)) ]] || fail "object size mismatch ($object_size, $((range_max*4+4)))"
+#
 #  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
 #  delete_test_files $bucket_file
 #}
--- a/tests/util.sh
+++ b/tests/util.sh
@@ -208,6 +208,7 @@ setup_bucket() {
    return 1
  fi
  local create_result
+  log 5 "util.setup_bucket: command type: $1, bucket name: $2"
  create_bucket "$1" "$2" || create_result=$?
  if [[ $create_result -ne 0 ]]; then
    log 2 "Error creating bucket"
--- a/tests/util_users.sh
+++ b/tests/util_users.sh
@@ -76,6 +76,7 @@ delete_user() {
      echo "delete user command requires user ID"
      return 1
    fi
+    log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1"
    error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1") || local delete_result=$?


--- a/tests/versity.sh
+++ b/tests/versity.sh
@@ -67,7 +67,7 @@ start_versity() {
    else
      echo "Warning: no .env file found in tests folder"
    fi
-  else
+  elif [[ $BYPASS_ENV_FILE != "true" ]]; then
    # shellcheck source=./tests/.env.default
    source "$VERSITYGW_TEST_ENV"
  fi
@@ -106,6 +106,7 @@ start_versity_process() {
    return 1
  fi
  base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1")
+  log 5 "versity command: ${base_command[*]}"
  ("${base_command[@]}") &
  # shellcheck disable=SC2181
  if [[ $? -ne 0 ]]; then
@@ -136,6 +137,9 @@ run_versity_app_posix() {
  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
    base_command+=(--cert "$CERT" --key "$KEY")
  fi
+  if [ -n "$PORT" ]; then
+    base_command+=(--port ":$PORT")
+  fi
  base_command+=(posix "$LOCAL_FOLDER")
  export base_command