From c9c05b4fbdbee3e003577ee0ac86430ee5359508 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Wed, 6 Mar 2024 11:49:28 -0300 Subject: [PATCH] test: mc - first command, static bucket, dockerfile, github-actions work --- .github/workflows/system.yml | 10 ++- Dockerfile_test_bats | 9 +++ tests/README.md | 9 ++- tests/run.sh | 77 ++++++++++++++++--- tests/run_all.sh | 25 +++++- tests/run_static.sh | 20 ----- tests/setup.sh | 85 ++++++--------------- tests/setup_mc.sh | 34 +++++++++ tests/{s3_bucket_tests.sh => test_aws.sh} | 0 tests/{posix_tests.sh => test_aws_posix.sh} | 0 tests/test_common.sh | 1 + tests/test_mc.sh | 9 +++ tests/{s3cmd_tests.sh => test_s3cmd.sh} | 0 tests/util.sh | 26 +++++-- tests/util_mc.sh | 24 ++++++ tests/versity.sh | 80 +++++++++++++++++++ 16 files changed, 306 insertions(+), 103 deletions(-) delete mode 100755 tests/run_static.sh create mode 100644 tests/setup_mc.sh rename tests/{s3_bucket_tests.sh => test_aws.sh} (100%) rename tests/{posix_tests.sh => test_aws_posix.sh} (100%) create mode 100755 tests/test_mc.sh rename tests/{s3cmd_tests.sh => test_s3cmd.sh} (100%) create mode 100644 tests/util_mc.sh create mode 100644 tests/versity.sh diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index e650030..dcb929b 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -33,11 +33,17 @@ jobs: run: | sudo apt-get install s3cmd + - name: Install mc + run: | + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc + chmod 755 /usr/local/bin/mc + - name: Build and run run: | make testbin - export AWS_ACCESS_KEY_ID=user - export AWS_SECRET_ACCESS_KEY=pass + echo $PATH + export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST + export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn export AWS_REGION=us-east-1 aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity diff --git a/Dockerfile_test_bats b/Dockerfile_test_bats index b7673e0..5d79911 100644 --- a/Dockerfile_test_bats +++ b/Dockerfile_test_bats @@ -19,8 +19,16 @@ RUN apt-get update && \ # Set working directory WORKDIR /tmp +# Install AWS cli RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install +# Install mc +RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \ + --create-dirs \ + -o /usr/local/minio-binaries/mc && \ +chmod -R 755 /usr/local/minio-binaries +ENV PATH="/usr/local/minio-binaries":${PATH} + # Download Go 1.21 (adjust the version and platform as needed) RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz @@ -40,6 +48,7 @@ RUN groupadd -r tester && useradd -r -g tester tester RUN mkdir /home/tester && chown tester:tester /home/tester ENV HOME=/home/tester +# install bats RUN git clone https://github.com/bats-core/bats-core.git && \ cd bats-core && \ ./install.sh /home/tester diff --git a/tests/README.md b/tests/README.md index 9f3c586..0398f12 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,7 +3,10 @@ ## Instructions - Running Locally 1. Build the `versitygw` binary. -2. Install the aws command-line interface if unavailable on your machine. Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). +2. Install the command-line interface(s) you want to test if unavailable on your machine. + * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). + * **s3cmd**: Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md). + * **mc**: Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html). 3. Install BATS. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html). 4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file. 5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example: @@ -22,8 +25,8 @@ openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048 openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 ``` -8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`. -9. In the root repo folder, run with `VERSITYGW_TEST_ENV= tests/run_all.sh`. +8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`. +9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV= tests/run.sh `. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV= tests/run_all.sh`. ## Instructions - Running With Docker diff --git a/tests/run.sh b/tests/run.sh index a153941..68a213d 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,16 +1,75 @@ #!/bin/bash +# Function to display help information +show_help() { + echo "Usage: $0 [option...]" + echo " -h, --help Display this help message and exit" + echo " -s, --static Don't remove buckets between tests" + echo " aws Run tests with aws cli" + echo " aws-posix Run posix tests with aws cli" + echo " s3cmd Run tests with s3cmd utility" + echo " mc Run tests with mc utility" +} + +handle_param() { + case $1 in + -h|--help) + show_help + exit 0 + ;; + -s|--static) + export RECREATE_BUCKETS=false + ;; + aws|aws-posix|s3cmd|mc) + set_command_type "$1" + ;; + *) # Handle unrecognized options or positional arguments + echo "Unrecognized option: $1" >&2 + exit 1 + ;; + esac +} + +set_command_type() { + if [[ -n $command_type ]]; then + echo "Error: command type already set" + exit 1 + fi + command_type=$1 + export command_type +} + +export RECREATE_BUCKETS=true +while [[ "$#" -gt 0 ]]; do + handle_param "$1" + shift # past argument or value +done + if [[ -z "$VERSITYGW_TEST_ENV" ]]; then echo "Error: VERSITYGW_TEST_ENV parameter must be set" exit 1 fi -export RECREATE_BUCKETS=true -if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then - exit 1 + +if [[ $RECREATE_BUCKETS == false ]]; then + ./tests/setup_static.sh || exit_code=$? + if [[ exit_code -ne 0 ]]; then + exit 1 + fi fi -if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then - exit 1 -fi -if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then - exit 1 -fi \ No newline at end of file + +case $command_type in + aws) + "$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$? + ;; + aws-posix) + "$HOME"/bin/bats ./tests/test_aws_posix.sh || exit_code=$? + ;; + s3cmd) + "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$? + ;; + mc) + "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$? + ;; +esac + +exit $exit_code diff --git a/tests/run_all.sh b/tests/run_all.sh index 74afc3c..61a9bbc 100755 --- a/tests/run_all.sh +++ b/tests/run_all.sh @@ -4,9 +4,28 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]]; then echo "Error: VERSITYGW_TEST_ENV parameter must be set" exit 1 fi -if ! ./tests/run.sh; then +if ! ./tests/run.sh aws; then exit 1 fi -if ! ./tests/run_static.sh; then +if ! ./tests/run.sh aws-posix; then exit 1 -fi \ No newline at end of file +fi +if ! ./tests/run.sh s3cmd; then + exit 1 +fi +if ! ./tests/run.sh mc; then + exit 1 +fi +if ! ./tests/run.sh -s aws; then + exit 1 +fi +if ! ./tests/run.sh -s aws-posix; then + exit 1 +fi +if ! ./tests/run.sh -s s3cmd; then + exit 1 +fi +if ! ./tests/run.sh -s mc; then + exit 1 +fi +exit 0 diff --git a/tests/run_static.sh b/tests/run_static.sh deleted file mode 100755 index c1df6aa..0000000 --- a/tests/run_static.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -if [[ -z "$VERSITYGW_TEST_ENV" ]]; then - echo "Error: VERSITYGW_TEST_ENV parameter must be set" - exit 1 -fi -result=0 -export RECREATE_BUCKETS=false -./tests/setup_static.sh -if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then - result=1 -fi -if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then - result=1 -fi -if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then - result=1 -fi -./tests/teardown_static.sh -exit $result \ No newline at end of file diff --git a/tests/setup.sh b/tests/setup.sh index 86b73b5..e4c28e0 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -1,73 +1,44 @@ #!/usr/bin/env bash +source ./tests/setup_mc.sh +source ./tests/versity.sh + # bats setup function setup() { - if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then - source tests/.secrets - else - echo "Warning: no secrets file found" - fi - if [ -z "$VERSITYGW_TEST_ENV" ]; then - if [ -r tests/.env ]; then - source tests/.env - else - echo "Warning: no .env file found in tests folder" - fi - else - # shellcheck source=./.env.default - source "$VERSITYGW_TEST_ENV" + start_versity || start_result=$? + if [[ $start_result -ne 0 ]]; then + echo "error starting versity executable" + return 1 fi - check_params - - base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE" - if [ -n "$CERT" ] && [ -n "$KEY" ]; then - base_command+=" --cert $CERT --key $KEY" + check_params || check_result=$? + if [[ $check_result -ne 0 ]]; then + echo "parameter check failed" + return 1 fi - base_command+=" $BACKEND $LOCAL_FOLDER &" - eval "$base_command" - versitygw_pid=$! S3CMD_OPTS=() S3CMD_OPTS+=(-c "$S3CMD_CONFIG") S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID") S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY") - export versitygw_pid \ - AWS_PROFILE \ - AWS_ENDPOINT_URL \ - LOCAL_FOLDER \ + + check_add_mc_alias || check_result=$? + if [[ $check_result -ne 0 ]]; then + echo "mc alias check/add failed" + return 1 + fi + + export AWS_PROFILE \ BUCKET_ONE_NAME \ BUCKET_TWO_NAME \ S3CMD_CONFIG \ - S3CMD_OPTS \ - RECREATE_BUCKETS + S3CMD_OPTS } -# make sure required environment variables are defined properly +# make sure required environment variables for tests are defined properly # return 0 for yes, 1 for no check_params() { - if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS access key set" - return 1 - elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then - echo "No AWS secret access key set" - return 1 - elif [ -z "$VERSITY_EXE" ]; then - echo "No versity executable location set" - return 1 - elif [ -z "$BACKEND" ]; then - echo "No backend parameter set (options: 'posix')" - return 1 - elif [ -z "$AWS_PROFILE" ]; then - echo "No AWS profile set" - return 1 - elif [ -z "$LOCAL_FOLDER" ]; then - echo "No local storage folder set" - return 1 - elif [ -z "$AWS_ENDPOINT_URL" ]; then - echo "No AWS endpoint URL set" - return 1 - elif [ -z "$BUCKET_ONE_NAME" ]; then + if [ -z "$BUCKET_ONE_NAME" ]; then echo "No bucket one name set" return 1 elif [ -z "$BUCKET_TWO_NAME" ]; then @@ -80,6 +51,7 @@ check_params() { echo "RECREATE_BUCKETS must be 'true' or 'false'" return 1 fi + return 0 } # fail a test @@ -91,14 +63,5 @@ fail() { # bats teardown function teardown() { - if [ -n "$versitygw_pid" ]; then - if ps -p "$versitygw_pid" > /dev/null; then - kill "$versitygw_pid" - wait "$versitygw_pid" || true - else - echo "Process with PID $versitygw_pid does not exist." - fi - else - echo "versitygw_pid is not set or empty." - fi + stop_versity } diff --git a/tests/setup_mc.sh b/tests/setup_mc.sh new file mode 100644 index 0000000..2e4d43d --- /dev/null +++ b/tests/setup_mc.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +check_for_alias() { + local alias_result + aliases=$(mc alias list) + if [[ $alias_result -ne 0 ]]; then + echo "error checking for aliases: $aliases" + return 2 + fi + while IFS= read -r line; do + if [[ $line =~ ^versity$ ]]; then + return 0 + fi + done <<< "$aliases" + return 1 +} + +check_add_mc_alias() { + check_for_alias || alias_result=$? + if [[ $alias_result -eq 2 ]]; then + echo "error checking for aliases" + return 1 + fi + if [[ $alias_result -eq 0 ]]; then + return 0 + fi + local set_result + error=$(mc alias set --insecure versity "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$? + if [[ $set_result -ne 0 ]]; then + echo "error setting alias: $error" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/s3_bucket_tests.sh b/tests/test_aws.sh similarity index 100% rename from tests/s3_bucket_tests.sh rename to tests/test_aws.sh diff --git a/tests/posix_tests.sh b/tests/test_aws_posix.sh similarity index 100% rename from tests/posix_tests.sh rename to tests/test_aws_posix.sh diff --git a/tests/test_common.sh b/tests/test_common.sh index beba2c6..6fc1226 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bats +source ./tests/util.sh source ./tests/util_file.sh # common test for creating, deleting buckets diff --git a/tests/test_mc.sh b/tests/test_mc.sh new file mode 100755 index 0000000..b16a0ff --- /dev/null +++ b/tests/test_mc.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bats + +source ./tests/test_common.sh +source ./tests/setup.sh + +# test mc bucket creation/deletion +@test "test_create_delete_bucket_mc" { + test_common_create_delete_bucket "mc" +} \ No newline at end of file diff --git a/tests/s3cmd_tests.sh b/tests/test_s3cmd.sh similarity index 100% rename from tests/s3cmd_tests.sh rename to tests/test_s3cmd.sh diff --git a/tests/util.sh b/tests/util.sh index 07ecdf6..8b9e859 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -1,17 +1,28 @@ #!/usr/bin/env bats +source ./tests/util_mc.sh + # create an AWS bucket # param: bucket name # return 0 for success, 1 for failure create_bucket() { - if [ $# -ne 1 ]; then - echo "create bucket missing bucket name" + if [ $# -ne 2 ]; then + echo "create bucket missing command type, bucket name" return 1 fi local exit_code=0 local error - error=$(aws --no-verify-ssl s3 mb s3://"$1" 2>&1) || exit_code=$? + if [[ $1 == "aws" ]]; then + error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? + elif [[ $1 == "s3cmd" ]]; then + error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$? + elif [[ $1 == "mc" ]]; then + error=$(mc --insecure mb versity/"$2" 2>&1) || exit_code=$? + else + echo "invalid command type $1" + return 1 + fi if [ $exit_code -ne 0 ]; then echo "error creating bucket: $error" return 1 @@ -57,6 +68,8 @@ delete_bucket_recursive() { error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?" elif [[ $1 == "s3cmd" ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?" + elif [[ $1 == "mc" ]]; then + error=$(delete_bucket_recursive_mc "$2") || exit_code="$?" else echo "invalid command type '$1'" return 1 @@ -88,6 +101,8 @@ delete_bucket_contents() { error=$(aws --no-verify-ssl s3 rm s3://"$2" --recursive 2>&1) || exit_code="$?" elif [[ $1 == "s3cmd" ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?" + elif [[ $1 == "mc" ]]; then + error=$(mc --insecure rm --force --recursive versity/"$2" 2>&1) || exit_code="$?" else echo "invalid command type $1" return 1 @@ -114,6 +129,8 @@ bucket_exists() { error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?" elif [[ $1 == 's3cmd' ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?" + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure ls versity) else echo "invalid command type: $1" return 2 @@ -184,7 +201,7 @@ setup_bucket() { return 1 fi local create_result - create_bucket "$2" || create_result=$? + create_bucket "$1" "$2" || create_result=$? if [[ $create_result -ne 0 ]]; then echo "Error creating bucket" return 1 @@ -287,7 +304,6 @@ delete_object() { if [[ $1 == 'aws' ]]; then error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - echo "delete object s3cmd" error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$? echo "$error" else diff --git a/tests/util_mc.sh b/tests/util_mc.sh new file mode 100644 index 0000000..ef734b3 --- /dev/null +++ b/tests/util_mc.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# use mc tool to delete bucket and contents +# params: bucket name +# return 0 for success, 1 for failure +delete_bucket_recursive_mc() { + if [[ $# -ne 1 ]]; then + echo "delete bucket recursive mc command requires bucket name" + return 1 + fi + local exit_code=0 + local error + error=$(mc --insecure rm --recursive --force versity/"$1" 2>&1) || exit_code="$?" + if [[ $exit_code -ne 0 ]]; then + echo "error deleting bucket contents: $error" + return 1 + fi + error=$(mc --insecure rb versity/"$1" 2>&1) || exit_code="$?" + if [[ $exit_code -ne 0 ]]; then + echo "error deleting bucket: $error" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/versity.sh b/tests/versity.sh new file mode 100644 index 0000000..07e02fc --- /dev/null +++ b/tests/versity.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +check_exe_params() { + if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS access key set" + return 1 + elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "No AWS secret access key set" + return 1 + elif [ -z "$VERSITY_EXE" ]; then + echo "No versity executable location set" + return 1 + elif [ -z "$BACKEND" ]; then + echo "No backend parameter set (options: 'posix')" + return 1 + elif [ -z "$AWS_PROFILE" ]; then + echo "No AWS profile set" + return 1 + elif [ -z "$LOCAL_FOLDER" ]; then + echo "No local storage folder set" + return 1 + elif [ -z "$AWS_ENDPOINT_URL" ]; then + echo "No AWS endpoint URL set" + return 1 + fi +} + +start_versity() { + if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then + source tests/.secrets + else + echo "Warning: no secrets file found" + fi + if [ -z "$VERSITYGW_TEST_ENV" ]; then + if [ -r tests/.env ]; then + source tests/.env + else + echo "Warning: no .env file found in tests folder" + fi + else + # shellcheck source=./.env.default + source "$VERSITYGW_TEST_ENV" + fi + + check_exe_params || check_result=$? + if [[ $check_result -ne 0 ]]; then + echo "error checking for parameters" + return 1 + fi + + base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE" + if [ -n "$CERT" ] && [ -n "$KEY" ]; then + base_command+=" --cert $CERT --key $KEY" + fi + base_command+=" $BACKEND $LOCAL_FOLDER &" + eval "$base_command" + + versitygw_pid=$! + export versitygw_pid \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + VERSITY_EXE \ + BACKEND \ + AWS_PROFILE \ + LOCAL_FOLDER \ + AWS_ENDPOINT_URL +} + +stop_versity() { + if [ -n "$versitygw_pid" ]; then + if ps -p "$versitygw_pid" > /dev/null; then + kill "$versitygw_pid" + wait "$versitygw_pid" || true + else + echo "Process with PID $versitygw_pid does not exist." + fi + else + echo "versitygw_pid is not set or empty." + fi +} \ No newline at end of file