mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-08 04:35:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: mc - first command, static bucket, dockerfile, github-actions work

Browse Source
This commit is contained in:
Luke McCrone
2024-03-06 11:49:28 -03:00
parent 29f87d5444
commit c9c05b4fbd
16 changed files with 306 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/system.yml vendored
View File

@@ -33,11 +33,17 @@ jobs:
run: |
sudo apt-get install s3cmd
- name: Install mc
run: |
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
chmod 755 /usr/local/bin/mc
- name: Build and run
run: |
make testbin
export AWS_ACCESS_KEY_ID=user
export AWS_SECRET_ACCESS_KEY=pass
echo $PATH
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_REGION=us-east-1
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity

9
Dockerfile_test_bats
View File

@@ -19,8 +19,16 @@ RUN apt-get update && \
# Set working directory
WORKDIR /tmp
# Install AWS cli
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
# Install mc
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
--create-dirs \
-o /usr/local/minio-binaries/mc && \
chmod -R 755 /usr/local/minio-binaries
ENV PATH="/usr/local/minio-binaries":${PATH}
# Download Go 1.21 (adjust the version and platform as needed)
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
@@ -40,6 +48,7 @@ RUN groupadd -r tester && useradd -r -g tester tester
RUN mkdir /home/tester && chown tester:tester /home/tester
ENV HOME=/home/tester
# install bats
RUN git clone https://github.com/bats-core/bats-core.git && \
cd bats-core && \
./install.sh /home/tester

9
tests/README.md
View File

@@ -3,7 +3,10 @@
## Instructions - Running Locally
1. Build the `versitygw` binary.
2. Install the aws command-line interface if unavailable on your machine. Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
2. Install the command-line interface(s) you want to test if unavailable on your machine.
* **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
* **s3cmd**: Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
* **mc**: Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
3. Install BATS. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example:
@@ -22,8 +25,8 @@
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
```
8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
9. In the root repo folder, run with `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
## Instructions - Running With Docker

77
tests/run.sh
View File

@@ -1,16 +1,75 @@
#!/bin/bash
# Function to display help information
show_help() {
echo "Usage: $0 [option...]"
echo " -h, --help Display this help message and exit"
echo " -s, --static Don't remove buckets between tests"
echo " aws Run tests with aws cli"
echo " aws-posix Run posix tests with aws cli"
echo " s3cmd Run tests with s3cmd utility"
echo " mc Run tests with mc utility"
}
handle_param() {
case $1 in
-h|--help)
show_help
exit 0
;;
-s|--static)
export RECREATE_BUCKETS=false
;;
aws|aws-posix|s3cmd|mc)
set_command_type "$1"
;;
*) # Handle unrecognized options or positional arguments
echo "Unrecognized option: $1" >&2
exit 1
;;
esac
}
set_command_type() {
if [[ -n $command_type ]]; then
echo "Error: command type already set"
exit 1
fi
command_type=$1
export command_type
}
export RECREATE_BUCKETS=true
while [[ "$#" -gt 0 ]]; do
handle_param "$1"
shift # past argument or value
done
if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
echo "Error: VERSITYGW_TEST_ENV parameter must be set"
exit 1
fi
export RECREATE_BUCKETS=true
if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
exit 1
if [[ $RECREATE_BUCKETS == false ]]; then
./tests/setup_static.sh || exit_code=$?
if [[ exit_code -ne 0 ]]; then
exit 1
fi
fi
if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
exit 1
fi
if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
exit 1
fi
case $command_type in
aws)
"$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$?
;;
aws-posix)
"$HOME"/bin/bats ./tests/test_aws_posix.sh || exit_code=$?
;;
s3cmd)
"$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
;;
mc)
"$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
;;
esac
exit $exit_code

25
tests/run_all.sh
View File

@@ -4,9 +4,28 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
echo "Error: VERSITYGW_TEST_ENV parameter must be set"
exit 1
fi
if ! ./tests/run.sh; then
if ! ./tests/run.sh aws; then
exit 1
fi
if ! ./tests/run_static.sh; then
if ! ./tests/run.sh aws-posix; then
exit 1
fi
fi
if ! ./tests/run.sh s3cmd; then
exit 1
fi
if ! ./tests/run.sh mc; then
exit 1
fi
if ! ./tests/run.sh -s aws; then
exit 1
fi
if ! ./tests/run.sh -s aws-posix; then
exit 1
fi
if ! ./tests/run.sh -s s3cmd; then
exit 1
fi
if ! ./tests/run.sh -s mc; then
exit 1
fi
exit 0

20
tests/run_static.sh
View File

@@ -1,20 +0,0 @@
#!/bin/bash
if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
echo "Error: VERSITYGW_TEST_ENV parameter must be set"
exit 1
fi
result=0
export RECREATE_BUCKETS=false
./tests/setup_static.sh
if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
result=1
fi
if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
result=1
fi
if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
result=1
fi
./tests/teardown_static.sh
exit $result

85
tests/setup.sh
View File

@@ -1,73 +1,44 @@
#!/usr/bin/env bash
source ./tests/setup_mc.sh
source ./tests/versity.sh
# bats setup function
setup() {
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then
source tests/.secrets
else
echo "Warning: no secrets file found"
fi
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
echo "Warning: no .env file found in tests folder"
fi
else
# shellcheck source=./.env.default
source "$VERSITYGW_TEST_ENV"
start_versity || start_result=$?
if [[ $start_result -ne 0 ]]; then
echo "error starting versity executable"
return 1
fi
check_params
base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE"
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=" --cert $CERT --key $KEY"
check_params || check_result=$?
if [[ $check_result -ne 0 ]]; then
echo "parameter check failed"
return 1
fi
base_command+=" $BACKEND $LOCAL_FOLDER &"
eval "$base_command"
versitygw_pid=$!
S3CMD_OPTS=()
S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
export versitygw_pid \
AWS_PROFILE \
AWS_ENDPOINT_URL \
LOCAL_FOLDER \
check_add_mc_alias || check_result=$?
if [[ $check_result -ne 0 ]]; then
echo "mc alias check/add failed"
return 1
fi
export AWS_PROFILE \
BUCKET_ONE_NAME \
BUCKET_TWO_NAME \
S3CMD_CONFIG \
S3CMD_OPTS \
RECREATE_BUCKETS
S3CMD_OPTS
}
# make sure required environment variables are defined properly
# make sure required environment variables for tests are defined properly
# return 0 for yes, 1 for no
check_params() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
echo "No AWS access key set"
return 1
elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
echo "No AWS secret access key set"
return 1
elif [ -z "$VERSITY_EXE" ]; then
echo "No versity executable location set"
return 1
elif [ -z "$BACKEND" ]; then
echo "No backend parameter set (options: 'posix')"
return 1
elif [ -z "$AWS_PROFILE" ]; then
echo "No AWS profile set"
return 1
elif [ -z "$LOCAL_FOLDER" ]; then
echo "No local storage folder set"
return 1
elif [ -z "$AWS_ENDPOINT_URL" ]; then
echo "No AWS endpoint URL set"
return 1
elif [ -z "$BUCKET_ONE_NAME" ]; then
if [ -z "$BUCKET_ONE_NAME" ]; then
echo "No bucket one name set"
return 1
elif [ -z "$BUCKET_TWO_NAME" ]; then
@@ -80,6 +51,7 @@ check_params() {
echo "RECREATE_BUCKETS must be 'true' or 'false'"
return 1
fi
return 0
}
# fail a test
@@ -91,14 +63,5 @@ fail() {
# bats teardown function
teardown() {
if [ -n "$versitygw_pid" ]; then
if ps -p "$versitygw_pid" > /dev/null; then
kill "$versitygw_pid"
wait "$versitygw_pid" || true
else
echo "Process with PID $versitygw_pid does not exist."
fi
else
echo "versitygw_pid is not set or empty."
fi
stop_versity
}

34
tests/setup_mc.sh Normal file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env bash
check_for_alias() {
local alias_result
aliases=$(mc alias list)
if [[ $alias_result -ne 0 ]]; then
echo "error checking for aliases: $aliases"
return 2
fi
while IFS= read -r line; do
if [[ $line =~ ^versity$ ]]; then
return 0
fi
done <<< "$aliases"
return 1
}
check_add_mc_alias() {
check_for_alias || alias_result=$?
if [[ $alias_result -eq 2 ]]; then
echo "error checking for aliases"
return 1
fi
if [[ $alias_result -eq 0 ]]; then
return 0
fi
local set_result
error=$(mc alias set --insecure versity "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
if [[ $set_result -ne 0 ]]; then
echo "error setting alias: $error"
return 1
fi
return 0
}

0
tests/s3_bucket_tests.sh → tests/test_aws.sh
View File

0
tests/posix_tests.sh → tests/test_aws_posix.sh
View File

1
tests/test_common.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bats
source ./tests/util.sh
source ./tests/util_file.sh
# common test for creating, deleting buckets

9
tests/test_mc.sh Executable file
View File

@@ -0,0 +1,9 @@
#!/usr/bin/env bats
source ./tests/test_common.sh
source ./tests/setup.sh
# test mc bucket creation/deletion
@test "test_create_delete_bucket_mc" {
test_common_create_delete_bucket "mc"
}

0
tests/s3cmd_tests.sh → tests/test_s3cmd.sh
View File

26
tests/util.sh
View File

@@ -1,17 +1,28 @@
#!/usr/bin/env bats
source ./tests/util_mc.sh
# create an AWS bucket
# param: bucket name
# return 0 for success, 1 for failure
create_bucket() {
if [ $# -ne 1 ]; then
echo "create bucket missing bucket name"
if [ $# -ne 2 ]; then
echo "create bucket missing command type, bucket name"
return 1
fi
local exit_code=0
local error
error=$(aws --no-verify-ssl s3 mb s3://"$1" 2>&1) || exit_code=$?
if [[ $1 == "aws" ]]; then
error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "s3cmd" ]]; then
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "mc" ]]; then
error=$(mc --insecure mb versity/"$2" 2>&1) || exit_code=$?
else
echo "invalid command type $1"
return 1
fi
if [ $exit_code -ne 0 ]; then
echo "error creating bucket: $error"
return 1
@@ -57,6 +68,8 @@ delete_bucket_recursive() {
error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?"
elif [[ $1 == "s3cmd" ]]; then
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?"
elif [[ $1 == "mc" ]]; then
error=$(delete_bucket_recursive_mc "$2") || exit_code="$?"
else
echo "invalid command type '$1'"
return 1
@@ -88,6 +101,8 @@ delete_bucket_contents() {
error=$(aws --no-verify-ssl s3 rm s3://"$2" --recursive 2>&1) || exit_code="$?"
elif [[ $1 == "s3cmd" ]]; then
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?"
elif [[ $1 == "mc" ]]; then
error=$(mc --insecure rm --force --recursive versity/"$2" 2>&1) || exit_code="$?"
else
echo "invalid command type $1"
return 1
@@ -114,6 +129,8 @@ bucket_exists() {
error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
elif [[ $1 == 's3cmd' ]]; then
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
elif [[ $1 == 'mc' ]]; then
error=$(mc --insecure ls versity)
else
echo "invalid command type: $1"
return 2
@@ -184,7 +201,7 @@ setup_bucket() {
return 1
fi
local create_result
create_bucket "$2" || create_result=$?
create_bucket "$1" "$2" || create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "Error creating bucket"
return 1
@@ -287,7 +304,6 @@ delete_object() {
if [[ $1 == 'aws' ]]; then
error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
echo "delete object s3cmd"
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$?
echo "$error"
else

24
tests/util_mc.sh Normal file
View File

@@ -0,0 +1,24 @@
#!/usr/bin/env bash
# use mc tool to delete bucket and contents
# params: bucket name
# return 0 for success, 1 for failure
delete_bucket_recursive_mc() {
if [[ $# -ne 1 ]]; then
echo "delete bucket recursive mc command requires bucket name"
return 1
fi
local exit_code=0
local error
error=$(mc --insecure rm --recursive --force versity/"$1" 2>&1) || exit_code="$?"
if [[ $exit_code -ne 0 ]]; then
echo "error deleting bucket contents: $error"
return 1
fi
error=$(mc --insecure rb versity/"$1" 2>&1) || exit_code="$?"
if [[ $exit_code -ne 0 ]]; then
echo "error deleting bucket: $error"
return 1
fi
return 0
}

80
tests/versity.sh Normal file
View File

@@ -0,0 +1,80 @@
#!/bin/bash
check_exe_params() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
echo "No AWS access key set"
return 1
elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
echo "No AWS secret access key set"
return 1
elif [ -z "$VERSITY_EXE" ]; then
echo "No versity executable location set"
return 1
elif [ -z "$BACKEND" ]; then
echo "No backend parameter set (options: 'posix')"
return 1
elif [ -z "$AWS_PROFILE" ]; then
echo "No AWS profile set"
return 1
elif [ -z "$LOCAL_FOLDER" ]; then
echo "No local storage folder set"
return 1
elif [ -z "$AWS_ENDPOINT_URL" ]; then
echo "No AWS endpoint URL set"
return 1
fi
}
start_versity() {
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then
source tests/.secrets
else
echo "Warning: no secrets file found"
fi
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
echo "Warning: no .env file found in tests folder"
fi
else
# shellcheck source=./.env.default
source "$VERSITYGW_TEST_ENV"
fi
check_exe_params || check_result=$?
if [[ $check_result -ne 0 ]]; then
echo "error checking for parameters"
return 1
fi
base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE"
if [ -n "$CERT" ] && [ -n "$KEY" ]; then
base_command+=" --cert $CERT --key $KEY"
fi
base_command+=" $BACKEND $LOCAL_FOLDER &"
eval "$base_command"
versitygw_pid=$!
export versitygw_pid \
AWS_ACCESS_KEY_ID \
AWS_SECRET_ACCESS_KEY \
VERSITY_EXE \
BACKEND \
AWS_PROFILE \
LOCAL_FOLDER \
AWS_ENDPOINT_URL
}
stop_versity() {
if [ -n "$versitygw_pid" ]; then
if ps -p "$versitygw_pid" > /dev/null; then
kill "$versitygw_pid"
wait "$versitygw_pid" || true
else
echo "Process with PID $versitygw_pid does not exist."
fi
else
echo "versitygw_pid is not set or empty."
fi
}