mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-15 19:56:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: go generateCommand executable, different platform tests, install scripts

Browse Source
This commit is contained in:
Luke McCrone
2026-06-26 19:00:30 -03:00
parent 0aa0f0d4fc
commit acfbeeee51
16 changed files with 925 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+3 -2
View File
@@ -45,5 +45,6 @@ dist/
/CODE_OF_CONDUCT.md
/README.md
# standard bats test file folder
/versity-gwtest-files
# standard bats test file folders
/versity-gwtest-files
/tester
.github/workflows/docker-bats.yml
+2
View File
@@ -28,3 +28,5 @@ jobs:
up --exit-code-from s3api_np_only s3api_np_only
docker compose -f tests/docker-compose-bats.yml --project-directory . \
up --exit-code-from openssl_only openssl_only
docker compose -f tests/docker-compose-bats.yml --project-directory . \
up --exit-code-from s3cmd s3cmd
.github/workflows/system-macos-15-intel.yml
+215
View File
@@ -0,0 +1,215 @@
name: MacOS 15 Intel s3cmd test
permissions: {}
on: pull_request
jobs:
build:
name: RunTests
runs-on: macos-15-intel
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Install base dependencies
run: |
./tests/install_base_dependencies.sh
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "stable"
id: go
- name: Get Dependencies
run: |
go get -v -t -d ./...
- name: Upgrade bash
run: |
brew install bash
echo "$(brew --prefix)/bin" >> "$GITHUB_PATH"
- name: Install BATS
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh ${{ github.workspace }}
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
# disable mc tests due to dl.min.io instability
# - name: Install mc
# run: |
# curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
# chmod 755 /usr/local/bin/mc
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Create test user
run: |
sudo sysadminctl -addUser tester -shell /bin/bash -home /Users/tester -createHome
sudo mkdir -p /Users/tester/test-files
sudo chown -R tester:staff /Users/tester
- name: Write env file for test user
env:
IAM_TYPE: folder
RUN_SET: tests/test_user_s3cmd.sh
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: posix
RECREATE_BUCKETS: true
DELETE_BUCKETS_AFTER_TEST: true
CERT: /Users/tester/test-files/cert.pem
KEY: /Users/tester/test-files/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /Users/tester/test-files/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: /Users/tester/test-files/cover
USERNAME_ONE: HIJKLMN
PASSWORD_ONE: 1234567
USERNAME_TWO: OPQRSTU
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: /Users/tester/test-files/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: /Users/tester/test-files/versioning
COMMAND_LOG: /Users/tester/test-files/command.log
TIME_LOG: /Users/tester/test-files/time.log
PYTHON_ENV_FOLDER: /Users/tester/test-files/env
AUTOGENERATE_USERS: true
USER_AUTOGENERATION_PREFIX: github-actions-test-
AWS_REGION: us-east-1
COVERAGE_LOG: /Users/tester/test-files/coverage.log
TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml
run: |
cat > /Users/runner/.env <<EOF
IAM_TYPE=$IAM_TYPE
RUN_SET=$RUN_SET
AWS_PROFILE=$AWS_PROFILE
VERSITY_EXE=$VERSITY_EXE
RUN_VERSITYGW=$RUN_VERSITYGW
BACKEND=$BACKEND
RECREATE_BUCKETS=$RECREATE_BUCKETS
DELETE_BUCKETS_AFTER_TEST=$DELETE_BUCKETS_AFTER_TEST
CERT=$CERT
KEY=$KEY
LOCAL_FOLDER=$LOCAL_FOLDER
BUCKET_ONE_NAME=$BUCKET_ONE_NAME
BUCKET_TWO_NAME=$BUCKET_TWO_NAME
USERS_FOLDER=$USERS_FOLDER
USERS_BUCKET=$USERS_BUCKET
AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL
PORT=$PORT
S3CMD_CONFIG=$S3CMD_CONFIG
MC_ALIAS=$MC_ALIAS
LOG_LEVEL=$LOG_LEVEL
GOCOVERDIR=$GOCOVERDIR
USERNAME_ONE=$USERNAME_ONE
PASSWORD_ONE=$PASSWORD_ONE
USERNAME_TWO=$USERNAME_TWO
PASSWORD_TWO=$PASSWORD_TWO
TEST_FILE_FOLDER=$TEST_FILE_FOLDER
REMOVE_TEST_FILE_FOLDER=$REMOVE_TEST_FILE_FOLDER
VERSIONING_DIR=$VERSIONING_DIR
COMMAND_LOG=$COMMAND_LOG
TIME_LOG=$TIME_LOG
PYTHON_ENV_FOLDER=$PYTHON_ENV_FOLDER
AUTOGENERATE_USERS=$AUTOGENERATE_USERS
USER_AUTOGENERATION_PREFIX=$USER_AUTOGENERATION_PREFIX
AWS_REGION=$AWS_REGION
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
GENERATE_COMMAND_EXECUTABLE=/Users/tester/test-files/generateCommand
EOF
sudo chown tester:staff /Users/runner/.env
sudo mv /Users/runner/.env /Users/tester/test-files/.env
sudo chmod 600 /Users/tester/test-files/.env
- name: Build and run
run: |
set -euo pipefail
make testbin
# github-actions normally sets this to 700, change so jailed tester can execute versitygw
sudo chmod 755 "$HOME"
GO_BIN_DIR="$(dirname "$(which go)")"
BREW_PREFIX="$(brew --prefix)/bin"
BREW_BASH="$(brew --prefix)/bin/bash"
nohup sudo -u tester -H env BYPASS_ENV_FILE=true PATH="$BREW_PREFIX/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" BREW_BASH="$BREW_BASH" $BREW_BASH -c '
set -euo pipefail
export PATH="'"$GO_BIN_DIR"':$PATH"
cd ${{ github.workspace }}
go mod download
cat /Users/tester/test-files/.env
set -a
source /Users/tester/test-files/.env
set +a
echo "bash lookup:"
type -a bash || true
which -a bash || true
command -v bash || true
bash --version || true
/bin/bash --version || true
"$BREW_BASH" --version
echo "$LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER"
mkdir $LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
if [[ $RECREATE_BUCKETS == "false" ]]; then
$BREW_BASH ${{ github.workspace }}/tests/setup_static.sh
fi
$BREW_BASH ${{ github.workspace }}/bin/bats ${{ github.workspace }}/$RUN_SET
if [ -e "$COVERAGE_LOG" ]; then
cat $COVERAGE_LOG
fi
' &
pid=$!
(
sleep 1200
echo "=== TIMEOUT DIAGNOSTICS ==="
ps -ef --forest || true
pgrep -af bash || true
pgrep -af versitygw || true
kill -TERM -"$pid" 2>/dev/null || true
sleep 10
kill -KILL -"$pid" 2>/dev/null || true
) &
watchdog=$!
wait "$pid"
status=$?
kill "$watchdog" 2>/dev/null || true
exit "$status"
.github/workflows/system-macos-15.yml
+215
View File
@@ -0,0 +1,215 @@
name: MacOS 15 s3cmd test
permissions: {}
on: pull_request
jobs:
build:
name: RunTests
runs-on: macos-15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Install base dependencies
run: |
./tests/install_base_dependencies.sh
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "stable"
id: go
- name: Get Go Dependencies
run: |
go get -v -t -d ./...
- name: Upgrade bash
run: |
brew install bash
echo "$(brew --prefix)/bin" >> "$GITHUB_PATH"
- name: Install BATS
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh ${{ github.workspace }}
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
# disable mc tests due to dl.min.io instability
# - name: Install mc
# run: |
# curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
# chmod 755 /usr/local/bin/mc
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Create test user
run: |
sudo sysadminctl -addUser tester -shell /bin/bash -home /Users/tester -createHome
sudo mkdir -p /Users/tester/test-files
sudo chown -R tester:staff /Users/tester
- name: Write env file for test user
env:
IAM_TYPE: folder
RUN_SET: tests/test_user_s3cmd.sh
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: posix
RECREATE_BUCKETS: true
DELETE_BUCKETS_AFTER_TEST: true
CERT: /Users/tester/test-files/cert.pem
KEY: /Users/tester/test-files/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /Users/tester/test-files/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: /Users/tester/test-files/cover
USERNAME_ONE: HIJKLMN
PASSWORD_ONE: 1234567
USERNAME_TWO: OPQRSTU
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: /Users/tester/test-files/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: /Users/tester/test-files/versioning
COMMAND_LOG: /Users/tester/test-files/command.log
TIME_LOG: /Users/tester/test-files/time.log
PYTHON_ENV_FOLDER: /Users/tester/test-files/env
AUTOGENERATE_USERS: true
USER_AUTOGENERATION_PREFIX: github-actions-test-
AWS_REGION: us-east-1
COVERAGE_LOG: /Users/tester/test-files/coverage.log
TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml
run: |
cat > /Users/runner/.env <<EOF
IAM_TYPE=$IAM_TYPE
RUN_SET=$RUN_SET
AWS_PROFILE=$AWS_PROFILE
VERSITY_EXE=$VERSITY_EXE
RUN_VERSITYGW=$RUN_VERSITYGW
BACKEND=$BACKEND
RECREATE_BUCKETS=$RECREATE_BUCKETS
DELETE_BUCKETS_AFTER_TEST=$DELETE_BUCKETS_AFTER_TEST
CERT=$CERT
KEY=$KEY
LOCAL_FOLDER=$LOCAL_FOLDER
BUCKET_ONE_NAME=$BUCKET_ONE_NAME
BUCKET_TWO_NAME=$BUCKET_TWO_NAME
USERS_FOLDER=$USERS_FOLDER
USERS_BUCKET=$USERS_BUCKET
AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL
PORT=$PORT
S3CMD_CONFIG=$S3CMD_CONFIG
MC_ALIAS=$MC_ALIAS
LOG_LEVEL=$LOG_LEVEL
GOCOVERDIR=$GOCOVERDIR
USERNAME_ONE=$USERNAME_ONE
PASSWORD_ONE=$PASSWORD_ONE
USERNAME_TWO=$USERNAME_TWO
PASSWORD_TWO=$PASSWORD_TWO
TEST_FILE_FOLDER=$TEST_FILE_FOLDER
REMOVE_TEST_FILE_FOLDER=$REMOVE_TEST_FILE_FOLDER
VERSIONING_DIR=$VERSIONING_DIR
COMMAND_LOG=$COMMAND_LOG
TIME_LOG=$TIME_LOG
PYTHON_ENV_FOLDER=$PYTHON_ENV_FOLDER
AUTOGENERATE_USERS=$AUTOGENERATE_USERS
USER_AUTOGENERATION_PREFIX=$USER_AUTOGENERATION_PREFIX
AWS_REGION=$AWS_REGION
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
GENERATE_COMMAND_EXECUTABLE=/Users/tester/test-files/generateCommand
EOF
sudo chown tester:staff /Users/runner/.env
sudo mv /Users/runner/.env /Users/tester/test-files/.env
sudo chmod 600 /Users/tester/test-files/.env
- name: Build and run
run: |
set -euo pipefail
make testbin
# github-actions normally sets this to 700, change so jailed tester can execute versitygw
sudo chmod 755 "$HOME"
GO_BIN_DIR="$(dirname "$(which go)")"
BREW_PREFIX="$(brew --prefix)/bin"
BREW_BASH="$(brew --prefix)/bin/bash"
nohup sudo -u tester -H env BYPASS_ENV_FILE=true PATH="$BREW_PREFIX/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" BREW_BASH="$BREW_BASH" $BREW_BASH -c '
set -euo pipefail
export PATH="'"$GO_BIN_DIR"':$PATH"
cd ${{ github.workspace }}
go mod download
cat /Users/tester/test-files/.env
set -a
source /Users/tester/test-files/.env
set +a
echo "bash lookup:"
type -a bash || true
which -a bash || true
command -v bash || true
bash --version || true
/bin/bash --version || true
"$BREW_BASH" --version
echo "$LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER"
mkdir $LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
if [[ $RECREATE_BUCKETS == "false" ]]; then
$BREW_BASH ${{ github.workspace }}/tests/setup_static.sh
fi
$BREW_BASH ${{ github.workspace }}/bin/bats ${{ github.workspace }}/$RUN_SET
if [ -e "$COVERAGE_LOG" ]; then
cat $COVERAGE_LOG
fi
' &
pid=$!
(
sleep 1200
echo "=== TIMEOUT DIAGNOSTICS ==="
ps -ef --forest || true
pgrep -af bash || true
pgrep -af versitygw || true
kill -TERM -"$pid" 2>/dev/null || true
sleep 10
kill -KILL -"$pid" 2>/dev/null || true
) &
watchdog=$!
wait "$pid"
status=$?
kill "$watchdog" 2>/dev/null || true
exit "$status"
.github/workflows/system-ubuntu-arm.yml
+194
View File
@@ -0,0 +1,194 @@
name: Ubuntu ARM s3cmd test
permissions: {}
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-24.04-arm
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Install base dependencies
run: |
./tests/install_base_dependencies.sh
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "stable"
id: go
- name: Get Go Dependencies
run: |
go get -v -t -d ./...
- name: Install BATS
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh ${{ github.workspace }}
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
# disable mc tests due to dl.min.io instability
# - name: Install mc
# run: |
# curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
# chmod 755 /usr/local/bin/mc
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Create test user
run: |
sudo useradd -m -s /bin/bash tester
sudo mkdir -p /home/tester/test-files
sudo chown -R tester:tester /home/tester
- name: Write env file for test user
env:
IAM_TYPE: folder
RUN_SET: tests/test_user_s3cmd.sh
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: posix
RECREATE_BUCKETS: true
DELETE_BUCKETS_AFTER_TEST: true
CERT: /home/tester/test-files/cert.pem
KEY: /home/tester/test-files/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /home/tester/test-files/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: /home/tester/test-files/cover
USERNAME_ONE: HIJKLMN
PASSWORD_ONE: 1234567
USERNAME_TWO: OPQRSTU
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: /home/tester/test-files/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: /home/tester/test-files/versioning
COMMAND_LOG: /home/tester/test-files/command.log
TIME_LOG: /home/tester/test-files/time.log
PYTHON_ENV_FOLDER: /home/tester/test-files/env
AUTOGENERATE_USERS: true
USER_AUTOGENERATION_PREFIX: github-actions-test-
AWS_REGION: us-east-1
COVERAGE_LOG: /home/tester/test-files/coverage.log
TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml
run: |
cat > /home/tester/test-files/.env <<EOF
IAM_TYPE=$IAM_TYPE
RUN_SET=$RUN_SET
AWS_PROFILE=$AWS_PROFILE
VERSITY_EXE=$VERSITY_EXE
RUN_VERSITYGW=$RUN_VERSITYGW
BACKEND=$BACKEND
RECREATE_BUCKETS=$RECREATE_BUCKETS
DELETE_BUCKETS_AFTER_TEST=$DELETE_BUCKETS_AFTER_TEST
CERT=$CERT
KEY=$KEY
LOCAL_FOLDER=$LOCAL_FOLDER
BUCKET_ONE_NAME=$BUCKET_ONE_NAME
BUCKET_TWO_NAME=$BUCKET_TWO_NAME
USERS_FOLDER=$USERS_FOLDER
USERS_BUCKET=$USERS_BUCKET
AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL
PORT=$PORT
S3CMD_CONFIG=$S3CMD_CONFIG
MC_ALIAS=$MC_ALIAS
LOG_LEVEL=$LOG_LEVEL
GOCOVERDIR=$GOCOVERDIR
USERNAME_ONE=$USERNAME_ONE
PASSWORD_ONE=$PASSWORD_ONE
USERNAME_TWO=$USERNAME_TWO
PASSWORD_TWO=$PASSWORD_TWO
TEST_FILE_FOLDER=$TEST_FILE_FOLDER
REMOVE_TEST_FILE_FOLDER=$REMOVE_TEST_FILE_FOLDER
VERSIONING_DIR=$VERSIONING_DIR
COMMAND_LOG=$COMMAND_LOG
TIME_LOG=$TIME_LOG
PYTHON_ENV_FOLDER=$PYTHON_ENV_FOLDER
AUTOGENERATE_USERS=$AUTOGENERATE_USERS
USER_AUTOGENERATION_PREFIX=$USER_AUTOGENERATION_PREFIX
AWS_REGION=$AWS_REGION
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
GENERATE_COMMAND_EXECUTABLE=/home/tester/test-files/generateCommand
EOF
sudo chown tester:tester /home/tester/test-files/.env
sudo chmod 600 /home/tester/test-files/.env
- name: Build and run
run: |
set -euo pipefail
make testbin
# github-actions normally sets this to 700, change so jailed tester can execute versitygw
sudo chmod 755 "$HOME"
GO_BIN_DIR="$(dirname "$(which go)")"
setsid sudo -u tester -H bash -lc '
set -euo pipefail
export PATH="'"$GO_BIN_DIR"':$PATH"
cd ${{ github.workspace }}
go mod download
set -a
source /home/tester/test-files/.env
set +a
mkdir $LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true bash ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/bin/bats ${{ github.workspace }}/$RUN_SET
if [ -e "$COVERAGE_LOG" ]; then
cat $COVERAGE_LOG
fi
' &
pid=$!
(
sleep 1200
echo "=== TIMEOUT DIAGNOSTICS ==="
ps -ef --forest || true
pgrep -af bash || true
pgrep -af versitygw || true
kill -TERM -"$pid" 2>/dev/null || true
sleep 10
kill -KILL -"$pid" 2>/dev/null || true
) &
watchdog=$!
wait "$pid"
status=$?
kill "$watchdog" 2>/dev/null || true
exit "$status"
.github/workflows/system.yml
+7 -12
View File
@@ -1,4 +1,4 @@
name: system tests
name: Ubuntu AMD System Tests
permissions: {}
on: pull_request
jobs:
@@ -28,13 +28,17 @@ jobs:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Install base dependencies
run: |
./tests/install_base_dependencies.sh
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "stable"
id: go
- name: Get Dependencies
- name: Get Go Dependencies
run: |
go get -v -t -d ./...
@@ -45,22 +49,12 @@ jobs:
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
- name: Install s3cmd
run: |
sudo apt-get update
sudo apt-get install s3cmd
# disable mc tests due to dl.min.io instability
# - name: Install mc
# run: |
# curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
# chmod 755 /usr/local/bin/mc
- name: Install xml libraries (for rest)
run: |
sudo apt-get update
sudo apt-get install libxml2-utils xmlstarlet
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
@@ -152,6 +146,7 @@ jobs:
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
GENERATE_COMMAND_EXECUTABLE=/home/tester/test-files/generateCommand
EOF
sudo chown tester:tester /home/tester/test-files/.env
sudo chmod 600 /home/tester/test-files/.env
tests/Dockerfile_test_bats
+15 -28
View File
@@ -1,40 +1,28 @@
FROM ubuntu:latest
ARG TARGETARCH
ARG DEBIAN_FRONTEND=noninteractive
ARG SECRETS_FILE=tests/.secrets
ARG CONFIG_FILE=tests/.env.docker
ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
# see https://github.com/versity/versitygw/issues/1034
ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
ARG MC_FOLDER=linux-arm64
ARG GO_LIBRARY=go1.21.13.linux-${TARGETARCH}.tar.gz
ARG MC_FOLDER=linux-${TARGETARCH}
ENV TZ=Etc/UTC
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
bc \
libxml2-utils \
xmlstarlet \
python3-pip \
python3-venv \
xxd \
uuid-runtime \
ca-certificates && \
update-ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /tmp
COPY ./tests/install_base_dependencies.sh .
RUN ./install_base_dependencies.sh
RUN update-ca-certificates && rm -rf /var/lib/apt/lists/*
RUN case "$TARGETARCH" in \
amd64) echo 'DOWNLOAD_ARCH=x86_64' > ./arch.env ;; \
arm64) echo 'DOWNLOAD_ARCH=aarch64' > ./arch.env ;; \
*) echo "Unsupported TARGETARCH: $TARGETARCH" && exit 1 ;; \
esac
# Install AWS cli
RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
RUN . ./arch.env && curl "https://awscli.amazonaws.com/awscli-exe-linux-${DOWNLOAD_ARCH}.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
# Install mc
RUN curl -fL https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
@@ -102,7 +90,6 @@ RUN mkdir config log && cp $CONFIG_FILE config/.env
ENV WORKSPACE=.
ENV VERSITYGW_TEST_ENV=/home/tester/config/.env
#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
ENTRYPOINT ["tests/run.sh"]
CMD ["s3api,s3,s3cmd,mc,rest"]
tests/commands/command.sh
+27 -1
View File
@@ -52,4 +52,30 @@ send_command() {
sleep "$DIRECT_POST_COMMAND_DELAY"
fi
return $command_result
}
}
generate_go_command() {
if ! check_param_count_gt "command params" 1 $#; then
return 1
fi
local response
if [ -n "$GENERATE_COMMAND_EXECUTABLE" ]; then
if [ ! -f "$GENERATE_COMMAND_EXECUTABLE" ] && ! response=$(go build -o "$GENERATE_COMMAND_EXECUTABLE" ./tests/rest_scripts/generateCommand.go 2>&1); then
log 2 "error building generateCommand executable: $response"
return 1
fi
log 5 "running go generateCommand executable"
if ! response=$("$GENERATE_COMMAND_EXECUTABLE" "$@" 2>&1); then
log 2 "error generating go command from executable: $response"
return 1
fi
else
if ! response=$(go run ./tests/rest_scripts/generateCommand.go "$@" 2>&1); then
log 2 "error generating go command from files: $response"
return 1
fi
fi
echo "$response"
return 0
}
tests/docker-compose-bats.yml
+7
View File
@@ -28,6 +28,13 @@ services:
- CONFIG_FILE=tests/.env.default
image: bats_test
command: ["s3api-bucket,s3api-object"]
s3cmd:
build:
dockerfile: tests/Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.default
image: bats_test
command: ["s3cmd"]
openssl_only:
build:
dockerfile: tests/Dockerfile_test_bats
tests/drivers/not_implemented/not_implemented_rest.sh
+1 -1
View File
@@ -21,7 +21,7 @@ send_not_implemented_expect_failure() {
if ! check_param_count_gt "parameters" 1 $#; then
return 1
fi
if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" \
if ! curl_command=$(generate_go_command -awsAccessKeyId "$AWS_ACCESS_KEY_ID" \
-awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "$@" 2>&1); then
log 2 "error: $curl_command"
return 1
tests/drivers/rest.sh
+1 -1
View File
@@ -244,7 +244,7 @@ rest_go_command_perform_send() {
output_file="$TEST_FILE_FOLDER/$response"
params+=("-outputFile" "$output_file")
fi
if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "-writeXMLPayloadToFile" "$TEST_FILE_FOLDER/$xml_file" "${params[@]}" 2>&1); then
if ! curl_command=$(generate_go_command -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "-writeXMLPayloadToFile" "$TEST_FILE_FOLDER/$xml_file" "${params[@]}" 2>&1); then
log 2 "error: $curl_command"
return 1
fi
tests/env.sh
+3
View File
@@ -202,6 +202,9 @@ check_universal_vars() {
if [ -n "$QUICK_COMPARE_SIZE" ]; then
export QUICK_COMPARE_SIZE
fi
if [ -n "$GENERATE_COMMAND_EXECUTABLE" ]; then
export GENERATE_COMMAND_EXECUTABLE
fi
if ! check_aws_vars; then
log 1 "error checking AWS-related env vars"
tests/install_base_dependencies.sh
Executable
+232
View File
@@ -0,0 +1,232 @@
#!/usr/bin/env bash
check_for_xcode() {
if xcode-select -p >/dev/null 2>&1; then
echo "Xcode Command Line Tools already installed: $(xcode-select -p)"
return 0
fi
echo "Xcode Command Line Tools are required."
if [ "${CI:-false}" = "true" ]; then
echo "Cannot use interactive xcode-select --install in CI." >&2
return 1
fi
echo "Opening installer..."
if xcode-select --install; then
echo "Installer opened. Re-run this script after installation completes."
else
echo "Failed to open installer, or installer is already active." >&2
fi
return 1
}
install_mac_libraries() {
if ! check_for_xcode; then
exit 1
fi
if ! command -v brew >/dev/null 2>&1; then
echo "Homebrew is required to install macOS packages." >&2
echo "Install it from https://brew.sh/, add to PATH as instructed, and rerun this script." >&2
return 1
fi
local packages=(
wget
s3cmd
libxml2
xmlstarlet
)
local package
for package in "${packages[@]}"; do
if brew list --formula "$package" >/dev/null 2>&1; then
echo "ok: $package already installed"
continue
fi
echo "Installing $package..."
if ! brew install "$package"; then
echo "error: failed to install $package" >&2
return 1
fi
done
}
is_root() {
[ "$(id -u)" -eq 0 ]
}
run_as_root() {
if is_root; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "error: root privileges required and sudo not found" >&2
return 1
fi
return 0
}
detect_linux_package_manager() {
if command -v apt-get >/dev/null 2>&1; then
echo "apt"
elif command -v dnf >/dev/null 2>&1; then
echo "dnf"
elif command -v yum >/dev/null 2>&1; then
echo "yum"
elif command -v apk >/dev/null 2>&1; then
echo "apk"
else
echo "unknown"
fi
return 0
}
install_linux_libraries() {
local pkg_manager
pkg_manager="$(detect_linux_package_manager)"
echo "Package manager: $pkg_manager"
case "$pkg_manager" in
apt)
run_as_root apt-get update
run_as_root env DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
apt-get install -y \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
bc \
libxml2-utils \
xmlstarlet \
python3-pip \
python3-venv \
xxd \
uuid-runtime \
ca-certificates
;;
dnf)
run_as_root dnf install -y \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
bc \
libxml2 \
xmlstarlet \
python3-pip \
python3-virtualenv \
vim-common \
util-linux \
ca-certificates
;;
yum)
run_as_root yum install -y \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
bc \
libxml2 \
xmlstarlet \
python3-pip \
python3-virtualenv \
vim-common \
util-linux \
ca-certificates
;;
apk)
run_as_root apk add --no-cache \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
bc \
libxml2-utils \
xmlstarlet \
py3-pip \
python3 \
py3-virtualenv \
xxd \
util-linux \
ca-certificates
;;
*)
echo "error: unsupported Linux package manager" >&2
return 1
;;
esac
}
check_required_commands() {
local commands=(
git
make
wget
curl
unzip
s3cmd
jq
bc
xmllint
xmlstarlet
python3
uuidgen
xxd
)
local cmd
for cmd in "${commands[@]}"; do
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "error: required command not found: $cmd" >&2
return 1
fi
done
}
os=$(uname -s)
case "$os" in
Darwin)
if ! install_mac_libraries; then
echo "error installing MacOS packages" >&2
exit 1
fi
;;
Linux)
if ! install_linux_libraries; then
echo "error installing Linux packages" >&2
exit 1
fi
;;
*)
echo "unsupported os type: $os" >&2
exit 1
esac
tests/remove_static.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
tests/run.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
tests/run_all.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0