diff --git a/.github/workflows/system-macos-15-intel.yml b/.github/workflows/system-macos-15-intel.yml index ef2817db..4e8a305d 100644 --- a/.github/workflows/system-macos-15-intel.yml +++ b/.github/workflows/system-macos-15-intel.yml @@ -11,7 +11,7 @@ jobs: - name: Install base dependencies run: | - ./tests/install_base_dependencies.sh + ./tests/install_test.sh - name: Create test user run: | diff --git a/.github/workflows/system-macos-15.yml b/.github/workflows/system-macos-15.yml index 6a16269f..4c008a17 100644 --- a/.github/workflows/system-macos-15.yml +++ b/.github/workflows/system-macos-15.yml @@ -11,7 +11,7 @@ jobs: - name: Install base dependencies run: | - ./tests/install_base_dependencies.sh + ./tests/install_test.sh - name: Create test user run: | diff --git a/.github/workflows/system-ubuntu-arm.yml b/.github/workflows/system-ubuntu-arm.yml index b0e3996d..c9ccbd00 100644 --- a/.github/workflows/system-ubuntu-arm.yml +++ b/.github/workflows/system-ubuntu-arm.yml @@ -11,7 +11,7 @@ jobs: - name: Install base dependencies run: | - ./tests/install_base_dependencies.sh + ./tests/install_test.sh - name: Create test user run: | diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 92502033..3e996fd3 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -17,9 +17,39 @@ jobs: MATRIX_JSON=$(echo -n "$matrix_output" | jq -c . ) echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT" + download: + name: Download + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Download go dependencies + run: | + go mod download + + - name: Download dependencies + run: | + mkdir -p download + ./tests/install_test.sh --download-only download + + - name: Archive Go module cache + run: | + tar -czf download/go-mod-cache.tar.gz \ + -C "$(go env GOPATH)" pkg/mod + + - uses: actions/upload-artifact@v6 + with: + name: download + path: download/ + build: name: RunTests - needs: generate + needs: [generate, download] runs-on: ubuntu-latest strategy: fail-fast: false @@ -28,9 +58,20 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v7 + - uses: actions/download-artifact@v6 + with: + name: download + path: download + + - name: Restore Go dependencies + run: | + mkdir -p "$(go env GOPATH)" + tar -xzf download/go-mod-cache.tar.gz \ + -C "$(go env GOPATH)" + - name: Install base dependencies run: | - ./tests/install_base_dependencies.sh + ./tests/install_test.sh --install-only download - name: Create test user run: | diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index b8127c42..b1257cc1 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -19,7 +19,7 @@ USER tester COPY --chown=tester:tester . /home/tester USER root -RUN INSTALL_MC=true /home/tester/tests/install_base_dependencies.sh +RUN INSTALL_MC=true /home/tester/tests/install_test.sh WORKDIR /tmp RUN update-ca-certificates && rm -rf /var/lib/apt/lists/* diff --git a/tests/README.md b/tests/README.md index d0b0145e..934ec67b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,11 +18,11 @@ ### Posix Backend -> **Note**: many of the required libraries, and a good rundown of the installation procedure, can be found in the `Dockerfile_test_bats` dockerfile in the root folder. +> **Note**: many of the required libraries, and a good rundown of the installation procedure, can be found in the `Dockerfile_test_bats` dockerfile in the root folder, as well as `system.yml` in the `.github/workflows` folder. #### Automatic -Run `tests/install_base_dependencies.sh` to automatically run steps 1 through 5 below. Then proceed with manual setup. +Run `tests/install_test.sh` to automatically run steps 1 through 5 below. Then proceed with manual setup. #### Manual diff --git a/tests/download_dependencies.sh b/tests/download_dependencies.sh new file mode 100644 index 00000000..c99dcba2 --- /dev/null +++ b/tests/download_dependencies.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +download_apt_packages() { + if [ $# -lt 1 ]; then + echo "download_apt_packages requires download folder, packages" >&2 + return 1 + fi + local download_folder + download_folder="$(realpath -m "$1")/apt" packages=("${@:2}") + + if ! mkdir -p "$download_folder"; then + echo "error creating apt download folder" >&2 + return 1 + fi + + if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get update; then + echo "error with apt update" >&2 + return 1 + fi + + if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get install --download-only --reinstall -y -o "Dir::Cache::archives=$download_folder" "${packages[@]}"; then + echo "error with apt download" >&2 + return 1 + else + echo "download successful" + fi + if ! sudo rm -rf "${download_folder}/partial" "${download_folder}/lock"; then + echo "error removing apt-specific repo files and folders" + return 1 + fi + return 0 +} + +download_aws() { + if [ $# -gt 1 ]; then + echo "download_aws should only have optional download folder" >&2 + return 1 + fi + local download_folder="$1" + local aws_folder response archive_name download_path + + if [ -n "$download_folder" ]; then + aws_folder="$1/aws" + if ! response=$(mkdir -p "$aws_folder" 2>&1); then + echo "error making download path: $response" >&2 + return 1 + fi + else + aws_folder="$(mktemp -d)" || return 1 + fi + + archive_name="$AWS_ARCHIVE_NAME" + download_path="${aws_folder}/${archive_name}" + + case "$(uname -m)" in + x86_64|amd64) + download_url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" + ;; + aarch64|arm64) + download_url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" + ;; + *) + echo "error: unsupported Linux architecture for AWS CLI: $(uname -m)" >&2 + rm -rf "$aws_folder" + return 1 + ;; + esac + if ! curl -fsSL "$download_url" -o "$download_path"; then + echo "error: failed to download AWS CLI" >&2 + rm -rf "$aws_folder" + return 1 + fi + printf '%s\n' "$download_path" + return 0 +} + +get_mc_download_link() { + if [ $# -ne 1 ]; then + echo "get_mc_download_link requires os" >&2 + return 1 + fi + local os="$1" + local arch download_url + + arch=$(uname -m) + case "$os" in + Darwin) + case "$arch" in + x86_64|amd64) + download_url="https://dl.min.io/client/mc/release/darwin-amd64/mc" + ;; + arm64|aarch64) + download_url="https://dl.min.io/client/mc/release/darwin-arm64/mc" + ;; + *) + echo "error: unsupported macOS architecture for mc: $arch" >&2 + return 1 + ;; + esac + ;; + Linux) + case "$arch" in + x86_64|amd64) + download_url="https://dl.min.io/client/mc/release/linux-amd64/mc" + ;; + arm64|aarch64) + download_url="https://dl.min.io/client/mc/release/linux-arm64/mc" + ;; + *) + echo "error: unsupported Linux architecture for mc: $arch" >&2 + return 1 + ;; + esac + ;; + *) + echo "error: unsupported OS type for installing mc: $os" >&2 + return 1 + ;; + esac + printf '%s\n' "$download_url" + return 0 +} + +download_mc() { + if [ $# -lt 1 ] || [ $# -gt 2 ]; then + echo "download_mc requires os, optional download folder" >&2 + return 1 + fi + local os="$1" download_folder="$2" + + if ! response=$(get_mc_download_link "$os" 2>&1); then + echo "error getting mc download link: $response" >&2 + return 1 + fi + download_url="$response" + + if [ -n "$download_folder" ]; then + mc_folder="$download_folder/mc" + if ! response=$(mkdir -p "$mc_folder" 2>&1); then + echo "error making download path: $response" >&2 + return 1 + fi + else + mc_folder="$(mktemp -d)" || return 1 + fi + download_path="${mc_folder}/mc" + + if ! curl -fsSL "$download_url" -o "$download_path"; then + echo "error: failed to download mc" >&2 + rm -rf "$mc_folder" + return 1 + fi + + if ! chmod 755 "$download_path"; then + echo "error: failed to mark mc executable" >&2 + rm -rf "$mc_folder" + return 1 + fi + printf '%s\n' "$download_path" + return 0 +} + +download_bats_helpers() { + if [ $# -ne 1 ]; then + echo "download_bats_helpers requires destination folder" >&2 + return 1 + fi + local destination_folder="$1" + + if ! mkdir -p "$destination_folder"; then + echo "error making folder $destination_folder" >&2 + return 1 + fi + if [ -d "${destination_folder}/bats-support" ]; then + echo "bats-support already installed" + else + if ! git clone https://github.com/bats-core/bats-support.git "${destination_folder}/bats-support"; then + echo "error getting bats-support library" >&2 + return 1 + fi + fi + if [ -d "${destination_folder}/bats-assert" ]; then + echo "bats-assert already installed" + else + if ! git clone https://github.com/ztombol/bats-assert.git "${destination_folder}/bats-assert"; then + echo "error getting bats-assert library" >&2 + return 1 + fi + fi + return 0 +} diff --git a/tests/install_base_dependencies.sh b/tests/install_test.sh similarity index 67% rename from tests/install_base_dependencies.sh rename to tests/install_test.sh index c87da0c1..09a110e7 100755 --- a/tests/install_base_dependencies.sh +++ b/tests/install_test.sh @@ -1,5 +1,56 @@ #!/usr/bin/env bash +source ./tests/download_dependencies.sh + +export AWS_ARCHIVE_NAME="awscliv2.zip" + +DOWNLOAD_ONLY_FOLDER="" +INSTALL_ONLY_FOLDER="" + +show_help() { + echo "Usage: $0 [--download-only ] [--install-only ]" + echo " --download-only Download dependency artifacts into " + echo " --install-only Install dependency artifacts from " +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) + show_help + exit 0 + ;; + --download-only) + if [ -z "$2" ]; then + echo "error: --download-only requires a folder argument" >&2 + return 1 + fi + DOWNLOAD_ONLY_FOLDER="$2" + shift 2 + ;; + --install-only) + if [ -z "$2" ]; then + echo "error: --install-only requires a folder argument" >&2 + return 1 + fi + INSTALL_ONLY_FOLDER="$2" + shift 2 + ;; + *) + echo "error: unknown argument '$1'" >&2 + show_help >&2 + return 1 + ;; + esac + done + + if [ -n "$DOWNLOAD_ONLY_FOLDER" ] && [ -n "$INSTALL_ONLY_FOLDER" ]; then + echo "error: --download-only and --install-only cannot be used together" >&2 + return 1 + fi + return 0 +} + pre_install_report() { local os arch cwd detected_pkg_manager xcode_clt cmd @@ -17,6 +68,8 @@ pre_install_report() { printf "CI: %s\n" "${CI:-false}" printf "PATH: %s\n" "$PATH" printf "Home: %s\n" "$HOME" + printf "Download-only folder: %s\n" "${DOWNLOAD_ONLY_FOLDER:-unset}" + printf "Install-only folder: %s\n" "${INSTALL_ONLY_FOLDER:-unset}" if [ "$os" = "Linux" ]; then printf "Detected Linux package manager: %s\n" "$detected_pkg_manager" @@ -220,6 +273,17 @@ detect_linux_package_manager() { return 0 } +run_apt_update_and_install() { + if [ $# -le 0 ]; then + echo "run_apt_update_and_install requires libraries" >&2 + return 1 + fi + local libraries=("$@") + + run_as_root env DEBIAN_FRONTEND=noninteractive apt-get update || return 1 + run_as_root env DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y "${libraries[@]}" || return 1 +} + install_apt_libraries() { local libraries=(git make \ @@ -245,8 +309,24 @@ install_apt_libraries() { else libraries+=(golang-go) fi - run_as_root env DEBIAN_FRONTEND=noninteractive apt-get update || return 1 - run_as_root env DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y "${libraries[@]}" + if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + if [ -z "$INSTALL_ONLY_FOLDER" ]; then + run_apt_update_and_install "${libraries[@]}" || return 1 + else + local install_result=0 + run_as_root env DEBIAN_FRONTEND=noninteractive apt install -y "$(realpath -m "$INSTALL_ONLY_FOLDER")"/apt/*.deb || install_result=$? + # if install from packages fails, fall back + if [ "$install_result" -ne 0 ]; then + run_apt_update_and_install "${libraries[@]}" || return 1 + fi + fi + else + if ! download_apt_packages "$DOWNLOAD_ONLY_FOLDER" "${libraries[@]}"; then + echo "error downloading apt packages" >&2 + return 1 + fi + fi + return 0 } install_dnf_libraries() { @@ -401,25 +481,19 @@ upgrade_bash_on_mac() { return 1 } -install_aws_from_url() { - if [ "$#" -ne 2 ]; then - echo "install_aws_from_url requires download url, archive name" >&2 +install_aws_cli() { + if [ "$#" -ne 1 ]; then + echo "install_aws requires archive location" >&2 return 1 fi - local download_url="$1" archive_name="$2" + local archive_location="$1" local tmp_dir install_dir bin_dir - tmp_dir=$(mktemp -d) || return 1 + tmp_dir=$(mktemp -d) install_dir="/usr/local/aws-cli" bin_dir="/usr/local/bin" - echo "Downloading AWS CLI from $download_url..." - if ! curl -fsSL "$download_url" -o "$tmp_dir/$archive_name"; then - echo "error: failed to download AWS CLI" >&2 - rm -rf "$tmp_dir" - return 1 - fi - if ! unzip -q "$tmp_dir/$archive_name" -d "$tmp_dir"; then + if ! unzip -q "$archive_location" -d "$tmp_dir"; then echo "error: failed to unzip AWS CLI archive" >&2 rm -rf "$tmp_dir" return 1 @@ -430,134 +504,80 @@ install_aws_from_url() { return 1 fi rm -rf "$tmp_dir" + return 0 } -install_linux_aws_command_line_tools() { +download_and_install_aws_cli() { + local response package_location + if command -v aws >/dev/null 2>&1; then printf "AWS CLI already installed: %s\n" "$(aws --version 2>&1)" return 0 fi - local response tmp_dir archive_name download_url - - archive_name="awscliv2.zip" - case "$(uname -m)" in - x86_64|amd64) - download_url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" - ;; - aarch64|arm64) - download_url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" - ;; - *) - echo "error: unsupported Linux architecture for AWS CLI: $(uname -m)" >&2 - rm -rf "$tmp_dir" + if [ -z "$INSTALL_ONLY_FOLDER" ]; then + if ! response=$(download_aws "$DOWNLOAD_ONLY_FOLDER" 2>&1); then + echo "error downloading aws package: $response" >&2 return 1 - ;; - esac - if ! install_aws_from_url "$download_url" "$archive_name"; then - echo "error installing AWS from download url '$download_url'" - return 1 + fi + package_location="$response" + else + package_location="$INSTALL_ONLY_FOLDER/aws/$AWS_ARCHIVE_NAME" fi - - if ! command -v aws >/dev/null 2>&1; then - echo "error: AWS CLI still not found after installation" >&2 - return 1 + printf '%s\n' "aws package location: $package_location" + if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + if ! install_aws_cli "$package_location"; then + echo "error installing aws cli" >&2 + return 1 + fi fi - - if ! response=$(aws --version 2>&1); then - echo "error: failed to run installed AWS CLI" >&2 - return 1 - fi - printf "Installed AWS CLI: %s\n" "$response" return 0 } -get_mc_download_link() { +download_and_install_mc() { if [ $# -ne 1 ]; then - echo "get_mc_download_link requires os" >&2 + echo "download_and_install_mc requires os" >&2 return 1 fi local os="$1" - local arch download_url - - arch=$(uname -m) - case "$os" in - Darwin) - case "$arch" in - x86_64|amd64) - download_url="https://dl.min.io/client/mc/release/darwin-amd64/mc" - ;; - arm64|aarch64) - download_url="https://dl.min.io/client/mc/release/darwin-arm64/mc" - ;; - *) - echo "error: unsupported macOS architecture for mc: $arch" >&2 - return 1 - ;; - esac - ;; - Linux) - case "$arch" in - x86_64|amd64) - download_url="https://dl.min.io/client/mc/release/linux-amd64/mc" - ;; - arm64|aarch64) - download_url="https://dl.min.io/client/mc/release/linux-arm64/mc" - ;; - *) - echo "error: unsupported Linux architecture for mc: $arch" >&2 - return 1 - ;; - esac - ;; - *) - echo "error: unsupported OS type for installing mc: $os" >&2 - return 1 - ;; - esac - printf '%s\n' "$download_url" - return 0 -} - -install_mc() { - if [ "$#" -ne 1 ]; then - echo "install_mc requires os" >&2 - return 1 - fi - local os="$1" - local download_url tmp_dir target_path response + local response package_location if command -v mc >/dev/null 2>&1; then printf "MinIO mc already installed: %s\n" "$(mc --version 2>&1 | head -n 1)" return 0 fi + if [ -z "$INSTALL_ONLY_FOLDER" ]; then + if ! response=$(download_mc "$os" "$DOWNLOAD_ONLY_FOLDER" 2>&1); then + echo "error downloading mc package: $response" >&2 + return 1 + fi + package_location="$response" + else + package_location="$INSTALL_ONLY_FOLDER/mc/mc" + fi + printf 'mc package location: %s\n' "$package_location" + if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + if ! install_mc "$package_location"; then + echo "error installing mc" >&2 + return 1 + fi + fi + return 0 +} - if ! response=$(get_mc_download_link "$os" 2>&1); then - echo "error getting mc download link: $response" >&2 +install_mc() { + if [ "$#" -ne 1 ]; then + echo "install_aws requires mc install script location" >&2 return 1 fi - download_url="$response" + local mc_location="$1" + local target_path="/usr/local/bin/mc" response - tmp_dir=$(mktemp -d) || return 1 - target_path="/usr/local/bin/mc" - echo "Downloading mc from $download_url..." - if ! curl -fsSL "$download_url" -o "$tmp_dir/mc"; then - echo "error: failed to download mc" >&2 - rm -rf "$tmp_dir" - return 1 - fi - - if ! chmod 755 "$tmp_dir/mc"; then - echo "error: failed to mark mc executable" >&2 - rm -rf "$tmp_dir" - return 1 - fi - - if ! run_as_root install "$tmp_dir/mc" "$target_path"; then + if ! run_as_root install "$mc_location" "$target_path"; then echo "error: failed to install mc to $target_path" >&2 - rm -rf "$tmp_dir" + rm -rf "$mc_location" return 1 fi - rm -rf "$tmp_dir" + rm -rf "$mc_location" if ! command -v mc >/dev/null 2>&1; then echo "error: mc still not found after installation" >&2 @@ -572,53 +592,81 @@ install_mc() { return 0 } +download_and_install_bats_helpers() { + local download_folder + + if [ -z "$INSTALL_ONLY_FOLDER" ]; then + if [ -n "$DOWNLOAD_ONLY_FOLDER" ]; then + download_folder="$DOWNLOAD_ONLY_FOLDER/bats-helpers" + else + download_folder="$(dirname "${BASH_SOURCE[0]}")" + fi + if ! download_bats_helpers "$download_folder"; then + echo "error downloading bats helpers" >&2 + return 1 + fi + fi + if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + destination_folder="$(dirname "${BASH_SOURCE[0]}")" + if [ -d "${destination_folder}/bats-support" ]; then + echo "bats-support already installed" + else + if ! mv "$INSTALL_ONLY_FOLDER/bats-helpers/bats-support" "${destination_folder}/bats-support"; then + echo "error installing bats-support" + return 1 + fi + fi + if [ -d "${destination_folder}/bats-assert" ]; then + echo "bats-assert already installed" + else + if ! mv "$INSTALL_ONLY_FOLDER/bats-helpers/bats-assert" "${destination_folder}/bats-assert"; then + echo "error installing bats-assert" + return 1 + fi + fi + fi + return 0 +} + os=$(uname -s) -pre_install_report +if ! parse_args "$@"; then + exit 1 +fi + +if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + pre_install_report +fi if ! install_managed_libraries "$os"; then echo "error installing package-managed libraries" >&2 exit 1 fi -if ! response=$(go get -v -t -d ./... 2>&1); then - printf "error installing go dependencies: %s\n" "$response" - exit 1 -fi - -if [ "$os" == "Linux" ] && ! install_linux_aws_command_line_tools; then - exit 1 -fi - -if [ "${INSTALL_MC:-false}" = "true" ]; then - if ! install_mc "$os"; then - exit 1 - fi -else - echo "Skipping mc installation (set INSTALL_MC=true to enable)" -fi - -tests_folder="$(dirname "${BASH_SOURCE[0]}")" -if [ -d "${tests_folder}/bats-support" ]; then - echo "bats-support already installed" -else - if ! git clone https://github.com/bats-core/bats-support.git "${tests_folder}/bats-support"; then - echo "error getting bats-support library" >&2 - exit 1 - fi -fi -if [ -d "${tests_folder}/bats-assert" ]; then - echo "bats-assert already installed" -else - if ! git clone https://github.com/ztombol/bats-assert.git "${tests_folder}/bats-assert"; then - echo "error getting bats-assert library" >&2 +if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + if ! response=$(go get -v -t -d ./... 2>&1); then + printf "error installing go dependencies: %s\n" "$response" exit 1 fi fi -if ! check_required_commands; then +if [ "$os" == "Linux" ] && ! download_and_install_aws_cli; then exit 1 fi +if ! download_and_install_mc "$os"; then + exit 1 +fi + +if ! download_and_install_bats_helpers; then + exit 1 +fi + +if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then + if ! check_required_commands; then + exit 1 + fi +fi + echo "Install complete" exit 0 diff --git a/tests/setup_common.sh b/tests/setup_common.sh index 2c7c1945..3558fd6e 100644 --- a/tests/setup_common.sh +++ b/tests/setup_common.sh @@ -55,7 +55,7 @@ setup_env() { setup_versitygw() { local params=("$@") - local response pid + local response if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then if ! run_versity_app "${params[@]}"; then