From d41ea787990db915d09e9010cd7ecf627c0be285 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 26 Jun 2026 19:28:33 -0300 Subject: [PATCH] test: go executable, base install script --- .dockerignore | 5 +- .github/workflows/docker-bats.yml | 2 + .github/workflows/system.yml | 19 +- tests/Dockerfile_test_bats | 43 ++-- tests/commands/command.sh | 36 ++- tests/docker-compose-bats.yml | 7 + .../not_implemented/not_implemented_rest.sh | 2 +- tests/drivers/rest.sh | 2 +- tests/env.sh | 3 + tests/install_base_dependencies.sh | 237 ++++++++++++++++++ tests/remove_static.sh | 2 +- tests/run.sh | 2 +- tests/run_all.sh | 2 +- tests/test_command.sh | 42 ++++ 14 files changed, 356 insertions(+), 48 deletions(-) create mode 100755 tests/install_base_dependencies.sh create mode 100755 tests/test_command.sh diff --git a/.dockerignore b/.dockerignore index 20c2acdf..24081a8e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,5 +45,6 @@ dist/ /CODE_OF_CONDUCT.md /README.md -# standard bats test file folder -/versity-gwtest-files \ No newline at end of file +# standard bats test file folders +/versity-gwtest-files +/tester \ No newline at end of file diff --git a/.github/workflows/docker-bats.yml b/.github/workflows/docker-bats.yml index 3808a853..80effaad 100644 --- a/.github/workflows/docker-bats.yml +++ b/.github/workflows/docker-bats.yml @@ -28,3 +28,5 @@ jobs: up --exit-code-from s3api_np_only s3api_np_only docker compose -f tests/docker-compose-bats.yml --project-directory . \ up --exit-code-from openssl_only openssl_only + docker compose -f tests/docker-compose-bats.yml --project-directory . \ + up --exit-code-from s3cmd s3cmd diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 16b7263c..a3279124 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -1,4 +1,4 @@ -name: system tests +name: Ubuntu AMD System Tests permissions: {} on: pull_request jobs: @@ -28,13 +28,17 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v7 + - name: Install base dependencies + run: | + ./tests/install_base_dependencies.sh + - name: Set up Go uses: actions/setup-go@v6 with: go-version: "stable" id: go - - name: Get Dependencies + - name: Get Go Dependencies run: | go get -v -t -d ./... @@ -45,22 +49,12 @@ jobs: git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert - - name: Install s3cmd - run: | - sudo apt-get update - sudo apt-get install s3cmd - # disable mc tests due to dl.min.io instability # - name: Install mc # run: | # curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc # chmod 755 /usr/local/bin/mc - - name: Install xml libraries (for rest) - run: | - sudo apt-get update - sudo apt-get install libxml2-utils xmlstarlet - # see https://github.com/versity/versitygw/issues/1034 - name: Install AWS cli run: | @@ -152,6 +146,7 @@ jobs: COVERAGE_LOG=$COVERAGE_LOG TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE QUICK_COMPARE_SIZE=1048576 + GENERATE_COMMAND_EXECUTABLE=/home/tester/test-files/generateCommand EOF sudo chown tester:tester /home/tester/test-files/.env sudo chmod 600 /home/tester/test-files/.env diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 234b461c..f8d02051 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -1,40 +1,28 @@ FROM ubuntu:latest +ARG TARGETARCH ARG DEBIAN_FRONTEND=noninteractive ARG SECRETS_FILE=tests/.secrets ARG CONFIG_FILE=tests/.env.docker -ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz -# see https://github.com/versity/versitygw/issues/1034 -ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip -ARG MC_FOLDER=linux-arm64 +ARG GO_LIBRARY=go1.21.13.linux-${TARGETARCH}.tar.gz +ARG MC_FOLDER=linux-${TARGETARCH} ENV TZ=Etc/UTC -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git \ - make \ - wget \ - curl \ - unzip \ - tzdata \ - s3cmd \ - jq \ - bc \ - libxml2-utils \ - xmlstarlet \ - python3-pip \ - python3-venv \ - xxd \ - uuid-runtime \ - ca-certificates && \ - update-ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -# Set working directory WORKDIR /tmp +COPY ./tests/install_base_dependencies.sh . +RUN ./install_base_dependencies.sh + +RUN update-ca-certificates && rm -rf /var/lib/apt/lists/* + +RUN case "$TARGETARCH" in \ + amd64) echo 'DOWNLOAD_ARCH=x86_64' > ./arch.env ;; \ + arm64) echo 'DOWNLOAD_ARCH=aarch64' > ./arch.env ;; \ + *) echo "Unsupported TARGETARCH: $TARGETARCH" && exit 1 ;; \ + esac + # Install AWS cli -RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install +RUN . ./arch.env && curl "https://awscli.amazonaws.com/awscli-exe-linux-${DOWNLOAD_ARCH}.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install # Install mc RUN curl -fL https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \ @@ -102,7 +90,6 @@ RUN mkdir config log && cp $CONFIG_FILE config/.env ENV WORKSPACE=. ENV VERSITYGW_TEST_ENV=/home/tester/config/.env -#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED ENTRYPOINT ["tests/run.sh"] CMD ["s3api,s3,s3cmd,mc,rest"] diff --git a/tests/commands/command.sh b/tests/commands/command.sh index 6192774a..45001c43 100644 --- a/tests/commands/command.sh +++ b/tests/commands/command.sh @@ -52,4 +52,38 @@ send_command() { sleep "$DIRECT_POST_COMMAND_DELAY" fi return $command_result -} \ No newline at end of file +} + +check_for_and_or_build_go_executable() { + if [ -z "$GO_COMMAND_GENERATOR_EXECUTABLE" ]; then + return 1 + fi + if [ ! -f "$GO_COMMAND_GENERATOR_EXECUTABLE" ]; then + if ! build_output=$(go build -o "$GO_COMMAND_GENERATOR_EXECUTABLE" ./tests/rest_scripts/generateCommand.go 2>&1); then + log 2 "error building generateCommand executable: $build_output" + return 1 + fi + fi + return 0 +} + +generate_go_command() { + if ! check_param_count_gt "command params" 1 $#; then + return 1 + fi + local go_response + + if check_for_and_or_build_go_executable 2>/dev/null; then + if ! go_response=$("$GO_COMMAND_GENERATOR_EXECUTABLE" "$@" 2>&1); then + log 2 "error generating go command from executable: $go_response" + return 1 + fi + else + if ! go_response=$(go run ./tests/rest_scripts/generateCommand.go "$@" 2>&1); then + log 2 "error generating go command from files (if using tester account, are you using -H tag?): $go_response" + return 1 + fi + fi + echo "$go_response" + return 0 +} diff --git a/tests/docker-compose-bats.yml b/tests/docker-compose-bats.yml index c3da60e6..7ea8dccf 100644 --- a/tests/docker-compose-bats.yml +++ b/tests/docker-compose-bats.yml @@ -28,6 +28,13 @@ services: - CONFIG_FILE=tests/.env.default image: bats_test command: ["s3api-bucket,s3api-object"] + s3cmd: + build: + dockerfile: tests/Dockerfile_test_bats + args: + - CONFIG_FILE=tests/.env.default + image: bats_test + command: ["s3cmd"] openssl_only: build: dockerfile: tests/Dockerfile_test_bats diff --git a/tests/drivers/not_implemented/not_implemented_rest.sh b/tests/drivers/not_implemented/not_implemented_rest.sh index 36ba2c6c..7fa61f1d 100644 --- a/tests/drivers/not_implemented/not_implemented_rest.sh +++ b/tests/drivers/not_implemented/not_implemented_rest.sh @@ -21,7 +21,7 @@ send_not_implemented_expect_failure() { if ! check_param_count_gt "parameters" 1 $#; then return 1 fi - if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" \ + if ! curl_command=$(generate_go_command -awsAccessKeyId "$AWS_ACCESS_KEY_ID" \ -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "$@" 2>&1); then log 2 "error: $curl_command" return 1 diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 527e37b1..e1e572b4 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -244,7 +244,7 @@ rest_go_command_perform_send() { output_file="$TEST_FILE_FOLDER/$response" params+=("-outputFile" "$output_file") fi - if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "-writeXMLPayloadToFile" "$TEST_FILE_FOLDER/$xml_file" "${params[@]}" 2>&1); then + if ! curl_command=$(generate_go_command -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "-writeXMLPayloadToFile" "$TEST_FILE_FOLDER/$xml_file" "${params[@]}" 2>&1); then log 2 "error: $curl_command" return 1 fi diff --git a/tests/env.sh b/tests/env.sh index 0b0959cc..dfc76969 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -202,6 +202,9 @@ check_universal_vars() { if [ -n "$QUICK_COMPARE_SIZE" ]; then export QUICK_COMPARE_SIZE fi + if [ -n "$GENERATE_COMMAND_EXECUTABLE" ]; then + export GENERATE_COMMAND_EXECUTABLE + fi if ! check_aws_vars; then log 1 "error checking AWS-related env vars" diff --git a/tests/install_base_dependencies.sh b/tests/install_base_dependencies.sh new file mode 100755 index 00000000..bd6f2521 --- /dev/null +++ b/tests/install_base_dependencies.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +check_for_xcode() { + if xcode-select -p >/dev/null 2>&1; then + echo "Xcode Command Line Tools already installed: $(xcode-select -p)" + return 0 + fi + + echo "Xcode Command Line Tools are required." + + if [ "${CI:-false}" = "true" ]; then + echo "Cannot use interactive xcode-select --install in CI." >&2 + return 1 + fi + + echo "Opening installer..." + if xcode-select --install; then + echo "Installer opened. Re-run this script after installation completes." + else + echo "Failed to open installer, or installer is already active." >&2 + fi + + return 1 +} + +install_mac_libraries() { + if ! check_for_xcode; then + exit 1 + fi + + if ! command -v brew >/dev/null 2>&1; then + echo "Homebrew is required to install macOS packages." >&2 + echo "Install it from https://brew.sh/, add to PATH as instructed, and rerun this script." >&2 + return 1 + fi + + local packages=( + wget + s3cmd + libxml2 + xmlstarlet + ) + + local package + for package in "${packages[@]}"; do + if brew list --formula "$package" >/dev/null 2>&1; then + echo "ok: $package already installed" + continue + fi + echo "Installing $package..." + if ! brew install "$package"; then + echo "error: failed to install $package" >&2 + return 1 + fi + done + return 0 +} + +is_root() { + [ "$(id -u)" -eq 0 ] +} + +run_as_root() { + if is_root; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "error: root privileges required and sudo not found" >&2 + return 1 + fi + # just returns specific error code from command run as root +} + +detect_linux_package_manager() { + if command -v apt-get >/dev/null 2>&1; then + echo "apt" + elif command -v dnf >/dev/null 2>&1; then + echo "dnf" + elif command -v yum >/dev/null 2>&1; then + echo "yum" + elif command -v apk >/dev/null 2>&1; then + echo "apk" + else + echo "unknown" + fi + return 0 +} + +install_linux_libraries() { + local pkg_manager + pkg_manager="$(detect_linux_package_manager)" + + echo "Package manager: $pkg_manager" + + case "$pkg_manager" in + apt) + run_as_root env DEBIAN_FRONTEND=noninteractive apt-get update || return 1 + run_as_root env DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ + apt-get install -y \ + git \ + make \ + wget \ + curl \ + unzip \ + tzdata \ + s3cmd \ + jq \ + bc \ + libxml2-utils \ + xmlstarlet \ + python3-pip \ + python3-venv \ + xxd \ + uuid-runtime \ + ca-certificates || return 1 + ;; + + dnf) + run_as_root dnf install -y \ + git \ + make \ + wget \ + curl \ + unzip \ + tzdata \ + s3cmd \ + jq \ + bc \ + libxml2 \ + xmlstarlet \ + python3-pip \ + python3-virtualenv \ + vim-common \ + util-linux \ + ca-certificates || return 1 + ;; + + yum) + run_as_root yum install -y \ + git \ + make \ + wget \ + curl \ + unzip \ + tzdata \ + s3cmd \ + jq \ + bc \ + libxml2 \ + xmlstarlet \ + python3-pip \ + python3-virtualenv \ + vim-common \ + util-linux \ + ca-certificates || return 1 + ;; + + apk) + run_as_root apk add --no-cache \ + git \ + make \ + wget \ + curl \ + unzip \ + tzdata \ + s3cmd \ + jq \ + bc \ + libxml2-utils \ + xmlstarlet \ + py3-pip \ + python3 \ + py3-virtualenv \ + xxd \ + util-linux \ + ca-certificates || return 1 + ;; + + *) + echo "error: unsupported Linux package manager" >&2 + return 1 + ;; + esac + return 0 +} + +check_required_commands() { + local commands=( + git + make + wget + curl + unzip + s3cmd + jq + bc + xmllint + xmlstarlet + python3 + uuidgen + xxd + ) + + local cmd + for cmd in "${commands[@]}"; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "error: required command not found: $cmd" >&2 + return 1 + fi + done + return 0 +} + +os=$(uname -s) + +case "$os" in + Darwin) + if ! install_mac_libraries; then + echo "error installing MacOS packages" >&2 + exit 1 + fi + ;; + + Linux) + if ! install_linux_libraries; then + echo "error installing Linux packages" >&2 + exit 1 + fi + ;; + + *) + echo "unsupported os type: $os" >&2 + exit 1 +esac + +exit 0 diff --git a/tests/remove_static.sh b/tests/remove_static.sh index 6394de10..6da77ef9 100755 --- a/tests/remove_static.sh +++ b/tests/remove_static.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 diff --git a/tests/run.sh b/tests/run.sh index a3390c89..5cbeb0c2 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 diff --git a/tests/run_all.sh b/tests/run_all.sh index 2c19017e..dc1d756a 100755 --- a/tests/run_all.sh +++ b/tests/run_all.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 diff --git a/tests/test_command.sh b/tests/test_command.sh new file mode 100755 index 00000000..ec80da96 --- /dev/null +++ b/tests/test_command.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/commands/command.sh +source ./tests/logger.sh +source ./tests/setup_unit.sh + +@test "check_for_and_or_build_go_executable" { + export GO_COMMAND_GENERATOR_EXECUTABLE= + run check_for_and_or_build_go_executable + assert_failure 1 + + export GO_COMMAND_GENERATOR_EXECUTABLE="/dev/null/dummy" + run check_for_and_or_build_go_executable + assert_failure 1 + assert_output -p "error building generateCommand executable" + + command_executable="$TEST_FILE_FOLDER/$(uuidgen)" + export GO_COMMAND_GENERATOR_EXECUTABLE="$command_executable" + run check_for_and_or_build_go_executable + assert_success + assert [ -f "$command_executable" ] + + run check_for_and_or_build_go_executable + assert_success +}