From 71f8e9a3421cb63928adfbfdf7e3b74cbf79dd52 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 1 Nov 2024 12:27:11 +0000 Subject: [PATCH] test: github-actions speedup, cleanup --- .github/workflows/system.yml | 198 +++++++----------- tests/.env.default | 3 +- tests/docker-compose-bats.yml | 2 +- tests/run.sh | 74 ++++++- tests/setup.sh | 10 +- tests/test_mc.sh | 4 - tests/test_mc_file_count.sh | 23 ++ tests/test_s3.sh | 4 - tests/test_s3_file_count.sh | 22 ++ tests/test_s3api_bucket.sh | 125 +++++++++++ tests/test_s3api_multipart.sh | 110 ++++++++++ tests/{test_s3api.sh => test_s3api_object.sh} | 162 -------------- tests/test_s3cmd.sh | 4 - tests/test_s3cmd_file_count.sh | 23 ++ 14 files changed, 459 insertions(+), 305 deletions(-) create mode 100755 tests/test_mc_file_count.sh create mode 100755 tests/test_s3_file_count.sh create mode 100755 tests/test_s3api_bucket.sh create mode 100755 tests/test_s3api_multipart.sh rename tests/{test_s3api.sh => test_s3api_object.sh} (62%) create mode 100755 tests/test_s3cmd_file_count.sh diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index b98e755..1691042 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -8,138 +8,96 @@ jobs: fail-fast: false matrix: include: - - set: "s3cmd, posix" - LOCAL_FOLDER: /tmp/gw1 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-1 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-1 + - set: "mc, posix, non-file count, non-static, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam1 - AWS_ENDPOINT_URL: https://127.0.0.1:7070 - RUN_SET: "s3cmd" + RUN_SET: "mc-non-file-count" RECREATE_BUCKETS: "true" - PORT: 7070 BACKEND: "posix" - - set: "s3, posix" - LOCAL_FOLDER: /tmp/gw2 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-2 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-2 + - set: "mc, posix, file count, non-static, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam2 - AWS_ENDPOINT_URL: https://127.0.0.1:7071 - RUN_SET: "s3" + RUN_SET: "mc-file-count" RECREATE_BUCKETS: "true" - PORT: 7071 BACKEND: "posix" - - set: "s3api non-policy, posix" - LOCAL_FOLDER: /tmp/gw3 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-3 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-3 + - set: "REST, posix, non-static, all, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam3 - AWS_ENDPOINT_URL: https://127.0.0.1:7072 - RUN_SET: "s3api" - RECREATE_BUCKETS: "true" - PORT: 7072 - BACKEND: "posix" - - set: "mc, posix" - LOCAL_FOLDER: /tmp/gw4 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-4 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-4 - IAM_TYPE: folder - USERS_FOLDER: /tmp/iam4 - AWS_ENDPOINT_URL: https://127.0.0.1:7073 - RUN_SET: "mc" - RECREATE_BUCKETS: "true" - PORT: 7073 - BACKEND: "posix" - - set: "s3api-user, posix, s3 IAM" - LOCAL_FOLDER: /tmp/gw5 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-5 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-5 - IAM_TYPE: s3 - USERS_BUCKET: versity-gwtest-iam - AWS_ENDPOINT_URL: https://127.0.0.1:7074 - RUN_SET: "s3api-user" - RECREATE_BUCKETS: "true" - PORT: 7074 - BACKEND: "posix" - - set: "s3api non-policy, static buckets" - LOCAL_FOLDER: /tmp/gw6 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-6 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-6 - IAM_TYPE: folder - USERS_FOLDER: /tmp/iam6 - AWS_ENDPOINT_URL: https://127.0.0.1:7075 - RUN_SET: "s3api-non-policy" - RECREATE_BUCKETS: "false" - PORT: 7075 - BACKEND: "posix" - - set: "s3api non-policy, s3 backend" - LOCAL_FOLDER: /tmp/gw7 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-7 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-7 - IAM_TYPE: folder - USERS_FOLDER: /tmp/iam7 - AWS_ENDPOINT_URL: https://127.0.0.1:7076 - RUN_SET: "s3api" - RECREATE_BUCKETS: "true" - PORT: 7076 - BACKEND: "s3" - - set: "REST, posix" - LOCAL_FOLDER: /tmp/gw8 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-7 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-7 - IAM_TYPE: folder - USERS_FOLDER: /tmp/iam8 - AWS_ENDPOINT_URL: https://127.0.0.1:7077 RUN_SET: "rest" RECREATE_BUCKETS: "true" - PORT: 7077 BACKEND: "posix" - - set: "s3api policy, static buckets" - LOCAL_FOLDER: /tmp/gw9 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-8 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-8 + - set: "s3, posix, non-file count, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3-non-file-count" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3, posix, file count, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3-file-count" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-bucket,s3api-object,s3api-multipart" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3api, posix, policy, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-policy" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3api, posix, user, non-static, s3 IAM" + IAM_TYPE: s3 + RUN_SET: "s3api-user" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3api, posix, bucket, static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-bucket" + RECREATE_BUCKETS: "false" + BACKEND: "posix" + - set: "s3api, posix, multipart, static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-multipart" + RECREATE_BUCKETS: "false" + BACKEND: "posix" + - set: "s3api, posix, object, static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-object" + RECREATE_BUCKETS: "false" + BACKEND: "posix" + - set: "s3api, posix, policy, static, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam9 - AWS_ENDPOINT_URL: https://127.0.0.1:7078 RUN_SET: "s3api-policy" RECREATE_BUCKETS: "false" - PORT: 7078 BACKEND: "posix" - - set: "s3api user, static buckets" - LOCAL_FOLDER: /tmp/gw10 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-9 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-9 + - set: "s3api, posix, user, static, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam10 - AWS_ENDPOINT_URL: https://127.0.0.1:7079 RUN_SET: "s3api-user" RECREATE_BUCKETS: "false" - PORT: 7079 BACKEND: "posix" - - set: "s3api policy and user, posix" - LOCAL_FOLDER: /tmp/gw11 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-10 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-10 + - set: "s3api, s3, multipart|object, non-static, folder IAM" IAM_TYPE: folder - USERS_FOLDER: /tmp/iam11 - AWS_ENDPOINT_URL: https://127.0.0.1:7080 - RUN_SET: "s3api-policy,s3api-user" + RUN_SET: "s3api-bucket,s3api-object,s3api-multipart" RECREATE_BUCKETS: "true" - PORT: 7080 - BACKEND: "posix" - - set: "s3api policy and user, s3 backend" - LOCAL_FOLDER: /tmp/gw12 - BUCKET_ONE_NAME: versity-gwtest-bucket-one-11 - BUCKET_TWO_NAME: versity-gwtest-bucket-two-11 - IAM_TYPE: folder - USERS_FOLDER: /tmp/iam12 - AWS_ENDPOINT_URL: https://127.0.0.1:7081 - RUN_SET: "s3api-policy,s3api-user" - RECREATE_BUCKETS: "true" - PORT: 7081 BACKEND: "s3" + - set: "s3api, s3, policy|user, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3api-policy,s3api-user" + RECREATE_BUCKETS: "true" + BACKEND: "s3" + - set: "s3cmd, posix, file count, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3cmd-file-count" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3cmd, posix, non-user, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3cmd-non-user" + RECREATE_BUCKETS: "true" + BACKEND: "posix" + - set: "s3cmd, posix, user, non-static, folder IAM" + IAM_TYPE: folder + RUN_SET: "s3cmd-user" + RECREATE_BUCKETS: "true" + BACKEND: "posix" steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -176,15 +134,8 @@ jobs: - name: Build and run env: - LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }} - BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }} - BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }} - USERS_FOLDER: ${{ matrix.USERS_FOLDER }} - USERS_BUCKET: ${{ matrix.USERS_BUCKET }} IAM_TYPE: ${{ matrix.IAM_TYPE }} - AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }} RUN_SET: ${{ matrix.RUN_SET }} - PORT: ${{ matrix.PORT }} AWS_PROFILE: versity VERSITY_EXE: ${{ github.workspace }}/versitygw RUN_VERSITYGW: true @@ -192,6 +143,13 @@ jobs: RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }} CERT: ${{ github.workspace }}/cert.pem KEY: ${{ github.workspace }}/versitygw.pem + LOCAL_FOLDER: /tmp/gw + BUCKET_ONE_NAME: versity-gwtest-bucket-one + BUCKET_TWO_NAME: versity-gwtest-bucket-two + USERS_FOLDER: /tmp/iam + USERS_BUCKET: versity-gwtest-iam + AWS_ENDPOINT_URL: https://127.0.0.1:7070 + PORT: 7070 S3CMD_CONFIG: tests/s3cfg.local.default MC_ALIAS: versity LOG_LEVEL: 4 @@ -204,6 +162,7 @@ jobs: REMOVE_TEST_FILE_FOLDER: true VERSIONING_DIR: ${{ github.workspace }}/versioning COMMAND_LOG: command.log + TIME_LOG: time.log run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST @@ -224,6 +183,9 @@ jobs: fi BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET + - name: Time report + run: cat ${{ github.workspace }}/time.log + - name: Coverage report run: | go tool covdata percent -i=cover diff --git a/tests/.env.default b/tests/.env.default index 674f771..7b47c09 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -28,4 +28,5 @@ PASSWORD_TWO=OPQRSTU TEST_FILE_FOLDER=$PWD/versity-gwtest-files REMOVE_TEST_FILE_FOLDER=true VERSIONING_DIR=/tmp/versioning -COMMAND_LOG=command.log \ No newline at end of file +COMMAND_LOG=command.log +TIME_LOG=time.log \ No newline at end of file diff --git a/tests/docker-compose-bats.yml b/tests/docker-compose-bats.yml index f63a0ce..10bda08 100644 --- a/tests/docker-compose-bats.yml +++ b/tests/docker-compose-bats.yml @@ -27,7 +27,7 @@ services: args: - CONFIG_FILE=tests/.env.default image: bats_test - command: ["s3api-non-policy"] + command: ["s3api-bucket,s3api-object"] direct: build: dockerfile: tests/Dockerfile_direct diff --git a/tests/run.sh b/tests/run.sh index e3ef30a..af89a0e 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -19,14 +19,21 @@ show_help() { echo "Usage: $0 [option...]" echo " -h, --help Display this help message and exit" echo " Separate the below by comma" - echo " s3api Run tests with s3api cli" - echo " s3api-non-policy Run policy tests with s3api cli" + echo " s3api Run all tests with s3api cli" + echo " s3api-multipart Run multipart tests with s3api cli" + echo " s3api-bucket Run bucket tests with s3api cli" + echo " s3api-object Run object tests with s3api cli" echo " s3api-policy Run policy tests with s3api cli" + echo " s3api-user Run user tests with s3api cli" echo " s3 Run tests with s3 cli" echo " s3cmd Run tests with s3cmd utility" + echo " s3cmd-user Run user tests with s3cmd utility" + echo " s3cmd-non-user Run non-user tests with s3cmd utility" + echo " s3cmd-file-count Run file count test with s3cmd utility" echo " mc Run tests with mc utility" + echo " mc-non-file-count Run non-file count tests with mc utility" + echo " mc-file-count Run file count test with mc utility" echo " rest Run tests with rest cli" - echo " s3api-user Run user tests with aws cli" } handle_param() { @@ -35,7 +42,7 @@ handle_param() { show_help exit 0 ;; - s3|s3api|s3cmd|mc|s3api-user|rest|s3api-policy|s3api-non-policy) + s3|s3-file-count|s3-non-file-count|s3api|s3cmd|s3cmd-user|s3cmd-non-user|s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|s3api-bucket|s3api-object|s3api-multipart) run_suite "$1" ;; *) # Handle unrecognized options or positional arguments @@ -50,25 +57,50 @@ run_suite() { case $1 in s3api) echo "Running all s3api tests ..." - "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$? + "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$? + if [[ $exit_code -eq 0 ]]; then + "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$? + fi if [[ $exit_code -eq 0 ]]; then "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$? fi + if [[ $exit_code -eq 0 ]]; then + "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$? + fi if [[ $exit_code -eq 0 ]]; then "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$? fi ;; + s3api-multipart) + echo "Running s3api multipart tests ..." + "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$? + ;; s3api-policy) echo "Running s3api policy tests ..." "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$? ;; - s3api-non-policy) - echo "Running s3api non-policy tests ..." - "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$? + s3api-bucket) + echo "Running s3api bucket tests ..." + "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$? + ;; + s3api-object) + echo "Running s3api object tests ..." + "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$? ;; s3) echo "Running s3 tests ..." "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$? + if [[ $exit_code -eq 0 ]]; then + "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$? + fi + ;; + s3-non-file-count) + echo "Running s3 non-file count tests ..." + "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$? + ;; + s3-file-count) + echo "Running s3 file count test ..." + "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$? ;; s3cmd) echo "Running s3cmd tests ..." @@ -76,10 +108,36 @@ run_suite() { if [[ $exit_code -eq 0 ]]; then "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$? fi + if [[ $exit_code -eq 0 ]]; then + "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$? + fi + ;; + s3cmd-user) + echo "Running s3cmd user tests ..." + "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$? + ;; + s3cmd-non-user) + echo "Running s3cmd non-user tests ..." + "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$? + ;; + s3cmd-file-count) + echo "Running s3cmd file count test ..." + "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$? ;; mc) echo "Running mc tests ..." "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$? + if [[ $exit_code -eq 0 ]]; then + "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$? + fi + ;; + mc-non-file-count) + echo "Running mc non-file count tests ..." + "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$? + ;; + mc-file-count) + echo "Running mc file count test ..." + "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$? ;; rest) echo "Running rest tests ..." diff --git a/tests/setup.sh b/tests/setup.sh index 25cd8fb..83666e7 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -28,7 +28,7 @@ setup() { base_setup log 4 "Running test $BATS_TEST_NAME" - if [[ $LOG_LEVEL -ge 5 ]]; then + if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then start_time=$(date +%s) export start_time fi @@ -75,9 +75,13 @@ teardown() { fi fi stop_versity - if [[ $LOG_LEVEL -ge 5 ]]; then + if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then end_time=$(date +%s) - log 4 "Total test time: $((end_time - start_time))" + total_time=$((end_time - start_time)) + log 4 "Total test time: $total_time" + if [[ -n "$TIME_LOG" ]]; then + echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG" + fi fi if [[ -n "$COVERAGE_DB" ]]; then record_result diff --git a/tests/test_mc.sh b/tests/test_mc.sh index 6b2f1c8..fb022e0 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -112,10 +112,6 @@ export RUN_MC=true test_common_presigned_url_utf8_chars "mc" } -@test "test_list_objects_file_count" { - test_common_list_objects_file_count "mc" -} - @test "test_create_bucket_invalid_name_mc" { if [[ $RECREATE_BUCKETS != "true" ]]; then return diff --git a/tests/test_mc_file_count.sh b/tests/test_mc_file_count.sh new file mode 100755 index 0000000..c8be937 --- /dev/null +++ b/tests/test_mc_file_count.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/test_common.sh + +export RUN_MC=true + +@test "test_list_objects_file_count" { + test_common_list_objects_file_count "mc" +} diff --git a/tests/test_s3.sh b/tests/test_s3.sh index 1e0a211..c4e4f38 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -51,10 +51,6 @@ source ./tests/util_file.sh test_common_list_buckets "s3" } -@test "test_list_objects_file_count" { - test_common_list_objects_file_count "s3" -} - @test "test_delete_bucket" { if [[ $RECREATE_BUCKETS == "false" ]]; then skip "will not test bucket deletion in static bucket test config" diff --git a/tests/test_s3_file_count.sh b/tests/test_s3_file_count.sh new file mode 100755 index 0000000..811daa8 --- /dev/null +++ b/tests/test_s3_file_count.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/test_common.sh +source ./tests/util_file.sh + +@test "test_list_objects_file_count" { + test_common_list_objects_file_count "s3" +} diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh new file mode 100755 index 0000000..b2a3e8d --- /dev/null +++ b/tests/test_s3api_bucket.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/setup.sh +source ./tests/util.sh +source ./tests/util_aws.sh +source ./tests/util_create_bucket.sh +source ./tests/util_file.sh +source ./tests/util_lock_config.sh +source ./tests/util_tags.sh +source ./tests/util_users.sh +source ./tests/test_aws_root_inner.sh +source ./tests/test_common.sh +source ./tests/test_common_acl.sh +source ./tests/commands/copy_object.sh +source ./tests/commands/delete_bucket_policy.sh +source ./tests/commands/delete_object_tagging.sh +source ./tests/commands/get_bucket_acl.sh +source ./tests/commands/get_bucket_policy.sh +source ./tests/commands/get_bucket_versioning.sh +source ./tests/commands/get_object.sh +source ./tests/commands/get_object_attributes.sh +source ./tests/commands/get_object_legal_hold.sh +source ./tests/commands/get_object_lock_configuration.sh +source ./tests/commands/get_object_retention.sh +source ./tests/commands/get_object_tagging.sh +source ./tests/commands/list_object_versions.sh +source ./tests/commands/put_bucket_acl.sh +source ./tests/commands/put_bucket_policy.sh +source ./tests/commands/put_bucket_versioning.sh +source ./tests/commands/put_object.sh +source ./tests/commands/put_object_legal_hold.sh +source ./tests/commands/put_object_lock_configuration.sh +source ./tests/commands/put_object_retention.sh +source ./tests/commands/put_public_access_block.sh +source ./tests/commands/select_object_content.sh + +export RUN_USERS=true + +# create-bucket +@test "test_create_delete_bucket_aws" { + test_common_create_delete_bucket "aws" +} + +@test "test_create_bucket_invalid_name" { + test_create_bucket_invalid_name_aws_root +} + +# delete-bucket - test_create_delete_bucket_aws + +# delete-bucket-policy +@test "test_get_put_delete_bucket_policy" { + if [[ -n $SKIP_POLICY ]]; then + skip "will not test policy actions with SKIP_POLICY set" + fi + test_common_get_put_delete_bucket_policy "aws" +} + +# delete-bucket-tagging +@test "test-set-get-delete-bucket-tags" { + test_common_set_get_delete_bucket_tags "aws" +} + +# get-bucket-acl +@test "test_get_bucket_acl" { + test_get_bucket_acl_aws_root +} + +# get-bucket-location +@test "test_get_bucket_location" { + test_common_get_bucket_location "aws" +} + +# get-bucket-policy - test_get_put_delete_bucket_policy + +# get-bucket-tagging - test_set_get_delete_bucket_tags + +@test "test_head_bucket_invalid_name" { + if head_bucket "aws" ""; then + fail "able to get bucket info for invalid name" + fi +} + +# test listing buckets on versitygw +@test "test_list_buckets" { + test_common_list_buckets "s3api" +} + +@test "test_put_bucket_acl" { + test_common_put_bucket_acl "s3api" +} + +@test "test_head_bucket" { + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info" + log 5 "INFO: $bucket_info" + region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" + [[ $region != "" ]] || fail "empty bucket region" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" +} + +@test "test_head_bucket_doesnt_exist" { + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$? + [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" + [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" +} diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh new file mode 100755 index 0000000..d510954 --- /dev/null +++ b/tests/test_s3api_multipart.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/setup.sh +source ./tests/test_aws_root_inner.sh +source ./tests/util_file.sh +source ./tests/util_multipart.sh +source ./tests/util_tags.sh +source ./tests/commands/get_object.sh +source ./tests/commands/put_object.sh +source ./tests/commands/list_multipart_uploads.sh + +# abort-multipart-upload +@test "test_abort_multipart_upload" { + test_abort_multipart_upload_aws_root +} + +# complete-multipart-upload +@test "test_complete_multipart_upload" { + test_complete_multipart_upload_aws_root +} + +# create-multipart-upload +@test "test_create_multipart_upload_properties" { + test_create_multipart_upload_properties_aws_root +} + +# test multi-part upload list parts command +@test "test-multipart-upload-list-parts" { + test_multipart_upload_list_parts_aws_root +} + +# test listing of active uploads +@test "test-multipart-upload-list-uploads" { + local bucket_file_one="bucket-file-one" + local bucket_file_two="bucket-file-two" + + if [[ $RECREATE_BUCKETS == false ]]; then + run abort_all_multipart_uploads "$BUCKET_ONE_NAME" + assert_success + fi + + run create_test_files "$bucket_file_one" "$bucket_file_two" + assert_success + + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two" + assert_success +} + +@test "test-multipart-upload-from-bucket" { + local bucket_file="bucket-file" + + run create_test_file "$bucket_file" + assert_success + + run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 + assert_success + + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 + assert_success + + run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy" + assert_success + + run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file + assert_success +} + +@test "test_multipart_upload_from_bucket_range_too_large" { + local bucket_file="bucket-file" + run create_large_file "$bucket_file" + assert_success + + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" + assert_success +} + +@test "test_multipart_upload_from_bucket_range_valid" { + local bucket_file="bucket-file" + run create_large_file "$bucket_file" + assert_success + + run setup_bucket "aws" "$BUCKET_ONE_NAME" + assert_success + + run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file" + assert_success +} diff --git a/tests/test_s3api.sh b/tests/test_s3api_object.sh similarity index 62% rename from tests/test_s3api.sh rename to tests/test_s3api_object.sh index 402c666..fd283c7 100755 --- a/tests/test_s3api.sh +++ b/tests/test_s3api_object.sh @@ -20,7 +20,6 @@ source ./tests/util_aws.sh source ./tests/util_create_bucket.sh source ./tests/util_file.sh source ./tests/util_lock_config.sh -source ./tests/util_multipart.sh source ./tests/util_tags.sh source ./tests/util_users.sh source ./tests/test_aws_root_inner.sh @@ -38,7 +37,6 @@ source ./tests/commands/get_object_legal_hold.sh source ./tests/commands/get_object_lock_configuration.sh source ./tests/commands/get_object_retention.sh source ./tests/commands/get_object_tagging.sh -source ./tests/commands/list_multipart_uploads.sh source ./tests/commands/list_object_versions.sh source ./tests/commands/put_bucket_acl.sh source ./tests/commands/put_bucket_policy.sh @@ -52,16 +50,6 @@ source ./tests/commands/select_object_content.sh export RUN_USERS=true -# abort-multipart-upload -@test "test_abort_multipart_upload" { - test_abort_multipart_upload_aws_root -} - -# complete-multipart-upload -@test "test_complete_multipart_upload" { - test_complete_multipart_upload_aws_root -} - # copy-object @test "test_copy_object" { test_common_copy_object "s3api" @@ -71,35 +59,6 @@ export RUN_USERS=true copy_object_empty || fail "copy objects with no parameters test failure" } -# create-bucket -@test "test_create_delete_bucket_aws" { - test_common_create_delete_bucket "aws" -} - -@test "test_create_bucket_invalid_name" { - test_create_bucket_invalid_name_aws_root -} - -# create-multipart-upload -@test "test_create_multipart_upload_properties" { - test_create_multipart_upload_properties_aws_root -} - -# delete-bucket - test_create_delete_bucket_aws - -# delete-bucket-policy -@test "test_get_put_delete_bucket_policy" { - if [[ -n $SKIP_POLICY ]]; then - skip "will not test policy actions with SKIP_POLICY set" - fi - test_common_get_put_delete_bucket_policy "aws" -} - -# delete-bucket-tagging -@test "test-set-get-delete-bucket-tags" { - test_common_set_get_delete_bucket_tags "aws" -} - # delete-object - tested with bucket cleanup before or after tests # delete-object-tagging @@ -115,20 +74,6 @@ export RUN_USERS=true test_delete_objects_aws_root } -# get-bucket-acl -@test "test_get_bucket_acl" { - test_get_bucket_acl_aws_root -} - -# get-bucket-location -@test "test_get_bucket_location" { - test_common_get_bucket_location "aws" -} - -# get-bucket-policy - test_get_put_delete_bucket_policy - -# get-bucket-tagging - test_set_get_delete_bucket_tags - # get-object @test "test_get_object_full_range" { test_get_object_full_range_aws_root @@ -143,12 +88,6 @@ export RUN_USERS=true test_get_object_attributes_aws_root } -@test "test_head_bucket_invalid_name" { - if head_bucket "aws" ""; then - fail "able to get bucket info for invalid name" - fi -} - @test "test_put_object" { test_put_object_aws_root } @@ -168,11 +107,6 @@ export RUN_USERS=true test_common_put_object_no_data "aws" } -# test listing buckets on versitygw -@test "test_list_buckets" { - test_common_list_buckets "s3api" -} - # test listing a bucket's objects on versitygw @test "test_list_objects" { test_common_list_objects "aws" @@ -186,10 +120,6 @@ export RUN_USERS=true test_get_put_object_retention_aws_root } -@test "test_put_bucket_acl" { - test_common_put_bucket_acl "s3api" -} - # test v1 s3api list objects command @test "test-s3api-list-objects-v1" { test_s3api_list_objects_v1_aws_root @@ -205,77 +135,6 @@ export RUN_USERS=true test_common_set_get_object_tags "aws" } -# test multi-part upload list parts command -@test "test-multipart-upload-list-parts" { - test_multipart_upload_list_parts_aws_root -} - -# test listing of active uploads -@test "test-multipart-upload-list-uploads" { - local bucket_file_one="bucket-file-one" - local bucket_file_two="bucket-file-two" - - if [[ $RECREATE_BUCKETS == false ]]; then - run abort_all_multipart_uploads "$BUCKET_ONE_NAME" - assert_success - fi - - run create_test_files "$bucket_file_one" "$bucket_file_two" - assert_success - - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two" - assert_success -} - -@test "test-multipart-upload-from-bucket" { - local bucket_file="bucket-file" - - run create_test_file "$bucket_file" - assert_success - - run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 - assert_success - - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 - assert_success - - run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy" - assert_success - - run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file - assert_success -} - -@test "test_multipart_upload_from_bucket_range_too_large" { - local bucket_file="bucket-file" - run create_large_file "$bucket_file" - assert_success - - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" - assert_success -} - -@test "test_multipart_upload_from_bucket_range_valid" { - local bucket_file="bucket-file" - run create_large_file "$bucket_file" - assert_success - - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file" - assert_success -} - @test "test-presigned-url-utf8-chars" { test_common_presigned_url_utf8_chars "aws" } @@ -324,31 +183,10 @@ export RUN_USERS=true # [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" #} -@test "test_head_bucket" { - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info" - log 5 "INFO: $bucket_info" - region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" - [[ $region != "" ]] || fail "empty bucket region" - bucket_cleanup "aws" "$BUCKET_ONE_NAME" -} - @test "test_retention_bypass" { test_retention_bypass_aws_root } -@test "test_head_bucket_doesnt_exist" { - run setup_bucket "aws" "$BUCKET_ONE_NAME" - assert_success - - head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$? - [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" - [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" - bucket_cleanup "aws" "$BUCKET_ONE_NAME" -} - @test "test_add_object_metadata" { object_one="object-one" test_key="x-test-data" diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 0072f27..19a2ae6 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -109,10 +109,6 @@ export RUN_USERS=true # test_common_presigned_url_utf8_chars "s3cmd" #} -@test "test_list_objects_file_count" { - test_common_list_objects_file_count "s3cmd" -} - @test "test_get_bucket_info_s3cmd" { run setup_bucket "s3cmd" "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_s3cmd_file_count.sh b/tests/test_s3cmd_file_count.sh new file mode 100755 index 0000000..3dc492d --- /dev/null +++ b/tests/test_s3cmd_file_count.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/test_common.sh + +export RUN_S3CMD=true + +@test "test_list_objects_file_count" { + test_common_list_objects_file_count "s3cmd" +}