diff --git a/.github/workflows/docker-bats.yml b/.github/workflows/docker-bats.yml index b6eee52..dc5e592 100644 --- a/.github/workflows/docker-bats.yml +++ b/.github/workflows/docker-bats.yml @@ -14,8 +14,9 @@ jobs: run: | cp tests/.env.docker.default tests/.env.docker cp tests/.secrets.default tests/.secrets + # see https://github.com/versity/versitygw/issues/1034 docker build \ - --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \ + --build-arg="GO_LIBRARY=go1.21.13.linux-amd64.tar.gz" \ --build-arg="AWS_CLI=awscli-exe-linux-x86_64-2.22.35.zip" \ --build-arg="MC_FOLDER=linux-amd64" \ --progress=plain \ diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 1691042..6db54e6 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -73,16 +73,17 @@ jobs: RUN_SET: "s3api-user" RECREATE_BUCKETS: "false" BACKEND: "posix" - - set: "s3api, s3, multipart|object, non-static, folder IAM" - IAM_TYPE: folder - RUN_SET: "s3api-bucket,s3api-object,s3api-multipart" - RECREATE_BUCKETS: "true" - BACKEND: "s3" - - set: "s3api, s3, policy|user, non-static, folder IAM" - IAM_TYPE: folder - RUN_SET: "s3api-policy,s3api-user" - RECREATE_BUCKETS: "true" - BACKEND: "s3" + # TODO fix/debug s3 gateway + #- set: "s3api, s3, multipart|object, non-static, folder IAM" + # IAM_TYPE: folder + # RUN_SET: "s3api-bucket,s3api-object,s3api-multipart" + # RECREATE_BUCKETS: "true" + # BACKEND: "s3" + #- set: "s3api, s3, policy|user, non-static, folder IAM" + # IAM_TYPE: folder + # RUN_SET: "s3api-policy,s3api-user" + # RECREATE_BUCKETS: "true" + # BACKEND: "s3" - set: "s3cmd, posix, file count, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3cmd-file-count" @@ -132,6 +133,14 @@ jobs: run: | sudo apt-get install libxml2-utils + # see https://github.com/versity/versitygw/issues/1034 + - name: Install AWS cli + run: | + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" + unzip -o awscliv2.zip + ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - name: Build and run env: IAM_TYPE: ${{ matrix.IAM_TYPE }} @@ -170,6 +179,7 @@ jobs: export AWS_REGION=us-east-1 export AWS_ACCESS_KEY_ID_TWO=user export AWS_SECRET_ACCESS_KEY_TWO=pass + export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity aws configure set aws_region $AWS_REGION --profile versity diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 7ab32f2..5f13469 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -3,8 +3,9 @@ FROM ubuntu:latest ARG DEBIAN_FRONTEND=noninteractive ARG SECRETS_FILE=tests/.secrets ARG CONFIG_FILE=tests/.env.docker -ARG GO_LIBRARY=go1.23.1.linux-arm64.tar.gz -ARG AWS_CLI=awscli-exe-linux-aarch64.zip +ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz +# see https://github.com/versity/versitygw/issues/1034 +ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip ARG MC_FOLDER=linux-arm64 ENV TZ=Etc/UTC @@ -85,6 +86,7 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen ENV WORKSPACE=. ENV VERSITYGW_TEST_ENV=$CONFIG_FILE +#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED ENTRYPOINT ["tests/run.sh"] CMD ["s3api,s3,s3cmd,mc,rest"] diff --git a/tests/remove_static.sh b/tests/remove_static.sh index d459261..9f1fe89 100755 --- a/tests/remove_static.sh +++ b/tests/remove_static.sh @@ -28,26 +28,23 @@ delete_bucket_if_exists() { return 1 fi if [[ $exists_result -eq 1 ]]; then - log 5 "bucket '$2' doesn't exist, skipping" + echo "bucket '$2' doesn't exist, skipping" return 0 fi if ! delete_bucket_recursive "$1" "$2"; then log 2 "error deleting bucket" return 1 fi - log 5 "bucket '$2' successfully deleted" + echo "bucket '$2' successfully deleted" return 0 } -if ! setup; then - log 2 "error starting versity to set up static buckets" - exit 1 -fi -if ! delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then +base_setup +if ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then log 2 "error deleting static bucket one" -elif ! delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then +elif ! RECREATE_BUCKETS=true delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then log 2 "error deleting static bucket two" fi -if ! teardown; then +if ! stop_versity; then log 2 "error stopping versity" fi \ No newline at end of file diff --git a/tests/rest_scripts/copy_object.sh b/tests/rest_scripts/copy_object.sh new file mode 100755 index 0000000..1f72db7 --- /dev/null +++ b/tests/rest_scripts/copy_object.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +copy_source="$COPY_SOURCE" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="PUT +/$bucket_name/$key + +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-copy-source:$copy_source +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-copy-source;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date,Signature=$signature\"") +curl_command+=(-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-copy-source: $copy_source\"" +-H "\"x-amz-date: $current_date_time\"") +curl_command+=(-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/setup_static.sh b/tests/setup_static.sh index 0bd2483..90e491b 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -29,14 +29,14 @@ create_bucket_if_not_exists() { return 1 fi if [[ $exists_result -eq 0 ]]; then - log 5 "bucket '$2' already exists, skipping" + echo "bucket '$2' already exists, skipping" return 0 fi if ! create_bucket_object_lock_enabled "$2"; then log 2 "error creating bucket" return 1 fi - log 5 "bucket '$2' successfully created" + echo "bucket '$2' successfully created" return 0 } diff --git a/tests/teardown_static.sh b/tests/teardown_static.sh deleted file mode 100755 index 569253a..0000000 --- a/tests/teardown_static.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source ./tests/setup.sh -source ./tests/util/util_object.sh - -if ! base_setup; then - log 2 "error starting versity to set up static buckets" - exit 1 -fi -if ! delete_bucket_recursive "s3api" "$BUCKET_ONE_NAME"; then - log 2 "error deleting static bucket one" -elif ! delete_bucket_recursive "s3api" "$BUCKET_TWO_NAME"; then - log 2 "error deleting static bucket two" -else - log 4 "buckets deleted successfully" -fi -if ! stop_versity; then - log 2 "error stopping versity" -fi \ No newline at end of file diff --git a/tests/test_common.sh b/tests/test_common.sh index 3329275..d958932 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -303,7 +303,8 @@ test_common_presigned_url_utf8_chars() { run create_test_file "$bucket_file" assert_success - dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 || fail "error creating test file" + run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 + assert_success run setup_bucket "$1" "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index ae78a2a..ed905b9 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -72,7 +72,7 @@ export RUN_USERS=true # delete-objects @test "test_delete_objects" { if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/888" + skip "https://github.com/versity/versitygw/issues/1029" fi test_delete_objects_s3api_root } @@ -131,14 +131,14 @@ export RUN_USERS=true # test adding and removing an object on versitygw @test "test_put_object_with_data" { if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/888" + skip "https://github.com/versity/versitygw/issues/1029" fi test_common_put_object_with_data "s3api" } @test "test_put_object_no_data" { if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/888" + skip "https://github.com/versity/versitygw/issues/1029" fi test_common_put_object_no_data "s3api" } @@ -236,3 +236,54 @@ export RUN_USERS=true test_common_ls_directory_object "s3api" } +@test "directory objects can't contain data" { + if [ "$DIRECT" == "true" ]; then + skip + fi + test_file="a" + + run create_test_file "$test_file" + assert_success + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file/" + assert_failure + assert_output -p "Directory object contains data payload" +} + +#@test "objects containing data can't be copied to directory objects" { +# # TODO finish test after https://github.com/versity/versitygw/issues/1021 +# skip "https://github.com/versity/versitygw/issues/1021" +# test_file="a" +# +# run create_test_file "$test_file" 0 +# assert_success +# +# run setup_bucket "s3api" "$BUCKET_ONE_NAME" +# assert_success +# +# run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" +# assert_success +# +# if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$test_file/" COPY_SOURCE="$BUCKET_ONE_NAME/$test_file" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/copy_object.sh); then +# log 2 "error listing multipart upload parts: $result" +# return 1 +# fi +# if [ "$result" != "400" ]; then +# log 2 "response code '$result': $(cat "$TEST_FILE_FOLDER/result.txt")" +# return 1 +# fi +# return 0 +#} + +@test "directory object - create multipart upload" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run create_multipart_upload "$BUCKET_ONE_NAME" "test_file/" + assert_failure + assert_output -p "Directory object contains data payload" +} +