From 4cd2635797720cac97f0bcc41a05c23a291ff811 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Mon, 23 Sep 2024 18:40:07 -0300 Subject: [PATCH] test: REST retention, versioning testing, misc cleanup --- tests/.env.default | 1 + tests/commands/delete_object.sh | 24 +++ tests/commands/get_bucket_versioning.sh | 51 +++++- .../commands/get_object_lock_configuration.sh | 40 +++++ tests/commands/get_object_retention.sh | 48 +++++- tests/commands/list_object_versions.sh | 36 +++- tests/commands/put_bucket_versioning.sh | 51 ++++++ tests/commands/put_object_retention.sh | 55 +++++- tests/env.sh | 18 ++ tests/rest_scripts/get_bucket_versioning.sh | 75 ++++++++ tests/rest_scripts/list_buckets.sh | 73 ++++++++ tests/rest_scripts/put_bucket_versioning.sh | 82 +++++++++ tests/rest_scripts/put_object_retention.sh | 90 ++++++++++ tests/rest_scripts/rest.sh | 33 ++++ tests/test_rest.sh | 114 +++++++++++++ tests/test_s3.sh | 2 - tests/test_user_common.sh | 43 ++--- tests/util.sh | 52 ++++-- tests/util_list_buckets.sh | 19 ++- tests/util_lock_config.sh | 47 +++++ tests/util_rest.sh | 1 + tests/util_time.sh | 18 ++ tests/util_users.sh | 12 ++ tests/util_versioning.sh | 160 ++++++++++++++++++ tests/versity.sh | 7 +- 25 files changed, 1097 insertions(+), 55 deletions(-) create mode 100755 tests/rest_scripts/get_bucket_versioning.sh create mode 100755 tests/rest_scripts/list_buckets.sh create mode 100755 tests/rest_scripts/put_bucket_versioning.sh create mode 100755 tests/rest_scripts/put_object_retention.sh create mode 100644 tests/rest_scripts/rest.sh create mode 100644 tests/util_time.sh create mode 100644 tests/util_versioning.sh diff --git a/tests/.env.default b/tests/.env.default index e83e675..ce4a71f 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -27,3 +27,4 @@ USERNAME_TWO=HIJKLMN PASSWORD_TWO=OPQRSTU TEST_FILE_FOLDER=$PWD/versity-gwtest-files REMOVE_TEST_FILE_FOLDER=true +VERSIONING_DIR=/tmp/versioning diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index b086415..e47b27b 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -58,6 +58,30 @@ delete_object_bypass_retention() { return 0 } +delete_object_version() { + if [[ $# -ne 3 ]]; then + log 2 "'delete_object_version' requires bucket, key, version ID" + return 1 + fi + if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then + log 2 "error deleting object version: $delete_object_error" + return 1 + fi + return 0 +} + +delete_object_version_bypass_retention() { + if [[ $# -ne 3 ]]; then + log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID" + return 1 + fi + if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then + log 2 "error deleting object version with bypass retention: $delete_object_error" + return 1 + fi + return 0 +} + delete_object_with_user() { record_command "delete-object" "client:$1" if [ $# -ne 5 ]; then diff --git a/tests/commands/get_bucket_versioning.sh b/tests/commands/get_bucket_versioning.sh index 4ae3c61..5c81bd9 100644 --- a/tests/commands/get_bucket_versioning.sh +++ b/tests/commands/get_bucket_versioning.sh @@ -17,15 +17,60 @@ get_bucket_versioning() { record_command "get-bucket-versioning" "client:s3api" if [[ $# -ne 2 ]]; then - log 2 "put bucket versioning command requires command type, bucket name" + log 2 "get bucket versioning command requires command type, bucket name" return 1 fi local get_result=0 if [[ $1 == 's3api' ]]; then - error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$? + versioning=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$? fi if [[ $get_result -ne 0 ]]; then - log 2 "error getting bucket versioning: $error" + log 2 "error getting bucket versioning: $versioning" + return 1 + fi + return 0 +} + +get_bucket_versioning_rest() { + log 6 "get_object_rest" + if [ $# -ne 1 ]; then + log 2 "'get_bucket_versioning_rest' requires bucket name" + return 1 + fi + + #generate_hash_for_payload "" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + # shellcheck disable=SC2154 + canonical_request="GET +/$1 +versioning= +host:$aws_endpoint_url_address +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + # shellcheck disable=SC2154 + if ! reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?versioning" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -H "x-amz-date: $current_date_time" \ + -o "$TEST_FILE_FOLDER/versioning.txt" 2>&1); then + log 2 "error retrieving curl reply: $reply" + return 1 + fi + log 5 "reply: $reply" + if [[ "$reply" != "200" ]]; then + log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/versioning.txt")" return 1 fi return 0 diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index c8c7258..f115dd7 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -28,4 +28,44 @@ get_object_lock_configuration() { fi lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning") return 0 +} + +get_object_lock_configuration_rest() { + log 6 "get_object_lock_configuration_rest" + if [ $# -ne 1 ]; then + log 2 "'get_object_lock_configuration_rest' requires bucket name" + return 1 + fi + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + # shellcheck disable=SC2154 + canonical_request="GET +/$1 +object-lock= +host:$aws_endpoint_url_address +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + # shellcheck disable=SC2154 + reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -H "x-amz-date: $current_date_time" \ + -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1) + log 5 "reply: $reply" + if [[ "$reply" != "200" ]]; then + log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/commands/get_object_retention.sh b/tests/commands/get_object_retention.sh index 82fc122..83daf58 100644 --- a/tests/commands/get_object_retention.sh +++ b/tests/commands/get_object_retention.sh @@ -27,4 +27,50 @@ get_object_retention() { return 1 fi return 0 -} \ No newline at end of file +} + +get_object_retention_rest() { + if [ $# -ne 2 ]; then + log 2 "'get_object_tagging_rest' requires bucket, key" + return 1 + fi + + generate_hash_for_payload "" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + # shellcheck disable=SC2154 + canonical_request="GET +/$1/$2 +retention= +host:$aws_endpoint_url_address +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + # shellcheck disable=SC2154 + reply=$(curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -o "$TEST_FILE_FOLDER"/object_retention.txt 2>&1) + log 5 "reply status code: $reply" + if [[ "$reply" != "200" ]]; then + if [ "$reply" == "404" ]; then + return 1 + fi + log 2 "reply error: $reply" + log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)" + return 2 + fi + log 5 "object tags: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)" + return 0 +} diff --git a/tests/commands/list_object_versions.sh b/tests/commands/list_object_versions.sh index ef4affd..5da7135 100644 --- a/tests/commands/list_object_versions.sh +++ b/tests/commands/list_object_versions.sh @@ -20,10 +20,44 @@ list_object_versions() { log 2 "'list object versions' command requires bucket name" return 1 fi - versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1") || local list_result=$? + versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$? if [[ $list_result -ne 0 ]]; then log 2 "error listing object versions: $versions" return 1 fi + versions=$(echo "$versions" | grep -v "InsecureRequestWarning") return 0 +} + +list_object_versions_rest() { + if [ $# -ne 1 ]; then + log 2 "'list_object_versions_rest' requires bucket name" + return 1 + fi + generate_hash_for_payload "" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + # shellcheck disable=SC2154 + canonical_request="GET +/$1 +versions= +host:${AWS_ENDPOINT_URL#*//} +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + + get_signature + # shellcheck disable=SC2034,SC2154 + reply=$(curl -ks "$AWS_ENDPOINT_URL/$1?versions" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1) } \ No newline at end of file diff --git a/tests/commands/put_bucket_versioning.sh b/tests/commands/put_bucket_versioning.sh index a8881cc..c5ba774 100644 --- a/tests/commands/put_bucket_versioning.sh +++ b/tests/commands/put_bucket_versioning.sh @@ -29,4 +29,55 @@ put_bucket_versioning() { return 1 fi return 0 +} + +put_bucket_versioning_rest() { + if [ $# -ne 2 ]; then + log 2 "'put_bucket_versioning_rest' requires bucket, 'Enabled' or 'Suspended'" + return 1 + fi + + versioning=" + + $2 +" + + generate_hash_for_payload "$versioning" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + content_md5=$(echo -n "$versioning" | openssl dgst -binary -md5 | openssl base64) + # shellcheck disable=SC2154 + canonical_request="PUT +/$1 +versioning= +content-md5:$content_md5 +host:$aws_endpoint_url_address +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +content-md5;host;x-amz-content-sha256;x-amz-date +$payload_hash" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + + # shellcheck disable=SC2154 + reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1?versioning" \ + -H "Content-MD5: $content_md5" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -d "$versioning" -o "$TEST_FILE_FOLDER"/put_versioning_error.txt 2>&1) + log 5 "reply status code: $reply" + if [[ "$reply" != "200" ]]; then + log 2 "reply error: $reply" + log 2 "put bucket versioning command returned error: $(cat "$TEST_FILE_FOLDER"/put_versioning_error.txt)" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/commands/put_object_retention.sh b/tests/commands/put_object_retention.sh index 6f50300..aa6ed76 100644 --- a/tests/commands/put_object_retention.sh +++ b/tests/commands/put_object_retention.sh @@ -26,4 +26,57 @@ put_object_retention() { return 1 fi return 0 -} \ No newline at end of file +} + +put_object_retention_rest() { + if [ $# -ne 4 ]; then + log 2 "'put_object_retention_rest' requires bucket, key, retention mode, retention date" + return 1 + fi + + retention=" + + $3 + $4 +" + + log 5 "retention payload: $retention" + generate_hash_for_payload "$retention" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + content_md5=$(echo -n "$retention" | openssl dgst -binary -md5 | openssl base64) + # shellcheck disable=SC2154 + canonical_request="PUT +/$1/$2 +retention= +content-md5:$content_md5 +host:$aws_endpoint_url_address +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +content-md5;host;x-amz-content-sha256;x-amz-date +$payload_hash" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + + # shellcheck disable=SC2154 + reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?retention" \ + -H "Content-MD5: $content_md5" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -d "$retention" -o "$TEST_FILE_FOLDER"/put_object_retention_error.txt 2>&1) + log 5 "reply status code: $reply" + if [[ "$reply" != "200" ]]; then + log 2 "reply error: $reply" + log 2 "put object retention command returned error: $(cat "$TEST_FILE_FOLDER"/put_object_retention_error.txt)" + return 1 + fi + return 0 +} diff --git a/tests/env.sh b/tests/env.sh index 7562fb0..f9a3634 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -120,6 +120,12 @@ check_universal_vars() { log 1 "TEST_FILE_FOLDER missing" exit 1 fi + if [ ! -d "$TEST_FILE_FOLDER" ]; then + if ! error=$(mkdir -p "$TEST_FILE_FOLDER"); then + log 2 "error creating test folder: $error" + exit 1 + fi + fi # exporting these since they're needed for subshells export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL } @@ -129,6 +135,18 @@ check_versity_vars() { log 1 "LOCAL_FOLDER missing" exit 1 fi + if [ ! -d "$LOCAL_FOLDER" ]; then + if ! error=$(mkdir -p "$LOCAL_FOLDER"); then + log 2 "error creating local posix folder: $error" + exit 1 + fi + fi + if [ -n "$VERSIONING_DIR" ] && [ ! -d "$VERSIONING_DIR" ]; then + if ! error=$(mkdir -p "$VERSIONING_DIR"); then + log 2 "error creating versioning folder: $error" + return 1 + fi + fi if [ -z "$VERSITY_EXE" ]; then log 1 "VERSITY_EXE missing" exit 1 diff --git a/tests/rest_scripts/get_bucket_versioning.sh b/tests/rest_scripts/get_bucket_versioning.sh new file mode 100755 index 0000000..98ac90b --- /dev/null +++ b/tests/rest_scripts/get_bucket_versioning.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +payload="" +# shellcheck disable=SC2153 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +get_host +get_aws_region + +# Step 1: generate payload hash + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" + +# Step 2: generate canonical hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="GET +/$bucket_name +versioning= +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +# Step 3: create STS data string + +year_month_day="$(echo "$current_date_time" | cut -c1-8)" + +sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + +# Step 4: generate signature + +date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') +date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') +date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') +signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') +signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + +# Step 5: send curl command + +curl -ks "https://$host/$BUCKET_NAME?versioning" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ No newline at end of file diff --git a/tests/rest_scripts/list_buckets.sh b/tests/rest_scripts/list_buckets.sh new file mode 100755 index 0000000..aa43130 --- /dev/null +++ b/tests/rest_scripts/list_buckets.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +payload="" +# shellcheck disable=SC2153 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" +get_host +get_aws_region + +# Step 1: generate payload hash + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" + +# Step 2: generate canonical hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="GET +/ + +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +# Step 3: create STS data string + +year_month_day="$(echo "$current_date_time" | cut -c1-8)" + +sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + +# Step 4: generate signature + +date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') +date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') +date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') +signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') +signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + +# Step 5: send curl command + +curl -ks "https://$host/" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ No newline at end of file diff --git a/tests/rest_scripts/put_bucket_versioning.sh b/tests/rest_scripts/put_bucket_versioning.sh new file mode 100755 index 0000000..9d52c6b --- /dev/null +++ b/tests/rest_scripts/put_bucket_versioning.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +status="$STATUS" +# shellcheck disable=SC2153 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +get_host +get_aws_region + +# Step 1: generate payload hash + +payload=" + + $status +" + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" + +# Step 2: generate canonical hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="PUT +/$bucket_name +versioning= +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +# Step 3: create STS data string + +year_month_day="$(echo "$current_date_time" | cut -c1-8)" + +sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + +# Step 4: generate signature + +date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') +date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') +date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') +signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') +signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + +# Step 5: send curl command + +curl -ks -X PUT "https://$host/$BUCKET_NAME?versioning" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -d "$payload" \ No newline at end of file diff --git a/tests/rest_scripts/put_object_retention.sh b/tests/rest_scripts/put_object_retention.sh new file mode 100755 index 0000000..f54aa25 --- /dev/null +++ b/tests/rest_scripts/put_object_retention.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +mode="$RETENTION_MODE" +# shellcheck disable=SC2153 +retain_until_date="$RETAIN_UNTIL_DATE" +# shellcheck disable=SC2153 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" +get_host +get_aws_region + +# Step 1: generate payload hash + +payload=" + + $mode + $retain_until_date +" + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" +content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64) + +# Step 2: generate canonical hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="PUT +/$bucket_name/$key +retention= +content-md5:$content_md5 +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +content-md5;host;x-amz-content-sha256;x-amz-date +$payload_hash" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +# Step 3: create STS data string + +year_month_day="$(echo "$current_date_time" | cut -c1-8)" + +sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + +# Step 4: generate signature + +date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') +date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') +date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') +signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') +signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + +# Step 5: send curl command + +curl -ks -X PUT "https://$host/$bucket_name/$key?retention" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "Content-MD5: $content_md5" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -d "$payload" diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh new file mode 100644 index 0000000..ef1e632 --- /dev/null +++ b/tests/rest_scripts/rest.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_host() { + if [ -z "$HOST" ]; then + host="localhost:7070" + return + fi + # shellcheck disable=SC2034 + host="$HOST" +} + +get_aws_region() { + if [ -z "$AWS_REGION" ]; then + aws_region="us-east-1" + return + fi + # shellcheck disable=SC2034 + aws_region="$AWS_REGION" +} diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 57df1f5..21bfc6b 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -1,17 +1,26 @@ #!/usr/bin/env bats source ./tests/commands/delete_object_tagging.sh +source ./tests/commands/get_bucket_versioning.sh source ./tests/commands/get_object.sh +source ./tests/commands/get_object_lock_configuration.sh +source ./tests/commands/get_object_retention.sh source ./tests/commands/list_buckets.sh +source ./tests/commands/list_object_versions.sh +source ./tests/commands/put_bucket_versioning.sh source ./tests/commands/put_object.sh +source ./tests/commands/put_object_retention.sh source ./tests/commands/put_object_tagging.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util.sh source ./tests/util_list_buckets.sh source ./tests/util_list_objects.sh +source ./tests/util_lock_config.sh source ./tests/util_rest.sh source ./tests/util_tags.sh +source ./tests/util_time.sh +source ./tests/util_versioning.sh @test "test_rest_list_objects" { run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -86,3 +95,108 @@ source ./tests/util_tags.sh run verify_no_object_tags "rest" "$BUCKET_ONE_NAME" "$test_file" assert_success } + +@test "test_rest_retention" { + test_file="test_file" + test_key="TestKey" + test_value="TestValue" + + run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + assert_success + # in static bucket config, bucket will still exist + if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then + run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" + assert_success + fi + + run create_test_files "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + if ! five_seconds_later=$(get_time_seconds_in_future 5 "%z"); then + log 2 "error getting future time" + return 1 + fi + log 5 "later: $five_seconds_later" + run put_object_retention_rest "$BUCKET_ONE_NAME" "$test_file" "GOVERNANCE" "$five_seconds_later" + assert_success +} + +@test "test_rest_set_get_versioning" { + skip "https://github.com/versity/versitygw/issues/866" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + log 5 "get versioning" + + run check_versioning_status_rest "$BUCKET_ONE_NAME" "" + assert_success + + run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended" + assert_success + + run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended" + assert_success +} + +@test "test_rest_set_get_lock_config" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run check_no_object_lock_config_rest "$BUCKET_ONE_NAME" + assert_success + + run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + assert_success + + # in static bucket config, bucket will still exist + if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then + run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" + assert_success + fi + + run check_object_lock_config_enabled_rest "$BUCKET_ONE_NAME" + assert_success +} + +@test "test_rest_versioning" { + skip "https://github.com/versity/versitygw/issues/864" + test_file="test_file" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + if [ "$DIRECT" == "true" ]; then + sleep 10 + fi + + run create_test_file "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true" + assert_success + + run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true" + assert_success +} diff --git a/tests/test_s3.sh b/tests/test_s3.sh index b6acec3..1e0a211 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -61,8 +61,6 @@ source ./tests/util_file.sh fi run setup_bucket "s3" "$BUCKET_ONE_NAME" assert_success - - delete_bucket "s3" "$BUCKET_ONE_NAME" || fail "error deleting bucket" } @test "test_ls_directory_object" { diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index f451557..75fa401 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -18,6 +18,7 @@ source ./tests/setup.sh source ./tests/util_users.sh source ./tests/util.sh source ./tests/util_create_bucket.sh +source ./tests/util_list_buckets.sh source ./tests/commands/list_buckets.sh test_admin_user() { @@ -30,40 +31,32 @@ test_admin_user() { user_username="$USERNAME_TWO" user_password="$PASSWORD_TWO" - setup_user "$admin_username" "$admin_password" "admin" || fail "error setting up admin user" + run setup_user "$admin_username" "$admin_password" "admin" + assert_success if user_exists "$user_username"; then - delete_user "$user_username" || fail "failed to delete user '$user_username'" + run delete_user "$user_username" + assert_success fi - create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" || fail "failed to create user '$user_username'" + run create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" + assert_success run setup_bucket "aws" "$BUCKET_ONE_NAME" assert_success - delete_bucket_or_contents_if_exists "aws" "versity-gwtest-admin-bucket" - create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || fail "error creating bucket with admin user" - - bucket_one_found=false - bucket_two_found=false - list_buckets_with_user "aws" "$admin_username" "$admin_password" || fail "error listing buckets with admin user" - # shellcheck disable=SC2154 - for bucket in "${bucket_array[@]}"; do - if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then - bucket_one_found=true - elif [ "$bucket" == "versity-gwtest-admin-bucket" ]; then - bucket_two_found=true - fi - if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then - break - fi - done - if [ $bucket_one_found == false ] || [ $bucket_two_found == false ]; then - fail "not all expected buckets listed" + if [ "$RECREATE_BUCKETS" == "true" ]; then + run create_bucket_with_user "s3api" "$BUCKET_TWO_NAME" "$admin_username" "$admin_password" + assert_success + else + run change_bucket_owner "$admin_username" "$admin_password" "$BUCKET_TWO_NAME" "$admin_username" + assert_success fi - change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || fail "error changing bucket owner" - run delete_bucket "aws" "versity-gwtest-admin-bucket" - assert_success "failed to delete bucket" + run list_and_check_buckets_with_user "aws" "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" "$admin_username" "$admin_password" + assert_success + + run change_bucket_owner "$admin_username" "$admin_password" "$BUCKET_TWO_NAME" "$user_username" + assert_success delete_user "$user_username" delete_user "$admin_username" diff --git a/tests/util.sh b/tests/util.sh index 9e269d7..1293dff 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -17,6 +17,7 @@ source ./tests/util_create_bucket.sh source ./tests/util_mc.sh source ./tests/util_multipart.sh +source ./tests/util_versioning.sh source ./tests/logger.sh source ./tests/commands/abort_multipart_upload.sh source ./tests/commands/complete_multipart_upload.sh @@ -29,6 +30,7 @@ source ./tests/commands/get_bucket_acl.sh source ./tests/commands/get_bucket_ownership_controls.sh source ./tests/commands/get_bucket_policy.sh source ./tests/commands/get_bucket_tagging.sh +source ./tests/commands/get_object_legal_hold.sh source ./tests/commands/get_object_lock_configuration.sh source ./tests/commands/get_object_tagging.sh source ./tests/commands/head_bucket.sh @@ -38,6 +40,7 @@ source ./tests/commands/list_objects.sh source ./tests/commands/list_parts.sh source ./tests/commands/put_bucket_acl.sh source ./tests/commands/put_bucket_ownership_controls.sh +source ./tests/commands/put_bucket_policy.sh source ./tests/commands/put_object_legal_hold.sh source ./tests/commands/put_object_lock_configuration.sh source ./tests/commands/upload_part_copy.sh @@ -149,6 +152,12 @@ list_and_delete_objects() { return 1 fi done + + if ! delete_old_versions "$1"; then + log 2 "error deleting old version" + return 1 + fi + return 0 } # param: bucket name @@ -173,28 +182,23 @@ check_ownership_rule_and_reset_acl() { fi } -# param: bucket name -# return 0 for success, 1 for error -check_and_disable_object_lock_config() { +check_object_lock_config() { if [ $# -ne 1 ]; then - log 2 "'check_and_disable_object_lock_config' requires bucket name" + log 2 "'check_object_lock_config' requires bucket name" return 1 fi - - local lock_config_exists=true + lock_config_exists=true if ! get_object_lock_configuration "$1"; then # shellcheck disable=SC2154 if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then + # shellcheck disable=SC2034 lock_config_exists=false else log 2 "error getting object lock config" return 1 fi fi - if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then - log 2 "error disabling object lock config" - return 1 - fi + return 0 } # restore bucket to pre-test state (or prep for deletion) @@ -214,6 +218,11 @@ clear_bucket_s3api() { fi fi + if ! check_object_lock_config "$1"; then + log 2 "error checking object lock config" + return 1 + fi + if ! list_and_delete_objects "$1"; then log 2 "error listing and deleting objects" return 1 @@ -227,8 +236,8 @@ clear_bucket_s3api() { #run check_ownership_rule_and_reset_acl "$1" #assert_success "error checking ownership rule and resetting acl" - if ! check_and_disable_object_lock_config "$1"; then - log 2 "error checking and disabling object lock config" + if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then + log 2 "error disabling object lock config" return 1 fi @@ -258,18 +267,18 @@ clear_object_in_bucket() { } # params: bucket, object, possible WORM error after deletion attempt -# return 0 for success, 1 for error +# return 0 for success, 1 for no WORM protection, 2 for error check_for_and_remove_worm_protection() { if [ $# -ne 3 ]; then log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error" - return 1 + return 2 fi if [[ $3 == *"WORM"* ]]; then log 5 "WORM protection found" if ! put_object_legal_hold "$1" "$2" "OFF"; then log 2 "error removing object legal hold" - return 1 + return 2 fi sleep 1 if [[ $LOG_LEVEL_INT -ge 5 ]]; then @@ -277,12 +286,15 @@ check_for_and_remove_worm_protection() { fi if ! add_governance_bypass_policy "$1"; then log 2 "error adding new governance bypass policy" - return 1 + return 2 fi if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then log 2 "error deleting object after legal hold removal" - return 1 + return 2 fi + else + log 5 "no WORM protection found" + return 1 fi return 0 } @@ -423,6 +435,12 @@ delete_bucket_or_contents() { log 2 "error aborting all multipart uploads" return 1 fi + + if [ "$RUN_USERS" == "true" ] && ! reset_bucket_owner "$2"; then + log 2 "error resetting bucket owner" + return 1 + fi + log 5 "bucket contents, policy, ACL deletion success" return 0 fi diff --git a/tests/util_list_buckets.sh b/tests/util_list_buckets.sh index 2735359..116d0b2 100644 --- a/tests/util_list_buckets.sh +++ b/tests/util_list_buckets.sh @@ -35,12 +35,12 @@ list_check_buckets_rest() { return 0 } -list_and_check_buckets() { - if [ $# -ne 3 ]; then - log 2 "'list_and_check_buckets' requires client, two bucket names" +list_and_check_buckets_with_user() { + if [ $# -ne 5 ]; then + log 2 "'list_and_check_buckets' requires client, two bucket names, id, key" return 1 fi - if ! list_buckets "$1"; then + if ! list_buckets_with_user "$1" "$4" "$5"; then log 2 "error listing buckets" return 1 fi @@ -68,4 +68,15 @@ list_and_check_buckets() { return 1 fi return 0 +} + +list_and_check_buckets() { + if [ $# -ne 3 ]; then + log 2 "'list_and_check_buckets' requires client, two bucket names" + fi + if ! list_and_check_buckets_with_user "$1" "$2" "$3" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then + log 2 "error listing and checking buckets" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/util_lock_config.sh b/tests/util_lock_config.sh index 2bb2420..02b310b 100644 --- a/tests/util_lock_config.sh +++ b/tests/util_lock_config.sh @@ -84,3 +84,50 @@ get_check_object_lock_config_enabled() { fi return 0 } + +check_no_object_lock_config_rest() { + if [ $# -ne 1 ]; then + log 2 "'get_check_object_lock_config_rest' requires bucket name" + return 1 + fi + if get_object_lock_configuration_rest "$1"; then + log 2 "object lock config should be missing" + return 1 + fi + log 5 "object lock config: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" + # shellcheck disable=SC2154 + if [[ "$reply" != "404" ]]; then + log 2 "incorrect response code: $reply" + return 1 + fi + if ! error=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1); then + log 2 "error getting object lock config error: $error" + return 1 + fi + if [[ "$error" != "ObjectLockConfigurationNotFoundError" ]]; then + log 2 "unexpected error: $error" + return 1 + fi + return 0 +} + +check_object_lock_config_enabled_rest() { + if [ $# -ne 1 ]; then + log 2 "'get_check_object_lock_config_rest' requires bucket name" + return 1 + fi + if ! get_object_lock_configuration_rest "$1"; then + log 2 "error getting object lock config" + return 1 + fi + log 5 "object lock config: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")" + if ! enabled=$(xmllint --xpath '//*[local-name()="ObjectLockEnabled"]/text()' "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1); then + log 2 "error getting object lock config enabled value: $enabled" + return 1 + fi + if [[ "$enabled" != "Enabled" ]]; then + log 2 "expected 'Enabled', is $enabled" + return 1 + fi + return 0 +} diff --git a/tests/util_rest.sh b/tests/util_rest.sh index 8ddd71f..4f29de5 100644 --- a/tests/util_rest.sh +++ b/tests/util_rest.sh @@ -2,6 +2,7 @@ parse_bucket_list() { # shellcheck disable=SC2154 + log 5 "bucket list: $reply" bucket_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' -) bucket_array=() while read -r bucket; do diff --git a/tests/util_time.sh b/tests/util_time.sh new file mode 100644 index 0000000..cb9dab3 --- /dev/null +++ b/tests/util_time.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +get_time_seconds_in_future() { + if [ $# -lt 1 ] || [ $# -gt 2 ]; then + log 2 "'get_time_seconds_in_future' requires seconds, zone format (optional)" + return 1 + fi + os_name="$(uname)" + if [[ "$os_name" == "Darwin" ]]; then + now=$(date -u +"%Y-%m-%dT%H:%M:%S$2") + later=$(date -j -v "+${1}S" -f "%Y-%m-%dT%H:%M:%S$2" "$now" +"%Y-%m-%dT%H:%M:%S$2") + else + now=$(date +"%Y-%m-%dT%H:%M:%S$2") + # shellcheck disable=SC2034 + later=$(date -d "$now $1 seconds" +"%Y-%m-%dT%H:%M:%S$2") + fi + echo "$later" +} diff --git a/tests/util_users.sh b/tests/util_users.sh index f0deddb..f14d700 100644 --- a/tests/util_users.sh +++ b/tests/util_users.sh @@ -339,7 +339,19 @@ change_bucket_owner_direct() { echo "change bucket owner command requires ID, key, bucket name, and new owner" return 1 fi + # TODO add +} +reset_bucket_owner() { + if [ $# -ne 1 ]; then + log 2 "'reset_bucket_owner' requires bucket name" + return 1 + fi + if ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then + log 2 "error changing bucket owner back to root" + return 1 + fi + return 0 } change_bucket_owner() { diff --git a/tests/util_versioning.sh b/tests/util_versioning.sh new file mode 100644 index 0000000..08754b6 --- /dev/null +++ b/tests/util_versioning.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +source ./tests/commands/get_bucket_versioning.sh +source ./tests/commands/list_object_versions.sh + +check_if_versioning_enabled() { + if [ $# -ne 1 ]; then + log 2 "'check_if_versioning_enabled' requires bucket name" + return 2 + fi + if ! get_bucket_versioning 's3api' "$1"; then + log 2 "error getting bucket versioning" + return 2 + fi + # shellcheck disable=SC2154 + if ! status=$(echo "$versioning" | grep -v "InsecureRequestWarning" | jq -r ".Status" 2>&1); then + log 2 "error parsing 'Status' value: $status" + return 2 + fi + if [[ "$status" == "Enabled" ]]; then + return 0 + fi + return 1 +} + +delete_old_versions() { + if [ $# -ne 1 ]; then + log 2 "'delete_old_versions' requires bucket name" + return 1 + fi + if ! list_object_versions "$1"; then + log 2 "error listing object versions" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "versions: $versions" + version_keys=() + version_ids=() + + if ! parse_version_data "$versions" '.Versions[]'; then + log 2 "error parsing Versions elements" + return 1 + fi + if ! parse_version_data "$versions" '.DeleteMarkers[]'; then + log 2 "error getting DeleteMarkers elements" + return 1 + fi + + log 5 "version keys: ${version_keys[*]}" + log 5 "version IDs: ${version_ids[*]}" + for idx in "${!version_keys[@]}"; do + log 5 "idx: $idx" + log 5 "version ID: ${version_ids[$idx]}" + # shellcheck disable=SC2154 + if [ "$lock_config_exists" == "true" ]; then + if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version" + return 1 + fi + else + if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version" + return 1 + fi + fi + done +} + +parse_version_data() { + if [ $# -ne 2 ]; then + log 2 "'parse_version_data' requires raw data, element name" + return 1 + fi + if ! version_data="$(echo "$1" | jq -c "$2" 2>&1)"; then + if [[ "$version_data" == *"Cannot iterate over null"* ]]; then + return 0 + fi + fi + log 5 "version data: ${version_data[*]}" + # shellcheck disable=SC2048 + for version in ${version_data[*]}; do + if ! key=$(echo "$version" | jq -r '.Key' 2>&1); then + log 2 "error getting version key: $key (version: $version)" + return 1 + fi + version_keys+=("$key") + if ! version_id=$(echo "$version" | jq -r '.VersionId' 2>&1); then + log 2 "error getting version id: $version_id (version: $version)" + return 1 + fi + version_ids+=("$version_id") + done +} + +check_versioning_status_rest() { + if [ $# -ne 2 ]; then + log 2 "'check_versioning_status_rest' requires bucket, expected value" + return 1 + fi + if ! get_bucket_versioning_rest "$BUCKET_ONE_NAME"; then + log 2 "error getting bucket versioning" + return 1 + fi + log 5 "versioning: $(cat "$TEST_FILE_FOLDER/versioning.txt")" + if ! versioning_info=$(xmllint --xpath '//*[local-name()="VersioningConfiguration"]' "$TEST_FILE_FOLDER/versioning.txt" 2>&1); then + log 2 "error getting VersioningConfiguration value: $versioning_info" + return 1 + fi + versioning_status="" + if ! has_status=$(echo "$versioning_info" | xmllint --xpath 'boolean(//*[local-name()="Status"]/text())' - 2>&1); then + log 2 "error getting if versioning status: $has_status" + return 1 + fi + if [ "$has_status" == true ]; then + if ! versioning_status=$(echo "$versioning_info" | xmllint --xpath '//*[local-name()="Status"]/text()' - 2>&1); then + log 2 "error getting versioning status: $versioning_status" + return 1 + fi + fi + if [ "$versioning_status" != "$2" ]; then + log 2 "versioning info should be '$2', is $versioning_status" + return 1 + fi + return 0 +} + +get_and_check_versions_rest() { + if [ $# -lt 5 ]; then + log 2 "'get_and_check_versionid_null_rest' requires bucket, key, count, expected islatest, expected id equal to null" + return 1 + fi + if ! list_object_versions_rest "$1"; then + log 2 "error listing object versions" + return 1 + fi + log 5 "versions: $(cat "$TEST_FILE_FOLDER/object_versions.txt")" + if ! version_count=$(xmllint --xpath 'count(//*[local-name()="Version"])' "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then + log 2 "error getting version count: $version_count" + return 1 + fi + log 5 "version count: $version_count" + if [ "$version_count" != "$3" ]; then + log 2 "version count mismatch (expected 1, actual $version_count)" + return 1 + fi + while [ $# -ge 5 ]; do + if [ "$5" == "true" ]; then + id_check="=" + else + id_check="!=" + fi + match_string="//*[local-name()=\"Version\"][*[local-name()=\"VersionId\" and text()$id_check\"null\"] and *[local-name()=\"IsLatest\" and text()=\"$4\"]]" + log 5 "match string: $match_string" + if ! xmllint --xpath "$match_string" "$TEST_FILE_FOLDER/object_versions.txt" 2>&1; then + return 1 + fi + shift 2 + done + return 0 +} diff --git a/tests/versity.sh b/tests/versity.sh index 507a71e..852366d 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -78,7 +78,12 @@ run_versity_app_posix() { if [ -n "$PORT" ]; then base_command+=(--port ":$PORT") fi - base_command+=(posix "$LOCAL_FOLDER") + # TODO remove or change + base_command+=(posix) + if [ -n "$VERSIONING_DIR" ]; then + base_command+=(--versioning-dir "$VERSIONING_DIR") + fi + base_command+=("$LOCAL_FOLDER") export base_command start_versity_process "$3"