Merge pull request #1322 from versity/test/rest_create_bucket

test - rest bucket creation, put-object test

tests/commands/delete_object.sh

@@ -63,21 +63,19 @@ delete_object_rest() {
 }
 
 delete_object_bypass_retention() {
-  if ! check_param_count "delete_object_bypass_retention" "client, bucket, key, user, password" 5 $#; then
+  if ! check_param_count "delete_object_bypass_retention" "bucket, key, user, password" 4 $#; then
     return 1
   fi
-  if [ "$1" == "rest" ]; then
-    if ! result=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" \
-        COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$2" OBJECT_KEY="$3" BYPASS_GOVERNANCE_RETENTION="true" \
-        OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
-      log 2 "error deleting object: $result"
-      return 1
-    fi
-  else
-    if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1); then
-      log 2 "error deleting object with bypass retention: $delete_object_error"
-      return 1
-    fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" \
+      COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" BYPASS_GOVERNANCE_RETENTION="true" \
+      OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
+    return 1
   fi
   return 0
 }

tests/logger.sh

@@ -40,7 +40,8 @@ log() {
 }
 
 log_with_stack_ref() {
-  if ! check_log_params "log_with_stack_ref" "level, string, stack reference" 3 $#; then
+  if [[ $# -ne 3 ]]; then
+    echo "log_with_stack_ref function requires level, message, stack reference"
     return 1
   fi
   # shellcheck disable=SC2153

tests/rest_scripts/create_bucket.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+cr_data=("PUT" "/$bucket_name" "" "host:$host")
+cr_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+build_canonical_request "${cr_data[@]}"
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_object_openssl.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+source ./tests/util/util_file.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+data_file="$DATA_FILE"
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+# shellcheck disable=SC2153
+key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+omit_content_length="${OMIT_CONTENT_LENGTH:=false}"
+# shellcheck disable=SC2153
+command_file="${COMMAND_FILE:=command.txt}"
+
+if ! file_size=$(get_file_size "$data_file"); then
+  log_rest 2 "error getting file size"
+  exit 1
+fi
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+
+cr_data=("PUT" "/$bucket_name/$key" "")
+cr_data+=("host:$host")
+cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
+build_canonical_request "${cr_data[@]}"
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+command="PUT /$bucket_name/$key HTTP/1.1\r
+Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\r
+"
+if [ "$omit_content_length" == "false" ]; then
+  command+="Content-Length: $file_size
+"
+fi
+for header_field in "${cr_data[@]}"; do
+  if [[ "$header_field" =~ ^.+:.+$ ]]; then
+    command+="$header_field\r
+"
+  fi
+done
+command+="\r\n"
+echo -en "$command" > "$command_file"
+dd if="$data_file" bs="$file_size" count=1 >> "$command_file"

tests/test_rest.sh

@@ -34,6 +34,7 @@ source ./tests/logger.sh
 source ./tests/setup.sh
 source ./tests/util/util_acl.sh
 source ./tests/util/util_attributes.sh
+source ./tests/util/util_chunked_upload.sh
 source ./tests/util/util_delete_object.sh
 source ./tests/util/util_head_object.sh
 source ./tests/util/util_legal_hold.sh
@@ -654,3 +655,28 @@ test_file="test_file"
   run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" 2000000
   assert_success
 }
+
+@test "REST - create bucket test" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "invalid test for static buckets"
+  fi
+  run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
+  assert_success
+
+  run create_bucket_rest "$BUCKET_ONE_NAME"
+  assert_success
+
+  run list_check_buckets_rest
+  assert_success
+}
+
+@test "REST - put object, missing Content-Length" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1321"
+  fi
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
+}

tests/test_s3api_object.sh

@@ -277,7 +277,7 @@ export RUN_USERS=true
   run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 
-  run delete_object_bypass_retention "s3api" "$BUCKET_ONE_NAME" "$test_file" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
+  run delete_object_bypass_retention "$BUCKET_ONE_NAME" "$test_file" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
   assert_failure
   assert_output -p "InvalidArgument"
   assert_output -p "x-amz-bypass-governance-retention is only applicable"

tests/test_s3api_root_inner.sh

@@ -208,7 +208,7 @@ test_retention_bypass_s3api_root() {
   run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
   assert_success
 
-  run delete_object_bypass_retention "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$username" "$secret_key"
+  run delete_object_bypass_retention "$BUCKET_ONE_NAME" "$bucket_file" "$username" "$secret_key"
   assert_success
 }
 

tests/util/util_chunked_upload.sh

@@ -281,9 +281,10 @@ send_via_openssl() {
     return 1
   fi
   host="${AWS_ENDPOINT_URL#http*://}"
-  if [ "$host" == "s3.amazonaws.com" ]; then
+  if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
     host+=":443"
   fi
+  log 5 "connecting to $host"
   if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
     log 2 "error sending openssl command: $result"
     return 1

tests/util/util_create_bucket.sh

@@ -59,3 +59,19 @@ create_and_check_bucket_invalid_name() {
   fi
   return 0
 }
+
+create_bucket_rest() {
+  if ! check_param_count "create_bucket_rest" "bucket name" 1 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/create_bucket.sh 2>&1); then
+    log 2 "error creating bucket: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    bucket_create_error="$(cat "$TEST_FILE_FOLDER/result.txt")"
+    log 2 "expected '200', was '$result' ($bucket_create_error)"
+    return 1
+  fi
+  return 0
+}

tests/util/util_object.sh

@@ -512,3 +512,17 @@ get_object_with_ranged_download() {
     file_byte_idx=$((last_byte + 1))
   done
 }
+
+put_object_without_content_length() {
+  if ! check_param_count "put_object_without_content_length" "bucket, key, data file" 3 $#; then
+    return 1
+  fi
+  if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$3" OMIT_CONTENT_LENGTH="true" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl.sh 2>&1); then
+    log 2 "error getting result: $result"
+    return 1
+  fi
+  if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 411; then
+    log 2 "error in sending or checking response code"
+    return 1
+  fi
+}

tests/util/util_retention.sh

@@ -61,7 +61,7 @@ check_for_and_remove_worm_protection() {
     if [[ $LOG_LEVEL_INT -ge 5 ]]; then
       log_worm_protection "$1" "$2"
     fi
-    if ! delete_object_bypass_retention "rest" "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then
+    if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then
       log 2 "error deleting object after legal hold removal"
       return 2
     fi