test: chunked upload - more tests, final signature test

tests/commands/get_bucket_policy.sh

@@ -105,6 +105,34 @@ get_bucket_policy_s3cmd() {
   return 0
 }
 
+get_bucket_policy_rest() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "s3cmd 'get bucket policy' command requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_policy_rest_expect_code "$1" "200"; then
+    log 2 "error getting REST bucket policy"
+    return 1
+  fi
+  return 0
+}
+
+get_bucket_policy_rest_expect_code() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "s3cmd 'get bucket policy' command requires bucket name, expected code"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
+    log 2 "error attempting to get bucket policy response: $result"
+    return 1
+  fi
+  if [ "$result" != "$2" ]; then
+    log 2 "unexpected response code, expected '$2', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))"
+    return 1
+  fi
+  bucket_policy="$(cat "$TEST_FILE_FOLDER/policy.txt")"
+}
+
 # return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
 check_and_load_policy_info() {
   if [[ $policy_brackets == false ]]; then

tests/rest_scripts/put_object_openssl_chunked_example.sh

@@ -48,6 +48,8 @@ load_parameters() {
     # shellcheck disable=SC2153
     data_file="$DATA_FILE"
     chunk_size="${CHUNK_SIZE:=65536}"
+    # shellcheck disable=SC2153
+    final_signature="$FINAL_SIGNATURE"
   fi
 
   readonly initial_sts_data="AWS4-HMAC-SHA256-PAYLOAD
@@ -203,8 +205,8 @@ $1
 $signature_no_data
 "
   if [ "$4" -ne 0 ]; then
-    if ! data=$(dd if="$2" of="$2.tmp" bs=1 skip="$3" count="$4" 2>&1); then
-      log_rest 2 "error retrieving data: $data"
+    if ! error=$(dd if="$2" of="$2.tmp" bs=1 skip="$3" count="$4" 2>&1); then
+      log_rest 2 "error retrieving data: $error"
       return 1
     fi
     payload_hash="$(sha256sum "$2.tmp" | awk '{print $1}')"
@@ -213,26 +215,31 @@ $signature_no_data
   fi
   chunk_sts_data+="$payload_hash"
   create_canonical_hash_sts_and_signature "$chunk_sts_data"
-  chunk="$(printf "%x" "$4");chunk-signature=$signature"
-  if [ "$4" -gt 0 ]; then
-    chunk+="
-$(cat "$2.tmp")"
+  if [ -n "$final_signature" ] && [ $((idx+1)) -eq ${#chunk_sizes[@]} ]; then
+    signature="$final_signature"
   fi
+  chunk="$(printf "%x" "$4");chunk-signature=$signature"
+  echo -e "$chunk\r" >> "$COMMAND_FILE"
+  if [ "$4" -gt 0 ]; then
+    dd if="$2.tmp" bs="$4" count=1 >> "$COMMAND_FILE"
+    echo -e "\r" >> "$COMMAND_FILE"
+  fi
+  return 0
 }
 
 build_chunks() {
   if [ $# -ne 1 ]; then
-    log_rest 2 "'add_chunks' requires first signature"
+    log_rest 2 "'build_chunks' requires first signature"
     return 1
   fi
 
   last_signature="$1"
   idx=0
   offset=0
-  chunks=""
+  log_rest 5 "chunk sizes: ${chunk_sizes[*]}"
   for chunk_size in "${chunk_sizes[@]}"; do
     if ! build_chunk; then
-      log 2 "error building chunk"
+      log_rest 2 "error building chunk"
       return 1
     fi
     if [ "$test_mode" == "true" ]; then
@@ -240,6 +247,7 @@ build_chunks() {
     fi
     ((idx++))
   done
+  return 0
 }
 
 build_chunk() {
@@ -256,8 +264,6 @@ build_chunk() {
     offset=$((offset+chunk_size))
     last_signature="$signature"
   fi
-  chunks+="$chunk
-"
 }
 
 check_chunks_and_signatures_in_test_mode() {
@@ -299,62 +305,70 @@ check_chunks_and_signatures_in_test_mode() {
   esac
 }
 
-build_command() {
-  command="PUT /$bucket_name/$key HTTP/1.1
-Host: $host
-x-amz-date: $current_date_time
-x-amz-storage-class: REDUCED_REDUNDANCY
-Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature
-x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD
-Content-Encoding: aws-chunked
-x-amz-decoded-content-length: $file_size
-Content-Length: $content_length
+record_command_lines() {
+  while IFS= read -r line; do
+    if ! mask_arg_array "$line"; then
+      return 1
+    fi
+    # shellcheck disable=SC2154
+    echo "${masked_args[*]}" >> "$COMMAND_LOG"
+  done <<< "$command"
+}
 
-"
+build_initial_command() {
+  command="PUT /$bucket_name/$key HTTP/1.1\r
+Host: $host\r
+x-amz-date: $current_date_time\r
+x-amz-storage-class: REDUCED_REDUNDANCY\r
+Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature\r
+x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD\r
+Content-Encoding: aws-chunked\r
+x-amz-decoded-content-length: $file_size\r
+Content-Length: $content_length\r
+\r\n"
 
 if [ "$test_mode" == "true" ] && [ "$command" != "$expected_command" ]; then
   log_rest 2 "command mismatch ($command)"
   return 1
 fi
+  echo -en "$command" > "$COMMAND_FILE"
+}
 
-  chunks+="
-"
-  command+="$chunks"
-  command="${command//$'\n'/$'\r\n'}"
-  echo -n "$command" > "$COMMAND_FILE"
+complete_command() {
+  echo -e "\r" >> "$COMMAND_FILE"
   if [ -n "$COMMAND_LOG" ]; then
-    while IFS= read -r line; do
-      if ! mask_arg_array "$line"; then
-        return 1
-      fi
-      # shellcheck disable=SC2154
-      echo "${masked_args[*]}" >> "$COMMAND_LOG"
-    done <<< "$command"
+    if ! record_command_lines; then
+      return 1
+    fi
   fi
 }
 
 load_parameters
 
 if ! get_file_size_and_content_length; then
-  log 2 "error getting file size and content length"
+  log_rest 2 "error getting file size and content length"
   exit 1
 fi
 
 if ! get_first_signature; then
-  log 2 "error getting first signature"
+  log_rest 2 "error getting first signature"
   exit 1
 fi
 
+if ! build_initial_command; then
+  log_rest 2 "error building command"
+  exit 1
+fi
 if ! build_chunks "$first_signature"; then
-  log 2 "error building chunks"
+  log_rest 2 "error building chunks"
+  exit 1
+fi
+if ! complete_command; then
+  log_rest 2 "error adding chunks"
   exit 1
 fi
 
-if ! build_command; then
-  log 2 "error building command"
-  exit 1
-fi
 if [ "$test_mode" == "true" ]; then
-  echo "TEST PASS"
+  log_rest 4 "TEST PASS"
 fi
 exit 0

tests/test_rest_chunked.sh

@@ -50,3 +50,74 @@ source ./tests/util/util_setup.sh
   run attempt_chunked_upload_with_bad_first_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 }
+
+@test "REST - chunked upload, final signature error" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1147"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test-file"
+  run create_test_file "$test_file" 0
+  assert_success
+
+  run attempt_chunked_upload_with_bad_final_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - chunked upload, success (file with just a's)" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  sleep 10
+
+  test_file="test-file"
+  run create_file_single_char "$test_file" 8192 'a'
+  assert_success
+
+  run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - chunked upload, success (null bytes)" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  sleep 10
+
+  test_file="test-file"
+  run create_file_single_char "$test_file" 8192 '\0'
+  assert_success
+
+  run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - chunked upload, success (random bytes)" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  sleep 10
+
+  test_file="test-file"
+  run create_test_file "$test_file" 10000
+  assert_success
+
+  run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - chunked upload, success (zero-byte file)" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  sleep 10
+
+  test_file="test-file"
+  run create_test_file "$test_file" 0
+  assert_success
+
+  run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}

tests/test_s3cmd.sh

@@ -97,6 +97,17 @@ export RUN_USERS=true
   test_put_bucket_acl_s3cmd
 }
 
+@test 's3cmd - can enable public ACLs for bucket' {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1154"
+  fi
+  run setup_bucket "s3cmd" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run put_public_access_block_enable_public_acls "$BUCKET_ONE_NAME"
+  assert_success
+}
+
 # test listing buckets on versitygw
 @test "test_list_buckets_s3cmd" {
   test_common_list_buckets "s3cmd"

tests/util/util_chunked_upload.sh

@@ -54,4 +54,79 @@ attempt_chunked_upload_with_bad_first_signature() {
     return 1
   fi
   return 0
-}
+}
+
+chunked_upload_success() {
+  if [ $# -ne 3 ]; then
+    log 2 "'chunked_upload_success_as' requires data file, bucket name, key"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
+         AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+         AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+         AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
+         DATA_FILE="$1" \
+         BUCKET_NAME="$2" \
+         OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
+    log 2 "error creating command: $result"
+    return 1
+  fi
+
+  host="${AWS_ENDPOINT_URL#http*://}"
+  if [ "$host" == "s3.amazonaws.com" ]; then
+    host+=":443"
+  fi
+  if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
+    log 2 "error sending openssl command: $result"
+    return 1
+  fi
+  response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
+  if [ "$response_code" != "200" ]; then
+    log 2 "expected response '200', was '$response_code'"
+    return 1
+  fi
+  return 0
+}
+
+attempt_chunked_upload_with_bad_final_signature() {
+  if [ $# -ne 3 ]; then
+    log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
+         AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+         AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+         AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
+         DATA_FILE="$1" \
+         BUCKET_NAME="$2" \
+         OBJECT_KEY="$3" \
+         CHUNK_SIZE=8192 \
+         TEST_MODE=false \
+         COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" \
+         FINAL_SIGNATURE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
+    log 2 "error creating command: $result"
+    return 1
+  fi
+  host="${AWS_ENDPOINT_URL#http*://}"
+  if [ "$host" == "s3.amazonaws.com" ]; then
+    host+=":443"
+  fi
+  if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
+    log 2 "error sending openssl command: $result"
+    return 1
+  fi
+  response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
+  log 5 "response code: $response_code"
+  if [ "$response_code" != "403" ]; then
+    log 2 "expected code '403', was '$response_code'"
+    return 1
+  fi
+  response_data="$(echo "$result" | grep "<")"
+  log 5 "response data: $response_data"
+  log 5 "END"
+  if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then
+    log 2 "error checking XML element"
+    return 1
+  fi
+  return 0
+}

tests/util/util_file.sh

@@ -65,6 +65,28 @@ create_test_file() {
   return 0
 }
 
+create_file_single_char() {
+  if [ "$#" -ne 3 ]; then
+    log 2 "'create_file_single_char' requires filename, size, char"
+    return 1
+  fi
+  if [[ -e "$TEST_FILE_FOLDER/$1" ]]; then
+    if ! error=$(rm "$TEST_FILE_FOLDER/$1" 2>&1); then
+      log 2 "error removing existing file: $error"
+      return 1
+    fi
+  fi
+  if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then
+    log 2 "error creating new file: $error"
+    return 1
+  fi
+  if ! error=$(dd if=/dev/zero bs=1 count="$2" | tr '\0' "$3" > "$TEST_FILE_FOLDER/$1" 2>&1); then
+    log 2 "error adding data to file: $error"
+    return 1
+  fi
+  return 0
+}
+
 # params:  folder name
 # fail if error
 create_test_folder() {

tests/util/util_policy.sh

@@ -260,16 +260,12 @@ get_and_check_no_policy_error() {
     log 2 "'get_and_check_no_policy_error' requires bucket name"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
-    log 2 "error attempting to get bucket policy response: $result"
+  if ! get_bucket_policy_rest_expect_code "$1" "404"; then
+    log 2 "GetBucketPolicy returned unexpected response code"
     return 1
   fi
-  if [ "$result" != "404" ]; then
-    log 2 "GetBucketOwnershipControls returned unexpected response code: $result, reply:  $(cat "$TEST_FILE_FOLDER/response.txt")"
-    return 1
-  fi
-  log 5 "response: $(cat "$TEST_FILE_FOLDER/response.txt")"
-  if ! bucket_name=$(xmllint --xpath '//*[local-name()="BucketName"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then
+  log 5 "response: $bucket_policy"
+  if ! bucket_name=$(xmllint --xpath '//*[local-name()="BucketName"]/text()' <(echo -n "$bucket_policy") 2>&1); then
     log 2 "error getting bucket name: $bucket_name"
     return 1
   fi
@@ -314,21 +310,16 @@ put_and_check_policy_rest() {
     log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
+  if ! get_bucket_policy_rest "$1"; then
     log 2 "error attempting to get bucket policy response: $result"
     return 1
   fi
-  if [ "$result" != "200" ]; then
-    log 2 "unexpected response code, expected '200', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))"
-    return 1
-  fi
-  log 5 "policy: $(cat "$TEST_FILE_FOLDER/policy.txt")"
   if [ "$DIRECT" == "true" ]; then
     principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$4"
   else
     principal="$4"
   fi
-  if ! check_policy "$(cat "$TEST_FILE_FOLDER/policy.txt")" "$3" "$principal" "$5" "$6"; then
+  if ! check_policy "$bucket_policy" "$3" "$principal" "$5" "$6"; then
     log 2 "policies not equal"
     return 1
   fi