Merge pull request #1010 from versity/test/rest_upload_part_copy

Test/rest upload part copy

tests/rest_scripts/list_objects.sh

@@ -36,19 +36,19 @@ canonical_request="GET
 /$bucket_name
 "
 
+queries=""
 if [ "$MARKER" != "" ]; then
-  add_parameter "canonical_request" "marker=$marker"
+  queries=$(add_parameter "$queries" "marker=$marker")
 fi
 if [ "$CONTINUATION_TOKEN" != "" ]; then
-  add_parameter "canonical_request" "continuation-token=$continuation_token"
+  queries=$(add_parameter "$queries" "continuation-token=$continuation_token")
 fi
 if [ "$version_two" != "FALSE" ]; then
-  add_parameter "canonical_request" "list-type=2"
+  queries=$(add_parameter "$queries" "list-type=2")
 fi
 if [ "$max_keys" -ne 0 ]; then
-  add_parameter "canonical_request" "max-keys=$max_keys"
+  queries=$(add_parameter "$queries" "max-keys=$max_keys")
 fi
-first_param_added="false"
 
 canonical_request+="
 host:$host
@@ -61,19 +61,9 @@ create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"")
 url="'$AWS_ENDPOINT_URL/$bucket_name"
-if [ "$MARKER" != "" ]; then
-  add_parameter "url" "marker=$marker"
+if [ "$queries" != "" ]; then
+  url+="?$queries"
 fi
-if [ "$CONTINUATION_TOKEN" != "" ]; then
-  add_parameter "url" "continuation-token=$continuation_token"
-fi
-if [ "$version_two" != "FALSE" ]; then
-  add_parameter "url" "list-type=2"
-fi
-if [ "$max_keys" -ne 0 ]; then
-  add_parameter "url" "max-keys=$max_keys"
-fi
-first_param_added="false"
 url+="'"
 curl_command+=("$url")
 curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""

tests/rest_scripts/rest.sh

@@ -70,13 +70,10 @@ add_parameter() {
   if [ "$#" -ne 2 ]; then
     return
   fi
-  if [ "$first_param_added" != "true" ]; then
-    if [ "$1" == "url" ]; then
-      eval "$1"+='?'
-    fi
-    eval "$1"+="$2"
-    first_param_added="true"
-  else
-    eval "$1"+="'&$2'"
+  current_string="$1"
+  if [ "$current_string" != "" ]; then
+    current_string+="&"
   fi
+  current_string+="$2"
+  echo "$current_string"
 }

tests/rest_scripts/upload_part.sh

@@ -46,8 +46,6 @@ $payload_hash"
 
 create_canonical_hash_sts_and_signature
 
-sleep 5
-
 curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
 -H "\"x-amz-content-sha256: $payload_hash\""

tests/rest_scripts/upload_part_copy.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+# shellcheck disable=SC2153
+key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+part_number="$PART_NUMBER"
+# shellcheck disable=SC2153
+upload_id="$UPLOAD_ID"
+# shellcheck disable=SC2153
+part_location=$PART_LOCATION
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+# shellcheck disable=SC2034
+header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+# shellcheck disable=SC2154
+canonical_request="PUT
+/$bucket_name/$key
+partNumber=$part_number&uploadId=$upload_id
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-copy-source:$part_location
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-copy-source;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-copy-source: $part_location\""
+-H "\"x-amz-date: $current_date_time\""
+-o "\"$OUTPUT_FILE\"")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/setup.sh

@@ -14,9 +14,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-load ./bats-support/load
-load ./bats-assert/load
-
 source ./tests/env.sh
 source ./tests/report.sh
 source ./tests/setup_mc.sh
@@ -68,11 +65,14 @@ delete_temp_log_if_exists() {
 # bats teardown function
 teardown() {
   # shellcheck disable=SC2154
-  if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
-    log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
-  fi
-  if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then
-    log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
+  if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then
+    log 5 "deleting or clearing buckets"
+    if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+      log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
+    fi
+    if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then
+      log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
+    fi
   fi
   if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
     log 3 "error deleting user $USERNAME_ONE"

tests/teardown_static.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2024 Versity Software
 # This file is licensed under the Apache License, Version 2.0
@@ -21,12 +21,13 @@ if ! base_setup; then
   log 2 "error starting versity to set up static buckets"
   exit 1
 fi
-if ! delete_bucket_recursive "s3" "$BUCKET_ONE_NAME"; then
-  log 2 "error creating static bucket one"
-elif ! delete_bucket_recursive "s3" "$BUCKET_TWO_NAME"; then
-  log 2 "error creating static bucket two"
+if ! delete_bucket_recursive "s3api" "$BUCKET_ONE_NAME"; then
+  log 2 "error deleting static bucket one"
+elif ! delete_bucket_recursive "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error deleting static bucket two"
+else
+  log 4 "buckets deleted successfully"
 fi
-log 4 "buckets deleted successfully"
 if ! stop_versity; then
   log 2 "error stopping versity"
 fi

tests/test_mc.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/setup.sh
 source ./tests/util/util_create_bucket.sh

tests/test_mc_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 
 export RUN_MC=true

tests/test_rest.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/commands/create_multipart_upload.sh
 source ./tests/commands/delete_object_tagging.sh
 source ./tests/commands/get_bucket_versioning.sh
@@ -383,9 +386,6 @@ export RUN_USERS=true
 }
 
 @test "REST - bucket tagging - tags" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/932"
-  fi
   test_key="testKey"
   test_value="testValue"
 
@@ -491,4 +491,34 @@ export RUN_USERS=true
 
   run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME"
   assert_success
-}
+}
+
+@test "REST - complete upload - invalid part" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1008"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  run create_large_file "$test_file"
+  assert_success
+
+  run create_upload_finish_wrong_etag "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - upload part copy" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  run create_large_file "$test_file"
+  assert_success
+
+  run create_upload_part_copy_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
+
+  run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
+  assert_success
+}

tests/test_rest_acl.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/commands/put_object.sh
 source ./tests/logger.sh
 source ./tests/setup.sh
@@ -23,9 +26,6 @@ source ./tests/util/util_acl.sh
 export RUN_USERS=true
 
 @test "REST - get ACL" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/971"
-  fi
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 

tests/test_s3.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/util/util_file.sh
 

tests/test_s3_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/util/util_file.sh
 

tests/test_s3api_bucket.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/util/util.sh
 source ./tests/util/util_create_bucket.sh

tests/test_s3api_multipart.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/test_s3api_root_inner.sh
 source ./tests/util/util_file.sh

tests/test_s3api_object.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/util/util.sh
 source ./tests/util/util_create_bucket.sh

tests/test_s3api_policy.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/logger.sh
 source ./tests/setup.sh
 source ./tests/test_s3api_policy_bucket.sh

tests/test_s3cmd.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/test_common_acl.sh

tests/test_s3cmd_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 
 export RUN_S3CMD=true

tests/test_user_aws.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_user_common.sh
 source ./tests/util/util_users.sh
 source ./tests/commands/get_object.sh

tests/test_user_s3cmd.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_user_common.sh
 
 export RUN_S3CMD=true

tests/util/util_bucket.sh

@@ -211,16 +211,12 @@ bucket_cleanup() {
 # return 0 for success, 1 for error
 bucket_cleanup_if_bucket_exists() {
   log 6 "bucket_cleanup_if_bucket_exists"
-  if [ $# -ne 2 ]; then
-    log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name"
+  if [ $# -lt 2 ]; then
+    log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name, bucket known to exist (optional)"
     return 1
   fi
 
-  if bucket_exists "$1" "$2"; then
-    if [ "$DELETE_BUCKETS_AFTER_TEST" == "false" ]; then
-      log 2 "skipping bucket cleanup/deletion"
-      return 0
-    fi
+  if [ "$3" == "true" ] || bucket_exists "$1" "$2"; then
     if ! bucket_cleanup "$1" "$2"; then
       log 2 "error deleting bucket and/or contents"
       return 1
@@ -256,12 +252,16 @@ setup_bucket() {
     return 1
   fi
 
-  if ! bucket_exists "$1" "$2" && [[ $RECREATE_BUCKETS == "false" ]]; then
-    log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
-    return 1
+  bucket_exists="true"
+  if ! bucket_exists "$1" "$2"; then
+    if [[ $RECREATE_BUCKETS == "false" ]]; then
+      log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
+      return 1
+    fi
+    bucket_exists="false"
   fi
 
-  if ! bucket_cleanup_if_bucket_exists "$1" "$2"; then
+  if ! bucket_cleanup_if_bucket_exists "$1" "$2" "$bucket_exists"; then
     log 2 "error deleting bucket or contents if they exist"
     return 1
   fi

tests/util/util_file.sh

@@ -118,9 +118,7 @@ split_file() {
   fi
 
   local error
-  local split_result
-  error=$(split -a 1 -d -b "$part_size" "$1" "$1"-) || split_result=$?
-  if [[ $split_result -ne 0 ]]; then
+  if ! error=$(split -a 1 -d -b "$part_size" "$1" "$1"- 2>&1); then
     log 2 "error splitting file: $error"
     return 1
   fi

tests/util/util_multipart.sh

@@ -34,7 +34,7 @@ multipart_upload_from_bucket() {
   }
 
   if ! create_multipart_upload "$1" "$2-copy"; then
-    log 2 "error running first multpart upload"
+    log 2 "error running first multipart upload"
     return 1
   fi
 
@@ -60,6 +60,25 @@ multipart_upload_from_bucket() {
   return 0
 }
 
+split_and_put_file() {
+  if [ $# -ne 4 ]; then
+    log 2 "'split_and_put_file' requires bucket, key, copy source, part count"
+    return 1
+  fi
+  if ! split_file "$3" "$4"; then
+    log 2 "error splitting file"
+    return 1
+  fi
+  for ((i=0;i<$4;i++)) {
+    log 5 "key: $2, file info: $(ls -l "$3"-"$i")"
+    if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then
+      log 2 "error copying object"
+      return 1
+    fi
+  }
+  return 0
+}
+
 multipart_upload_from_bucket_range() {
   if [ $# -ne 5 ]; then
     log 2 "multipart upload from bucket with range command requires bucket, copy source, key, part count, and range"
@@ -189,3 +208,90 @@ run_and_verify_multipart_upload_with_valid_range() {
   fi
   return 0
 }
+
+create_upload_part_copy_rest() {
+  if [ $# -ne 3 ]; then
+    log 2 "'run_and_verify_multipart_upload_with_valid_range' requires bucket, key, >20MB file"
+    return 1
+  fi
+  if ! split_and_put_file "$1" "$2" "$3" 4; then
+    log 2 "error splitting and putting file"
+    return 1
+  fi
+  if ! create_upload_and_get_id_rest "$1" "$2"; then
+    log 2 "error creating upload and getting ID"
+    return 1
+  fi
+  parts_payload=""
+  for ((i=0; i<=3; i++)); do
+    part_number=$((i+1))
+    if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="$1/$2-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then
+      # shellcheck disable=SC2154
+      log 2 "error uploading part $i: $result"
+      return 1
+    fi
+    log 5 "result: $result"
+    if [ "$result" != "200" ]; then
+      log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/response.txt")"
+      return 1
+    fi
+    if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then
+      log 2 "error retrieving etag: $etag"
+      return 1
+    fi
+    parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
+  done
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
+    log 2 "error completing multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
+    return 1
+  fi
+  return 0
+}
+
+create_upload_finish_wrong_etag() {
+  if [ $# -ne 2 ]; then
+    log 2 "'create_upload_finish_wrong_etag' requires bucket, key"
+    return 1
+  fi
+
+  etag="gibberish"
+  part_number=1
+  if ! create_upload_and_get_id_rest "$1" "$2"; then
+    log 2 "error creating upload and getting ID"
+    return 1
+  fi
+  parts_payload="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
+    log 2 "error completing multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "400" ]; then
+    log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
+    return 1
+  fi
+  if ! error=$(xmllint --xpath '//*[local-name()="Error"]' "$TEST_FILE_FOLDER/result.txt" 2>&1); then
+    log 2 "error retrieving error info: $error"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$error") "InvalidPart" "Code"; then
+    log 2 "code mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$error") "$upload_id" "UploadId"; then
+    log 2 "upload ID mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$error") "$part_number" "PartNumber"; then
+    log 2 "part number mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$error") "$etag" "ETag"; then
+    log 2 "etag mismatch"
+    return 1
+  fi
+  return 0
+}