test: some not-implemented's, add list-object-v2 working test

tests/commands/command.sh

@@ -18,7 +18,7 @@ source ./tests/logger.sh
 source ./tests/report.sh
 
 send_command() {
-  if [ $# -eq 0 ]; then
+  if ! check_param_count_gt "command data" 1 $#; then
     return 1
   fi
   if [ -n "$COMMAND_LOG" ] || [ -n "$COVERAGE_LOG" ]; then

tests/commands/list_objects_v2.sh

@@ -28,4 +28,15 @@ list_objects_v2() {
     log 2 "error listing objects: $objects"
     return 1
   fi
+}
+
+list_objects_v2_rest_callback() {
+  if ! check_param_count_gt "bucket, expected response code, callback fn, params" 3 $#; then
+    return 1
+  fi
+  if ! send_rest_go_command_callback "$2" "$3" "-bucketName" "$1" "-method" "GET" "-query" "list-type=2" "${@:4}"; then
+    log 2 "error sending REST ListObjectsV2 command or parsing callback"
+    return 1
+  fi
+  return 0
 }

tests/drivers/list_objects/list_objects_rest.sh

@@ -51,3 +51,32 @@ list_objects_success_or_access_denied() {
   fi
   return 0
 }
+
+check_v2_objects() {
+  if ! check_param_count_v2 "data file" 1 $#; then
+    return 1
+  fi
+  if ! check_xml_element "$1" "$object_count" "ListBucketResult" "KeyCount"; then
+    log 2 "error checking KeyCount element"
+    return 1
+  fi
+  for object in "${expected_objects[@]}"; do
+    if ! check_if_element_exists "$1" "$object" "ListBucketResult" "Contents" "Key"; then
+      log 2 "error checking if element '$object' exists"
+      return 1
+    fi
+  done
+  return 0
+}
+
+list_check_objects_rest_v2() {
+  if ! check_param_count_v2 "bucket name, object count, objects, params" 4 $#; then
+    return 1
+  fi
+  object_count=$2
+  expected_objects=("${@:3:$object_count}")
+  if ! list_objects_v2_rest_callback "$1" "200" "check_v2_objects" "${@:((3+$object_count))}"; then
+    log 2 "error sending list objects v2 command and checking callback"
+    return 1
+  fi
+}

tests/drivers/not_implemented/not_implemented_rest.sh

@@ -18,8 +18,7 @@ source ./tests/rest_scripts/rest.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 
 send_not_implemented_expect_failure() {
-  if [ $(($# % 2)) -ne 0 ]; then
-    log 2 "'send_not_implemented_expect_failure' param count must be multiple of 2 (key/value pairs)"
+  if ! check_param_count_gt "parameters" 1 $#; then
     return 1
   fi
   if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" \

tests/drivers/put_object/put_object_rest.sh

@@ -43,18 +43,31 @@ setup_bucket_and_add_file() {
   if ! check_param_count_v2 "bucket, filename" 2 $#; then
     return 1
   fi
+  if ! setup_bucket_and_add_files "$1" "$2"; then
+    log 2 "error setting up bucket and adding file"
+    return 1
+  fi
+  return 0
+}
+
+setup_bucket_and_add_files() {
+  if ! check_param_count_gt "bucket, filenames" 2 $#; then
+    return 1
+  fi
   if ! setup_bucket_v2 "$1"; then
     log 2 "error setting up bucket"
     return 1
   fi
-  if ! create_test_files "$2"; then
-    log 2 "error creating test file"
-    return 1
-  fi
-  if ! put_object_rest "$TEST_FILE_FOLDER/$2" "$1" "$2"; then
-    log 2 "error putting REST object"
+  if ! create_test_files "${@:2}"; then
+    log 2 "error creating test files"
     return 1
   fi
+  for file in "${@:2}"; do
+    if ! put_object_rest "$TEST_FILE_FOLDER/$file" "$1" "$file"; then
+      log 2 "error adding file '$TEST_FILE_FOLDER/$file' to bucket '$1'"
+      return 1
+    fi
+  done
   return 0
 }
 

tests/drivers/rest.sh

@@ -331,7 +331,7 @@ check_specific_argument_name_and_value() {
     return 1
   fi
   if ! check_error_parameter "$1" "$argument_name" "$argument_value"; then
-    log 2 "error checking 'ArgumentName' parameter"
+    log 2 "error checking '$argument_name' parameter"
     return 1
   fi
 }
@@ -348,3 +348,52 @@ send_rest_go_command_expect_error_with_specific_arg_name_value() {
   fi
   return 0
 }
+
+check_specific_argument_names_and_values() {
+  if ! check_param_count_v2 "data file" 1 $#; then
+    return 1
+  fi
+  for ((idx=0; idx<${#arg_names_and_values[@]}; idx+=2)); do
+    if ! check_error_parameter "$1" "${arg_names_and_values[$idx]}" "${arg_names_and_values[(($idx+1))]}"; then
+      log 2 "error checking '${arg_names_and_values[$idx]}' parameter"
+      return 1
+    fi
+  done
+}
+
+send_rest_go_command_expect_error_with_specific_arg_names_values() {
+  if ! check_param_count_gt "response code, error code, message, arg count, pairs of arg names and values, params" 6 $#; then
+    return 1
+  fi
+  arg_names_and_values=("${@:5:$4}")
+  if ! send_rest_go_command_expect_error_callback "$1" "$2" "$3" "check_specific_argument_names_and_values" "${@:((5+$4))}"; then
+    log 2 "error checking error response values"
+    return 1
+  fi
+  return 0
+}
+
+check_header_key_and_value() {
+  if ! check_param_count_v2 "data file" 1 $#; then
+    return 1
+  fi
+  if ! check_for_header_key_and_value "$1" "$header_key" "$header_value"; then
+    log 2 "error checking header key and value"
+    return 1
+  fi
+  return 0
+}
+
+send_rest_go_command_check_header_key_and_value() {
+  if ! check_param_count_gt "response code, header key, header values, params" 3 $#; then
+    return 1
+  fi
+  header_key="$2"
+  header_value="$3"
+  if ! send_rest_go_command_callback "$1" "check_header_key_and_value" "${@:4}"; then
+    log 2 "error sending command and checking header key and value"
+    return 1
+  fi
+  return 0
+}
+

tests/drivers/xml.sh

@@ -162,6 +162,31 @@ check_xml_error_contains_with_single_error_field() {
   return 0
 }
 
+check_if_element_exists() {
+  if ! check_param_count_gt "data file, element, XML tree" 3 $#; then
+    return 1
+  fi
+  if ! build_xpath_string_for_element "${@:3}"; then
+    log 2 "error building XPath search string"
+    return 1
+  fi
+
+  log 5 "data: $(cat "$1")"
+  log 5 "xpath: $xpath"
+  if ! get_xml_data "$1" "$1.xml"; then
+    log 2 "error getting XML data"
+    return 1
+  fi
+  if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$1.xml" 2>&1); then
+    log 2 "error getting result: $result"
+    return 1
+  fi
+  if [ "$result" == "true" ]; then
+    return 0
+  fi
+  return 1
+}
+
 get_xml_data() {
   if ! check_param_count_v2 "data file, output file" 2 $#; then
     return 1

tests/logger.sh

@@ -69,7 +69,6 @@ log_mask() {
   if ! check_log_params "log_mask" "level, string, stack reference" 3 $#; then
     return 1
   fi
-
   if ! mask_args "$2"; then
     echo "error masking args"
     return 1
@@ -130,7 +129,7 @@ check_arg_for_mask() {
       masked_args+=("${arg:0:4}****")
       is_access=false
     else
-      masked_args+=("********")
+      masked_args+=("\********")
     fi
     mask_next=false
   elif [[ "$arg" == --secret_key=* ]]; then
@@ -163,10 +162,10 @@ log_message() {
   local bash_source_ref=$(($3+1))
   now="$(date "+%Y-%m-%d %H:%M:%S")"
   if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then
-    echo "$now $1 $2" >&2
+    printf "%s\n" "$now $1 $2" >&2
   fi
   if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp"
+    printf "%s\n" "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp"
   fi
   sync
 }

tests/report.sh

@@ -337,7 +337,6 @@ record_command_v2() {
   if ! check_param_count_v2 "command string" 1 $#; then
     return 1
   fi
-  log 5 "parsing command '$1'"
   if ! parse_command_info "$1"; then
     log 2 "error parsing command info"
     return 1

tests/rest_scripts/command/s3Command.go

@@ -291,7 +291,20 @@ func (s *S3Command) generateCanonicalRequestString() {
 	canonicalRequestLines := []string{s.Method}
 
 	canonicalRequestLines = append(canonicalRequestLines, s.path)
-	canonicalRequestLines = append(canonicalRequestLines, s.Query)
+	var queryRequestLine string
+	if strings.Contains(s.Query, "&") {
+		queries := strings.Split(s.Query, "&")
+		if !strings.HasSuffix(queries[0], "=") && !strings.Contains(queries[0], "=") {
+			queries[0] += "="
+			queryRequestLine = strings.Join(queries, "&")
+		}
+	} else if s.Query != "" && !strings.HasSuffix(s.Query, "=") && !strings.Contains(s.Query, "=") {
+		queryRequestLine = s.Query + "="
+	}
+	if queryRequestLine == "" {
+		queryRequestLine = s.Query
+	}
+	canonicalRequestLines = append(canonicalRequestLines, queryRequestLine)
 
 	var signedParams []string
 	for _, headerValue := range s.headerValues {

tests/test_rest_list_buckets.sh

@@ -203,3 +203,22 @@ export RUN_USERS=true
   run list_check_buckets_user "$username" "$password" "$bucket_two_name"
   assert_success
 }
+
+@test "REST - ListBuckets - invalid POST route" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1810"
+  fi
+  run get_file_name
+  assert_success
+  file_name=$output
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_add_file "$bucket_name" "$file_name"
+  assert_success
+
+  run send_rest_go_command_expect_error_with_specific_arg_names_values "405" "MethodNotAllowed" "is not allowed" 4 "Method" "POST" "ResourceType" "SERVICE" "-method" "POST"
+  assert_success
+}

tests/test_rest_list_objects.sh

@@ -80,6 +80,50 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh
   assert_success
 }
 
+@test "REST - ListObjectsV2 - includes bucket header" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1814"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run get_file_name
+  assert_success
+  test_file="$output"
+
+  run get_file_name
+  assert_success
+  test_file_two="$output"
+
+  run setup_bucket_and_add_files "$bucket_name" "$test_file" "$test_file_two"
+  assert_success
+
+  run send_rest_go_command_check_header_key_and_value "200" "x-amz-bucket-region" "$AWS_REGION" "-method" "GET" \
+    "-bucketName" "$bucket_name" "-query" "list-type=2"
+  assert_success
+}
+
+@test "REST - ListObjectsV2 - success" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run get_file_name
+  assert_success
+  test_file="$output"
+
+  run get_file_name
+  assert_success
+  test_file_two="$output"
+
+  run setup_bucket_and_add_files "$bucket_name" "$test_file" "$test_file_two"
+  assert_success
+
+  run list_check_objects_rest_v2 "$bucket_name" 2 "$test_file" "$test_file_two"
+  assert_success
+}
+
 @test "REST - list objects v1 - no NextMarker without delimiter" {
   if [ "$DIRECT" != "true" ]; then
     skip "https://github.com/versity/versitygw/issues/999"

tests/test_rest_not_implemented.sh

@@ -195,3 +195,53 @@ source ./tests/setup.sh
   run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "requestPayment=" "PUT"
   assert_success
 }
+
+@test "REST - GetObjectAcl" {
+  run get_file_name
+  assert_success
+  file_name=$output
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_add_file "$bucket_name" "$file_name"
+  assert_success
+
+  run send_not_implemented_expect_failure "-bucketName" "$bucket_name" "-query" "acl=" "-method" "GET" "-objectKey" "$file_name"
+  assert_success
+}
+
+@test "REST - PutObjectAcl" {
+  run get_file_name
+  assert_success
+  file_name=$output
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_add_file "$bucket_name" "$file_name"
+  assert_success
+
+  run send_not_implemented_expect_failure "-bucketName" "$bucket_name" "-query" "acl=" "-method" "PUT" "-objectKey" "$file_name"
+  assert_success
+}
+
+@test "REST - RestoreObject" {
+  skip "https://github.com/versity/versitygw/issues/1805"
+
+  run get_file_name
+  assert_success
+  file_name=$output
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_add_file "$bucket_name" "$file_name"
+  assert_success
+
+  run send_not_implemented_expect_failure "-bucketName" "$bucket_name" "-query" "restore=" "-method" "POST" "-objectKey" "$file_name"
+  assert_success
+}