test: checksum mode test

tests/commands/head_bucket.sh

@@ -39,12 +39,13 @@ head_bucket() {
     log 2 "invalid command type $1"
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error getting bucket info: $bucket_info"
     if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
       return 1
     fi
+    log 2 "error getting bucket info: $bucket_info"
     return 2
   fi
+  bucket_info="$(echo -n "$bucket_info" | grep -v "InsecureRequestWarning")"
   echo "$bucket_info"
   return 0
 }

tests/commands/put_bucket_acl.sh

@@ -24,7 +24,7 @@ put_bucket_acl_s3api() {
     log 2 "put bucket acl command requires bucket name, acl file"
     return 1
   fi
-  log 5 "bucket name: $1, acls: $2"
+  log 5 "bucket name: $1, acls: $(cat "$2")"
   if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1

tests/rest_scripts/put_object_openssl_chunked_example.sh

@@ -322,6 +322,15 @@ fi
   command+="$chunks"
   command="${command//$'\n'/$'\r\n'}"
   echo -n "$command" > "$COMMAND_FILE"
+  if [ -n "$COMMAND_LOG" ]; then
+    while IFS= read -r line; do
+      if ! mask_arg_array "$line"; then
+        return 1
+      fi
+      # shellcheck disable=SC2154
+      echo "${masked_args[*]}" >> "$COMMAND_LOG"
+    done <<< "$command"
+  fi
 }
 
 load_parameters

tests/test_common_acl.sh

@@ -46,15 +46,14 @@ test_put_bucket_acl_s3cmd() {
 test_common_put_bucket_acl() {
   assert [ $# -eq 1 ]
 
-  run setup_bucket "$1" "$BUCKET_ONE_NAME"
+  run setup_bucket_and_user "$BUCKET_ONE_NAME" "$USERNAME_ONE" "$PASSWORD_ONE" "user"
   assert_success
+  # shellcheck disable=SC2154
+  username="${lines[${#lines[@]}-2]}"
 
   run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
   assert_success
 
-  run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"
-  assert_success
-
   run get_check_acl_id "$1" "$BUCKET_ONE_NAME"
   assert_success
 
@@ -67,7 +66,7 @@ test_common_put_bucket_acl() {
     grantee_id="http://acs.amazonaws.com/groups/global/AllUsers"
   else
     grantee_type="CanonicalUser"
-    grantee_id="$USERNAME_ONE"
+    grantee_id="$username"
   fi
   run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "$grantee_type" "$grantee_id" "READ" "$AWS_ACCESS_KEY_ID"
   assert_success
@@ -79,7 +78,7 @@ test_common_put_bucket_acl() {
   run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME"
   assert_success
 
-  run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
+  run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$username" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
   assert_success
 
   run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"

tests/test_rest_checksum.sh

@@ -130,3 +130,14 @@ test_file="test_file"
   run add_correct_checksum "sha1"
   assert_success
 }
+
+@test "REST - attempt to get checksum without checksum mode" {
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run add_correct_checksum "sha256"
+  assert_success
+
+  run head_object_without_and_with_checksum "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}

tests/test_rest_chunked.sh

@@ -38,7 +38,7 @@ source ./tests/util/util_setup.sh
 
 @test "REST - chunked upload, signature error" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1056 - gibberish at end"
+    skip "https://github.com/versity/versitygw/issues/1123"
   fi
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success

tests/test_s3cmd.sh

@@ -125,7 +125,6 @@ export RUN_USERS=true
 
   run head_bucket "s3cmd" "$BUCKET_ONE_NAME"a
   assert_failure 1
-  assert_output -p "404"
 }
 
 @test "test_ls_directory_object" {

tests/test_user_common.sh

@@ -18,6 +18,7 @@ source ./tests/setup.sh
 source ./tests/util/util_create_bucket.sh
 source ./tests/util/util_list_buckets.sh
 source ./tests/util/util_object.sh
+source ./tests/util/util_setup.sh
 source ./tests/util/util_users.sh
 source ./tests/commands/list_buckets.sh
 
@@ -59,30 +60,26 @@ test_admin_user() {
 
   run change_bucket_owner "$admin_username" "$admin_password" "$BUCKET_TWO_NAME" "$user_username"
   assert_success
-
-  delete_user "$user_username"
-  delete_user "$admin_username"
 }
 
 test_create_user_already_exists() {
-  if [[ $# -ne 1 ]]; then
-    fail "test admin user command requires command type"
-  fi
+  assert [ $# -eq 1 ]
 
   username="$USERNAME_ONE"
   password="$PASSWORD_ONE"
 
-  run setup_user "$username" "123456" "admin"
-  assert_success "error setting up user"
+  run setup_user "$username" "$password" "admin"
+  assert_success
 
-  if create_user "$username" "123456" "admin"; then
-    fail "'user already exists' error not returned"
-  fi
-
-  delete_user "$username"
+  run create_user_versitygw "$username" "$password" "admin"
+  assert_failure
 }
 
 test_user_user() {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip
+  fi
+
   if [[ $# -ne 1 ]]; then
     fail "test admin user command requires command type"
   fi
@@ -90,41 +87,31 @@ test_user_user() {
   username="$USERNAME_ONE"
   password="$PASSWORD_ONE"
 
-  setup_user "$username" "$password" "user" || fail "error setting up user"
-  bucket_cleanup_if_bucket_exists "s3api" "versity-gwtest-user-bucket"
-
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  run setup_bucket_and_user "$BUCKET_ONE_NAME" "$username" "$password" "user"
   assert_success
 
-  if create_bucket_with_user "s3api" "versity-gwtest-user-bucket" "$username" "$password"; then
-    fail "creating bucket with 'user' account failed to return error"
-  fi
-  # shellcheck disable=SC2154
-  [[ $error == *"Access Denied"* ]] || fail "error message '$error' doesn't contain 'Access Denied'"
-
-  create_bucket "s3api" "versity-gwtest-user-bucket" || fail "error creating bucket"
-
-  change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || fail "error changing bucket owner"
-  if change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin"; then
-    fail "user shouldn't be able to change bucket owner"
+  if [ "$RECREATE_BUCKETS" == "true" ]; then
+    run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"
+    assert_success
+    run create_bucket "s3api" "$BUCKET_TWO_NAME"
+    assert_success
+  else
+    run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$BUCKET_TWO_NAME" "$AWS_ACCESS_KEY_ID"
+    assert_success
   fi
 
-  list_buckets_with_user "s3api" "$username" "$password" || fail "error listing buckets with user '$username'"
-  bucket_found=false
-  for bucket in "${bucket_array[@]}"; do
-    if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
-      fail "$BUCKET_ONE_NAME shouldn't show up in 'user' bucket list"
-    elif [ "$bucket" == "versity-gwtest-user-bucket" ]; then
-      bucket_found=true
-    fi
-  done
-  if [ $bucket_found == false ]; then
-    fail "user-owned bucket not found in user list"
-  fi
+  run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$BUCKET_TWO_NAME" "$username"
+  assert_success
+
+  run change_bucket_owner "$username" "$password" "$BUCKET_TWO_NAME" "admin"
+  assert_failure
+  assert_output -p "AccessDenied"
+
+  run list_and_check_buckets_omit_without_permission "$username" "$password" "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
+  assert_success
 
   run delete_bucket "s3api" "versity-gwtest-user-bucket"
   assert_success "failed to delete bucket"
-  delete_user "$username"
 }
 
 test_userplus_operation() {
@@ -135,32 +122,22 @@ test_userplus_operation() {
   username="$USERNAME_ONE"
   password="$PASSWORD_ONE"
 
-  bucket_cleanup_if_bucket_exists "s3api" "versity-gwtest-userplus-bucket"
-  setup_user "$username" "$password" "userplus" || fail "error creating user '$username'"
-
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  run setup_bucket_and_user "$BUCKET_ONE_NAME" "$username" "$password" "userplus"
   assert_success
 
-  create_bucket_with_user "s3api" "versity-gwtest-userplus-bucket" "$username" "$password" || fail "error creating bucket with user '$username'"
-
-  list_buckets_with_user "s3api" "$username" "$password" || fail "error listing buckets with user '$username'"
-  bucket_found=false
-  for bucket in "${bucket_array[@]}"; do
-    if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
-      fail "$BUCKET_ONE_NAME shouldn't show up in 'userplus' bucket list"
-    elif [ "$bucket" == "versity-gwtest-userplus-bucket" ]; then
-      bucket_found=true
-    fi
-  done
-  if [ $bucket_found == false ]; then
-    fail "userplus-owned bucket not found in user list"
+  if [ "$RECREATE_BUCKETS" == "true" ]; then
+    run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"
+    assert_success
+    run create_bucket_with_user "s3api" "$BUCKET_TWO_NAME" "$username" "$password"
+    assert_success
+  else
+    run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$BUCKET_TWO_NAME" "$username"
+    assert_success
   fi
 
-  if change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin"; then
-    fail "userplus shouldn't be able to change bucket owner"
-  fi
+  run list_and_check_buckets_omit_without_permission "$username" "$password" "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
+  assert_success
 
-  run delete_bucket "s3api" "versity-gwtest-admin-bucket"
-  assert_success "failed to delete bucket"
-  delete_user "$username"
+  run change_bucket_owner "$username" "$password" "$BUCKET_TWO_NAME" "admin"
+  assert_failure
 }

tests/util/util_acl.sh

@@ -102,6 +102,8 @@ get_check_acl_id() {
       log 2 "error getting canonical ID: $canonical_id"
       return 1
     fi
+    canonical_id="$(echo -n "$canonical_id" | grep -v "InsecureRequestWarning" | sed "s/\"//g")"
+    log 5 "canonical ID: $canonical_id"
     if [[ $id != "$canonical_id" ]]; then
       log 2 "acl ID doesn't match AWS key or canonical ID"
       return 1

tests/util/util_chunked_upload.sh

@@ -21,7 +21,8 @@ attempt_chunked_upload_with_bad_first_signature() {
     log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key"
     return 1
   fi
-  if ! result=$(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
+         AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
          AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
          AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
          DATA_FILE="$1" \

tests/util/util_head_object.sh

@@ -166,3 +166,29 @@ check_checksum_rest_crc32() {
   fi
   return 0
 }
+
+head_object_without_and_with_checksum() {
+  if [ $# -ne 2 ]; then
+    log 2 "'head_object_without_checksum' requires bucket, file"
+    return 1
+  fi
+  if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then
+    log 2 "error getting result: $result"
+    return 1
+  fi
+  head_checksum=$(grep -i "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
+  if [ "$head_checksum" != "" ]; then
+    log 2 "head checksum shouldn't be returned, is $head_checksum"
+    return 1
+  fi
+  if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" ./tests/rest_scripts/head_object.sh); then
+    log 2 "error getting result: $result"
+    return 1
+  fi
+  head_checksum=$(grep -i "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
+  if [ "$head_checksum" == "" ]; then
+    log 2 "head checksum should be returned"
+    return 1
+  fi
+  return 0
+}

tests/util/util_list_buckets.sh

@@ -79,4 +79,29 @@ list_and_check_buckets() {
     return 1
   fi
   return 0
+}
+
+list_and_check_buckets_omit_without_permission() {
+  if [ $# -ne 4 ]; then
+    log 2 "'list_and_check_buckets_with_user' requires username, password, non-visible bucket, visible bucket"
+    return 1
+  fi
+  if ! list_buckets_with_user "s3api" "$1" "$2"; then
+    log 2 "error listing buckets with user '$1'"
+    return 1
+  fi
+  bucket_found=false
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == "$3" ]; then
+      log 2 "bucket '$3' shouldn't show up in user '$1' bucket list"
+      return 1
+    elif [ "$bucket" == "$4" ]; then
+      bucket_found=true
+    fi
+  done
+  if [ $bucket_found == false ]; then
+    log 2 "user-owned bucket '$4' not found in user list"
+    return 1
+  fi
+  return 0
 }

tests/util/util_setup.sh

@@ -43,3 +43,20 @@ setup_bucket_and_large_file() {
   fi
   return 0
 }
+
+setup_bucket_and_user() {
+  if [ $# -ne 4 ]; then
+    log 2 "'setup_bucket_and_user' requires bucket name, username, password, user type"
+    return 1
+  fi
+  if ! setup_bucket "s3api" "$1"; then
+    log 2 "error setting up bucket"
+    return 1
+  fi
+  if ! result=$(setup_user_versitygw_or_direct "$2" "$3" "$4" "$1"); then
+    log 2 "error setting up user"
+    return 1
+  fi
+  echo "$result"
+  return 0
+}

tests/util/util_users.sh

@@ -100,7 +100,7 @@ create_user_if_nonexistent() {
     log 5 "user $1 already exists"
     return 0
   fi
-  create_user "$1" "$2" "$3"
+  setup_user_versitygw_or_direct "$1" "$2" "$3" "$BUCKET_ONE_NAME"
   return $?
 }