versitygw/tests/util_file.sh

#!/usr/bin/env bats

# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

source ./tests/logger.sh

# create a test file and export folder.  do so in temp folder
# params:  filenames
# return 0 for success, 1 for failure
create_test_files() {
  log 6 "create_test_files"
  if [ $# -lt 1 ]; then
    log 2 "'create_test_files' requires minimum of one file name"
    return 1
  fi
  #test_file_folder=$PWD
  if [[ -z "$GITHUB_ACTIONS" ]]; then
    if ! create_test_file_folder; then
      log 2 "error creating test file folder"
      return 1
    fi
  fi
  for name in "$@"; do
    if ! create_test_file "$name"; then
      log 2 "error creating test file"
      return 1
    fi
  done
  #export test_file_folder
  return 0
}

create_test_file() {
  if [ $# -ne 1 ]; then
    log 2 "'create_test_file' requires name"
    return 1
  fi
  if [[ -e "$TEST_FILE_FOLDER/$name" ]]; then
    if ! error=$(rm "$TEST_FILE_FOLDER/$name" 2>&1); then
      log 2 "error removing old test file: $error"
      return 1
    fi
  fi
  if ! error=$(touch "$TEST_FILE_FOLDER/$name"); then
    log 2 "error creating new test file: $error"
    return 1
  fi
  return 0
}

create_test_file_with_size() {
  if [ $# -ne 2 ]; then
    log 2 "'create test file with size' function requires name, size"
    return 1
  fi
  if ! create_test_file_folder "$1"; then
    log 2 "error creating test file"
    return 1
  fi
  if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1 count="$2" 2>&1); then
    log 2 "error writing file data: $error"
    return 1
  fi
  return 0
}

create_test_folder() {
  if [ $# -lt 1 ]; then
    log 2 "'create_test_folder' command requires at least one folder"
    return 1
  fi
  #test_file_folder=$PWD
  if [[ -z "$GITHUB_ACTIONS" ]]; then
    if ! create_test_file_folder; then
      log 2 "error creating test file folder"
      return 1
    fi
  fi
  for name in "$@"; do
    if ! error=$(run mkdir -p "$TEST_FILE_FOLDER"/"$name" 2>&1); then
      log 2 "error creating test folder $name: $error"
      return 1
    fi
  done
  return 0
}

# delete a test file
# params:  filename
# return:  0 for success, 1 for error
delete_test_files() {
  if [ $# -lt 1 ]; then
    echo "delete test files command missing filenames"
    return 1
  fi
  if [ -z "$TEST_FILE_FOLDER" ]; then
    echo "no test file folder defined, not deleting"
    return 1
  fi
  for name in "$@"; do
    rm -rf "${TEST_FILE_FOLDER:?}"/"${name:?}" || rm_result=$?
    if [[ $rm_result -ne 0 ]]; then
      echo "error deleting file $name"
    fi
  done
  return 0
}

# split file into pieces to test multipart upload
# param: file location
# return 0 for success, 1 for error
split_file() {
  file_size=$(stat -c %s "$1" 2>/dev/null || stat -f %z "$1" 2>/dev/null)
  part_size=$((file_size / $2))
  remainder=$((file_size % $2))
  if [[ remainder -ne 0 ]]; then
    part_size=$((part_size+1))
  fi

  local error
  local split_result
  error=$(split -a 1 -d -b "$part_size" "$1" "$1"-) || split_result=$?
  if [[ $split_result -ne 0 ]]; then
    echo "error splitting file: $error"
    return 1
  fi
  return 0
}

# compare files
# input:  two files
# return 0 for same data, 1 for different data, 2 for error
compare_files() {
  if [ $# -ne 2 ]; then
    echo "file comparison requires two files"
    return 2
  fi
  os=$(uname)
  if [[ $os == "Darwin" ]]; then
    file_one_md5=$(md5 -q "$1")
    file_two_md5=$(md5 -q "$2")
  else
    file_one_md5=$(md5sum "$1" | cut -d " " -f 1)
    file_two_md5=$(md5sum "$2" | cut -d " " -f 1)
  fi
  if [[ $file_one_md5 == "$file_two_md5" ]]; then
    return 0
  fi
  return 1
}

# return 0 on success, 1 on failure
create_test_file_folder() {
  log 6 "create_test_file_folder"
  if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
    # shellcheck disable=SC2035
    if [[ "$error" != *"File exists"* ]]; then
      log 2 "error creating test file folder: $error"
      return 1
    fi
  fi
  export test_file_folder=$TEST_FILE_FOLDER
  return 0
}

# generate 160MB file
# input: filename
# return 0 on success, 1 on failure
create_large_file() {
  log 6 "create_large_file"
  if [ $# -ne 1 ]; then
    log 2 "'create_large_file' requires filename"
    return 1
  fi

  #test_file_folder=$PWD/versity-gwtest-files
  if [[ -z "$GITHUB_ACTIONS" ]]; then
    if ! create_test_file_folder; then
      log 2 "error creating test file"
      return 1
    fi
  fi

  filesize=$((160*1024*1024))
  if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1024 count=$((filesize/1024)) 2>&1); then
    log 2 "error creating large file: $error"
    return 1
  fi
  return 0
}

create_test_file_count() {
  if [[ $# -ne 1 ]]; then
    echo "create test file count function missing bucket name, count"
    return 1
  fi
  #test_file_folder=$PWD
  if [[ -z "$GITHUB_ACTIONS" ]]; then
    create_test_file_folder
  fi
  local touch_result
  for ((i=1;i<=$1;i++)) {
    error=$(touch "$TEST_FILE_FOLDER/file_$i") || touch_result=$?
    if [[ $touch_result -ne 0 ]]; then
      echo "error creating file_$i:  $error"
      return 1
    fi
  }
  # shellcheck disable=SC2153
  if [[ $LOG_LEVEL -ge 5 ]]; then
    ls_result=$(ls "$TEST_FILE_FOLDER/file_*")
    log 5 "$ls_result"
  fi
  return 0
}

download_and_compare_file() {
  if [[ $# -ne 5 ]]; then
    log 2 "'download and compare file' requires command type, original file, bucket, key, local file"
    return 1
  fi
  download_and_compare_file_with_user "$1" "$2" "$3" "$4" "$5" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
  return "$?"
}

download_and_compare_file_with_user() {
  if [[ $# -ne 7 ]]; then
    log 2 "'download and compare file with user' command requires command type, original file, bucket, key, local file, user, password"
    return 1
  fi
  if ! get_object_with_user "$1" "$3" "$4" "$5" "$6" "$7"; then
    log 2 "error retrieving file"
    return 1
  fi
  log 5 "files: $2, $5"
  if ! compare_files "$2" "$5"; then
    log 2 "files don't match"
    return 1
  fi
  return 0
}