diff --git a/changelogs/unreleased/10652-krishhna24 b/changelogs/unreleased/10652-krishhna24 new file mode 100644 index 000000000..44604341e --- /dev/null +++ b/changelogs/unreleased/10652-krishhna24 @@ -0,0 +1 @@ +Add a make target that runs the e2e tests against a kind cluster and MinIO it creates itself diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh new file mode 100755 index 000000000..fc37fce2c --- /dev/null +++ b/hack/run-e2e-kind.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash + +# Copyright the Velero contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the e2e suite against a kind cluster and a MinIO container that this +# script creates and removes again, so running the tests locally does not need +# a cluster, an object store or a set of environment variables to be assembled +# by hand first. +# +# The cluster's kubeconfig is written to a temporary directory and never merged +# into the caller's: ~/.kube/config is untouched and the current context does +# not change. Everything created is removed on exit, including when the tests +# fail or the run is interrupted. FAIL_FAST=true leaves it all up when the run +# failed, as it does for the suite itself, and CLEANUP=false leaves it up +# whatever the outcome. +# +# Usage: +# make -C test/ run-e2e-kind GINKGO_LABELS='Basic && ClusterResource' +# FEATURES=EnableCSI make -C test/ run-e2e-kind GINKGO_LABELS='BackupVolumeInfo && CSISnapshot' +# CLEANUP=false make -C test/ run-e2e-kind GINKGO_LABELS='Basic && NodePort' + +set -euo pipefail + +KIND_CLUSTER="${KIND_CLUSTER:-velero-e2e}" +KIND_IMAGE="${KIND_IMAGE:-}" +# Named after the cluster so a run cannot remove a MinIO container someone else +# is using. The teardown force-removes this name. +MINIO_CONTAINER="${MINIO_CONTAINER:-minio-${KIND_CLUSTER}}" +# MinIO's own images are no longer pullable anonymously, which is why CI builds +# one from the reviewed bitnami/containers commit instead of pulling. There is +# therefore no tag worth defaulting to that is guaranteed to resolve: the image +# has to be present already, or named explicitly. Checked before anything is +# created, since a pull failure three minutes into a run is a poor way to find out. +MINIO_IMAGE="${MINIO_IMAGE:-minio/minio:latest}" +# The image's own entrypoint is left to run by default, which is what CI relies +# on: its bitnami image starts the server itself and would fail if a command +# replaced that, since it runs as UID 1001 and cannot create a data directory at +# the filesystem root. Upstream minio/minio needs an explicit command instead, +# so set MINIO_ARGS="server /data" for that one. +MINIO_ARGS="${MINIO_ARGS:-}" +MINIO_ROOT_USER="${MINIO_ROOT_USER:-minio}" +MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minio123}" +BSL_BUCKET="${BSL_BUCKET:-bucket}" +ADDITIONAL_BSL_BUCKET="${ADDITIONAL_BSL_BUCKET:-additional-bucket}" + +VELERO_IMAGE="${VELERO_IMAGE:-velero:e2e-local}" +if [ "${VELERO_IMAGE}" = "${VELERO_IMAGE%:*}" ]; then + echo "ERROR: VELERO_IMAGE must include a tag, for example velero:e2e-local." >&2 + echo " Without one the image and tag would both be '${VELERO_IMAGE}'." >&2 + exit 1 +fi +# Skip rebuilding the image and CLI when re-running against an unchanged tree. +SKIP_BUILD="${SKIP_BUILD:-false}" + +FEATURES="${FEATURES:-}" +GINKGO_LABELS="${GINKGO_LABELS:-}" + +# --features takes a comma-separated list, so an exact comparison stops +# matching as soon as a second feature is enabled. Mirrors FeatureEnabled in +# test/types.go. +feature_enabled() { + local want="$1" entry + local IFS=, + for entry in ${FEATURES}; do + entry="$(echo "${entry}" | tr -d '[:space:]')" + if [ "$(echo "${entry}" | tr '[:upper:]' '[:lower:]')" = "$(echo "${want}" | tr '[:upper:]' '[:lower:]')" ]; then + return 0 + fi + done + return 1 +} +FAIL_FAST="${FAIL_FAST:-false}" +# CLEANUP=false keeps the cluster and MinIO after the run whatever the outcome, +# for poking at a passing run. FAIL_FAST only keeps them when the run failed. +CLEANUP="${CLEANUP:-true}" + +repo_root="$(git rev-parse --show-toplevel)" +work_dir="$(mktemp -d -t velero-e2e-XXXXXX)" +export KUBECONFIG="${work_dir}/kubeconfig" +creds_file="${work_dir}/credentials" + +minio_started=false +# Ownership, not just existence. Teardown removes the cluster only if this run +# created it, for the same reason it refuses to remove a MinIO container it did +# not create. +cluster_created=false + +# Printed whenever the test bed is deliberately left behind, so the caller is +# never left guessing what to remove. +keep_notice() { + echo + echo "$1" + echo " cluster: ${KIND_CLUSTER}" + echo " kubeconfig: ${KUBECONFIG}" + echo " MinIO: ${MINIO_CONTAINER}" + echo "Remove them with:" + echo " kind delete cluster --name ${KIND_CLUSTER} && docker rm -f ${MINIO_CONTAINER} && rm -rf ${work_dir}" +} + +cleanup() { + local status=$? + if [ "${CLEANUP}" = "false" ]; then + keep_notice "CLEANUP is false, so the test bed is being kept." + return + fi + if [ "${FAIL_FAST}" = "true" ] && [ "${status}" -ne 0 ]; then + keep_notice "FAIL_FAST is set and the run failed, so the test bed is being kept." + return + fi + echo "==> Removing the test bed" + if [ "${cluster_created}" = "true" ]; then + kind delete cluster --name "${KIND_CLUSTER}" > /dev/null 2>&1 || true + fi + if [ "${minio_started}" = "true" ]; then + docker rm -f "${MINIO_CONTAINER}" > /dev/null 2>&1 || true + fi + rm -rf "${work_dir}" +} +trap cleanup EXIT + +if ! docker image inspect "${MINIO_IMAGE}" > /dev/null 2>&1; then + if ! docker pull -q "${MINIO_IMAGE}" > /dev/null 2>&1; then + echo "ERROR: ${MINIO_IMAGE} is neither present locally nor pullable." >&2 + echo " MinIO's images are not available anonymously, so build one the way" >&2 + echo " CI does, from the bitnami/containers commit pinned in" >&2 + echo " .github/workflows/e2e-test-kind.yaml as BITNAMI_CONTAINERS_COMMIT:" >&2 + echo >&2 + echo " git clone --depth 1 https://github.com/bitnami/containers /tmp/bitnami-containers" >&2 + echo " docker build -t minio:local /tmp/bitnami-containers/bitnami/minio//debian-12" >&2 + echo " MINIO_IMAGE=minio:local make -C test/ run-e2e-kind" >&2 + echo >&2 + echo " Or set MINIO_IMAGE to any S3-compatible image you already have." >&2 + exit 1 + fi +fi + +if kind get clusters 2>/dev/null | grep -qx "${KIND_CLUSTER}"; then + echo "ERROR: a kind cluster named ${KIND_CLUSTER} already exists, and this script" >&2 + echo " will not delete one it did not create. Remove it yourself, or set" >&2 + echo " KIND_CLUSTER to a name that is free." >&2 + exit 1 +fi + +if docker inspect "${MINIO_CONTAINER}" > /dev/null 2>&1; then + echo "ERROR: a container named ${MINIO_CONTAINER} already exists, and this script" >&2 + echo " will not remove one it did not create. Remove it yourself, or set" >&2 + echo " MINIO_CONTAINER or KIND_CLUSTER to a name that is free." >&2 + exit 1 +fi + +echo "==> Creating kind cluster ${KIND_CLUSTER}" +# --kubeconfig keeps this out of the caller's kubeconfig entirely. +# Set before the command rather than after, so a cluster left half created by a +# failure part way through is still torn down. +cluster_created=true +if [ -n "${KIND_IMAGE}" ]; then + kind create cluster --name "${KIND_CLUSTER}" --image "${KIND_IMAGE}" --kubeconfig "${KUBECONFIG}" --wait 120s +else + kind create cluster --name "${KIND_CLUSTER}" --kubeconfig "${KUBECONFIG}" --wait 120s +fi + +echo "==> Starting MinIO" +# Published on an ephemeral host port rather than 9000, so a MinIO already +# running on the usual port does not stop the run. The port is published on all +# interfaces because the cluster reaches it over the kind gateway, so this is a +# throwaway store with throwaway credentials and nothing else should use it. +# shellcheck disable=SC2086 # MINIO_ARGS is a command line and must word-split. +docker run -d --name "${MINIO_CONTAINER}" -p 9000 \ + -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ + -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ + "${MINIO_IMAGE}" ${MINIO_ARGS} > /dev/null +minio_started=true +minio_port="$(docker port "${MINIO_CONTAINER}" 9000/tcp | head -1 | sed 's/.*://')" + +# mc ships inside the MinIO image. minio/mc is no longer available on Docker Hub. +minio_ready=false +for _ in $(seq 1 30); do + if docker exec "${MINIO_CONTAINER}" mc alias set local \ + "http://127.0.0.1:9000" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" > /dev/null 2>&1; then + minio_ready=true + break + fi + sleep 2 +done +# Without this the loop falls through and the next command fails on its own, +# reporting a bucket error for what is really a MinIO that never came up. +if [ "${minio_ready}" != "true" ]; then + echo "ERROR: MinIO did not become ready within 60s. Container logs:" >&2 + docker logs --tail 20 "${MINIO_CONTAINER}" >&2 || true + if [ -z "${MINIO_ARGS}" ]; then + echo >&2 + echo " If the logs show a usage message, this image needs an explicit" >&2 + echo " command. Upstream minio/minio does: MINIO_ARGS=\"server /data\"." >&2 + fi + exit 1 +fi +docker exec "${MINIO_CONTAINER}" mc mb --ignore-existing \ + "local/${BSL_BUCKET}" "local/${ADDITIONAL_BSL_BUCKET}" > /dev/null + +# The cluster reaches MinIO over the kind network's gateway. Do not use +# `hostname -i`, which resolves to a different address under WSL2 and leaves the +# BackupStorageLocation unreachable from inside the cluster. +minio_host="$(docker network inspect kind \ + -f '{{range .IPAM.Config}}{{.Gateway}} {{end}}' | tr ' ' '\n' | grep -m1 '\.')" +bsl_config="region=minio,s3ForcePathStyle=\"true\",s3Url=http://${minio_host}:${minio_port}" +echo "==> MinIO reachable at ${minio_host}:${minio_port}" + +cat > "${creds_file}" <: with BUILD_OUTPUT_TYPE=docker the +# Makefile fixes ALL_OS to linux and takes only the architecture from go env, so +# the OS is linux even on a Darwin host, which still needs a Linux image to load +# into kind. Hardcoding the architecture instead would load a tag that does not +# exist on an arm64 workstation. +velero_image_arch="$(go env GOARCH 2>/dev/null)" +if [ -z "${velero_image_arch}" ]; then + echo "ERROR: could not read the build architecture from 'go env'. The image tag the" >&2 + echo " container target produces ends in -linux-, so this script needs" >&2 + echo " go on PATH to know which tag to load." >&2 + exit 1 +fi +velero_image_built="${VELERO_IMAGE}-linux-${velero_image_arch}" + +if [ "${SKIP_BUILD}" != "true" ]; then + echo "==> Building the Velero CLI and image" + make -C "${repo_root}" local + IMAGE="${velero_image_name}" VERSION="${velero_image_tag}" BUILD_OUTPUT_TYPE=docker \ + make -C "${repo_root}" container +fi +echo "==> Loading ${velero_image_built} into the cluster" +kind load docker-image "${velero_image_built}" --name "${KIND_CLUSTER}" + +# The suite's own StorageClass comes from testdata/storage-class/.yaml +# unless told otherwise, and the kind one provisions with rancher.io/local-path. +# A CSI run needs a CSI-backed class or the driver installed below has nothing +# to snapshot, which is the same file the CSI job in CI passes. +storage_class_file="" +if feature_enabled EnableCSI; then + if [ -x "${repo_root}/hack/install-csi-hostpath.sh" ]; then + echo "==> Installing CSI snapshot support" + "${repo_root}/hack/install-csi-hostpath.sh" + else + echo "ERROR: FEATURES=EnableCSI needs hack/install-csi-hostpath.sh to install a" >&2 + echo " CSI driver that can snapshot; kind's default storage cannot." >&2 + exit 1 + fi + storage_class_file="../testdata/storage-class/kind-csi.yaml" +fi + +echo "==> Running the e2e suite" +CLOUD_PROVIDER=kind \ +OBJECT_STORE_PROVIDER=aws \ +FEATURES="${FEATURES}" \ +BSL_CONFIG="${bsl_config}" \ +BSL_BUCKET="${BSL_BUCKET}" \ +CREDS_FILE="${creds_file}" \ +ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ +ADDITIONAL_BSL_CONFIG="${bsl_config}" \ +ADDITIONAL_BSL_BUCKET="${ADDITIONAL_BSL_BUCKET}" \ +ADDITIONAL_CREDS_FILE="${creds_file}" \ +GINKGO_LABELS="${GINKGO_LABELS}" \ +STORAGE_CLASS_FILE="${storage_class_file}" \ +FAIL_FAST="${FAIL_FAST}" \ +make -C "${repo_root}/test" run-e2e VELERO_IMAGE="${velero_image_built}" diff --git a/test/Makefile b/test/Makefile index 1ff7cef10..aba3f8c69 100644 --- a/test/Makefile +++ b/test/Makefile @@ -192,6 +192,10 @@ ifneq ($(origin STORAGE_CLASS_FILE_2),undefined) COMMON_ARGS += --storage-class-file-2=$(STORAGE_CLASS_FILE_2) endif +.PHONY: run-e2e-kind +run-e2e-kind: ## Run the e2e tests against a kind cluster and MinIO that this target creates and removes + @$(REPO_ROOT)/hack/run-e2e-kind.sh + # Make sure ginkgo is in $GOBIN .PHONY:ginkgo ginkgo: ${GOBIN}/ginkgo