velero/.github/workflows/e2e-test-kind.yaml

name: "Run the E2E test on kind"
permissions:
  contents: read
env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
  # Reviewed commit pins for third-party sources this workflow clones and executes.
  # Bump them deliberately after reviewing the upstream changes.
  # bitnami/containers: [bitnami/minio] Release 2026.7.17-debian-12-r0
  BITNAMI_CONTAINERS_COMMIT: 19fb570e551f15ab0c8264aafa93774266761b8d
  # vmware-tanzu-experiments/distributed-data-generator: main as of 2026-10-15
  KIBISHII_COMMIT: f81fc67f52b9cafe945f7d44ca0507ff65ea4cbd
on:
  push:
  pull_request:
    # Only run when the change affects Go code or a workflow this job depends on.
    paths:
      - "**/*.go"
      - "go.mod"
      - "go.sum"
      - ".github/workflows/e2e-test-kind.yaml"
      - ".github/workflows/get-go-version.yaml"
jobs:
  get-go-version:
    uses: ./.github/workflows/get-go-version.yaml
    with:
      ref: ${{ github.event.pull_request.base.ref }}

  # Build the Velero CLI and image once for all Kubernetes versions, and cache it so the fan-out workers can get it.
  build:
    runs-on: ubuntu-latest
    needs: get-go-version
    steps:
      - name: Check out the code
        uses: actions/checkout@v7
      
      - name: Set up Go version
        uses: actions/setup-go@v7
        with:
          go-version: ${{ needs.get-go-version.outputs.version }}
          cache: false

      # One Go module cache for this job and the e2e jobs. The key leaves out the
      # Go patch version, which differs between runner images in the same run.
      - name: Restore Go module cache
        id: go-mod-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/go/pkg/mod
          key: go-mod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
          restore-keys: go-mod-${{ runner.os }}-${{ runner.arch }}-
      # Download everything the e2e jobs need, including the ginkgo CLI's own
      # dependencies, so they can run from the cache.
      - name: Download Go modules
        if: steps.go-mod-cache.outputs.cache-hit != 'true'
        run: |
          go mod download
          make -C test/ ginkgo

      # Look for a CLI that's made for this PR
      - name: Fetch built CLI
        id: cli-cache
        uses: actions/cache/restore@v6
        with:
          path: ./_output/bin/linux/amd64/velero
          # The cache key a combination of the current PR number and the commit SHA
          key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Fetch built image
        id: image-cache
        uses: actions/cache/restore@v6
        with:
          path: ./velero.tar
          # The cache key a combination of the current PR number and the commit SHA
          key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }}
      # If no binaries were built for this PR, build it now.
      - name: Build Velero CLI
        if: steps.cli-cache.outputs.cache-hit != 'true'
        run: |
          make local
      # If no image were built for this PR, build it now.
      - name: Build Velero Image
        if: steps.image-cache.outputs.cache-hit != 'true'
        run: |
          IMAGE=velero VERSION=pr-test BUILD_OUTPUT_TYPE=docker make container
          docker save velero:pr-test-linux-amd64 -o ./velero.tar
      # Build the MinIO image once for all e2e tests, from the reviewed bitnami/containers commit.
      - name: Cache MinIO Image
        uses: actions/cache/restore@v6
        id: minio-cache
        with:
          path: ./minio-image.tar
          key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }}
      - name: Build MinIO Image from Bitnami Dockerfile
        if: steps.minio-cache.outputs.cache-hit != 'true'
        run: |
          set -euo pipefail
          echo "Building MinIO image from Bitnami Dockerfile at ${BITNAMI_CONTAINERS_COMMIT}..."
          git init -q /tmp/bitnami-containers
          git -C /tmp/bitnami-containers remote add origin https://github.com/bitnami/containers.git
          git -C /tmp/bitnami-containers fetch --depth 1 origin "${BITNAMI_CONTAINERS_COMMIT}"
          git -C /tmp/bitnami-containers checkout -q "${BITNAMI_CONTAINERS_COMMIT}"
          cd /tmp/bitnami-containers/bitnami/minio/2026/debian-12
          docker build -t bitnami/minio:local .
          docker save bitnami/minio:local > ${{ github.workspace }}/minio-image.tar
      # Save the freshly built artifacts to the cache explicitly, before this
      # job reports completion. actions/cache saves in a post-job hook that
      # runs *after* the job finishes, so the dependent run-e2e-test jobs (which
      # start as soon as build completes) would race the save and miss the cache
      # on a force push. See #9927.
      - name: Save built CLI to cache
        if: steps.cli-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ./_output/bin/linux/amd64/velero
          key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Save built image to cache
        if: steps.image-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ./velero.tar
          key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Save MinIO image to cache
        if: steps.minio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ./minio-image.tar
          key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }}
      - name: Save Go module cache
        if: steps.go-mod-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/go/pkg/mod
          key: ${{ steps.go-mod-cache.outputs.cache-primary-key }}
  # Create json of k8s versions to test
  # from guide: https://stackoverflow.com/a/65094398/4590470
  setup-test-matrix:
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ github.token }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      csi-matrix: ${{ steps.set-matrix.outputs.csi-matrix }}
    steps:
      - name: Set k8s versions
        id: set-matrix
        # everything excluding older tags. limits needs to be high enough to cover all latest versions
        # and test labels
        # grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" filters for well-formed v1.25.x to v9.99.x
        # GA releases only, so a pre-release tag like v1.37.0-rc.1 can't reach the
        # awk step below and be misparsed as a patch release (e.g. "1.37.1")
        # and removes older patches of the same minor version
        # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}'
        run: |
          set -euo pipefail
          candidates=$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g)

          # Docker Hub's tag listing can include tags whose manifest was never
          # published or has since been removed (e.g. kindest/node:v1.37.1). If such
          # a tag reaches the matrix, its job is guaranteed to fail with
          # "manifest unknown" once helm/kind-action tries to pull it, which clogs
          # up the e2e queue on every PR with a red job unrelated to the change
          # under test. Test-pull each candidate's manifest here and drop any tag
          # that isn't actually available before building the matrix.
          valid=()
          while IFS= read -r v; do
            [ -z "$v" ] && continue
            echo "Verifying kindest/node:v${v} image is available..."
            if docker manifest inspect "kindest/node:v${v}" > /dev/null 2>&1; then
              valid+=("$v")
            else
              echo "::warning::kindest/node:v${v} manifest not found on Docker Hub; excluding from e2e test matrix"
            fi
          done <<< "$candidates"

          if [ ${#valid[@]} -eq 0 ]; then
            echo "::warning::No kindest/node tags passed the manifest availability check; the e2e test matrix will have no Kubernetes versions to test"
          fi

          k8s_json=$(printf '%s\n' "${valid[@]+"${valid[@]}"}" | jq -R -c -s 'split("\n") | map(select(length > 0))')

          echo "matrix={\
            \"k8s\":${k8s_json},\
            \"labels\":[\
              \"Basic && (ClusterResource || NodePort || StorageClass)\", \
              \"ResourceFiltering && !FSBackup\", \
              \"ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources\", \
              \"(NamespaceMapping && Single && FSBackup) || (NamespaceMapping && Multiple && FSBackup)\"\
            ]}" >> $GITHUB_OUTPUT

          # The CSI cases need a snapshot stack installed per job, which the
          # other label sets do not, so they run as their own job rather than
          # adding that cost to all of them.
          #
          # The newest three versions rather than every one. csi-driver-host-path
          # ships deploy manifests only for recent Kubernetes releases, so the
          # older end of the matrix has nothing to install, and running every
          # version would add about a quarter to the matrix for coverage of a
          # driver that is only used by this job.
          csi_k8s_json=$(printf '%s\n' "${valid[@]+"${valid[@]}"}" | jq -R -c -s '
            split("\n") | map(select(length > 0))
            | sort_by(split(".") | map(tonumber))
            | .[-3:]')
          # Every CSI label is listed explicitly. A Ginkgo label filter matches a
          # bare token by exact equality, not as a substring, so CSISnapshot does
          # not select CSIVolumeGroupSnapshot. A new CSI label has to be added
          # here or its cases silently never run.
          #
          # The Kibishii-backed Snapshot cases run here rather than in
          # run-e2e-test because Kibishii brings its own StorageClass, and only
          # the kind-csi overlay this job reaches defines it on a CSI driver.
          echo "csi-matrix={\
            \"k8s\":${csi_k8s_json},\
            \"labels\":[\
              \"BackupVolumeInfo && (CSISnapshot || CSIDataMover || CSIVolumeGroupSnapshot)\", \
              \"Basic && Snapshot\", \
              \"Backups && Deletion && Snapshot\", \
              \"NamespaceMapping && Snapshot\", \
              \"Basic && FSBackup\", \
              \"BSL && Deletion && Snapshot\", \
              \"Backups && Deletion && FSBackup\"\
            ]}" >> $GITHUB_OUTPUT

  # Run E2E test against all Kubernetes versions on kind
  run-e2e-test:
    needs:
      - build
      - setup-test-matrix
      - get-go-version
    runs-on: ubuntu-latest
    strategy:
      matrix: ${{fromJson(needs.setup-test-matrix.outputs.matrix)}}
      fail-fast: false
    steps:
      - name: Check out the code
        uses: actions/checkout@v7

      - name: Set up Go version
        uses: actions/setup-go@v7
        with:
          go-version: ${{ needs.get-go-version.outputs.version }}
          cache: false
      - name: Restore Go module cache
        uses: actions/cache/restore@v6
        with:
          path: ~/go/pkg/mod
          key: go-mod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
          restore-keys: go-mod-${{ runner.os }}-${{ runner.arch }}-

      # Fetch the pre-built MinIO image from the build job
      - name: Fetch built MinIO Image
        uses: actions/cache/restore@v6
        id: minio-cache
        with:
          path: ./minio-image.tar
          key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }}
      - name: Load MinIO Image
        run: |
          echo "Loading MinIO image..."
          docker load < ./minio-image.tar
      - name: Install MinIO
        run: |
          docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local
      - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1 + fix: add curl retry flags (https://github.com/helm/kind-action/pull/165)
        with:
          cluster_name: "kind"
          version: "v0.32.0"
          node_image: "kindest/node:v${{ matrix.k8s }}"
      - name: Fetch built CLI
        id: cli-cache
        uses: actions/cache/restore@v6
        with:
          path: ./_output/bin/linux/amd64/velero
          key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Fetch built Image
        id: image-cache
        uses: actions/cache/restore@v6
        with:
          path: ./velero.tar
          key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Load Velero Image
        run:
          kind load image-archive velero.tar
      - name: Run E2E test
        run: |
          cat << EOF > /tmp/credential
          [default]
          aws_access_key_id=minio
          aws_secret_access_key=minio123
          EOF

          # Match kubectl version to k8s server version
          curl -fLO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

          git init -q /tmp/kibishii
          git -C /tmp/kibishii remote add origin https://github.com/vmware-tanzu-experiments/distributed-data-generator.git
          git -C /tmp/kibishii fetch --depth 1 origin "${KIBISHII_COMMIT}"
          git -C /tmp/kibishii checkout -q "${KIBISHII_COMMIT}"

          GOPATH=~/go \
              CLOUD_PROVIDER=kind \
              OBJECT_STORE_PROVIDER=aws \
              BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              CREDS_FILE=/tmp/credential \
              BSL_BUCKET=bucket \
              ADDITIONAL_OBJECT_STORE_PROVIDER=aws \
              ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              ADDITIONAL_CREDS_FILE=/tmp/credential \
              ADDITIONAL_BSL_BUCKET=additional-bucket \
              VELERO_IMAGE=velero:pr-test-linux-amd64 \
              PLUGINS=velero/velero-plugin-for-aws:latest \
              GINKGO_LABELS="${{ matrix.labels }}" \
              KIBISHII_DIRECTORY=/tmp/kibishii/kubernetes/yaml/ \
              make -C test/ run-e2e
        timeout-minutes: 30
      - name: Upload debug bundle
        if: ${{ failure() }}
        uses: actions/upload-artifact@v7
        with:
          name: DebugBundle-k8s-${{ matrix.k8s }}-job-${{ strategy.job-index }}
          path: /home/runner/work/velero/velero/test/e2e/debug-bundle*

  # The CSI cases need a driver that can take snapshots, which kind does not
  # have, so they run as their own job with the snapshot stack installed and the
  # StorageClass pointed at it. Everything else mirrors run-e2e-test.
  run-csi-e2e-test:
    needs:
      - build
      - setup-test-matrix
      - get-go-version
    runs-on: ubuntu-latest
    strategy:
      matrix: ${{fromJson(needs.setup-test-matrix.outputs.csi-matrix)}}
      fail-fast: false
    steps:
      - name: Check out the code
        uses: actions/checkout@v7

      - name: Set up Go version
        uses: actions/setup-go@v7
        with:
          go-version: ${{ needs.get-go-version.outputs.version }}
          cache: false
      - name: Restore Go module cache
        uses: actions/cache/restore@v6
        with:
          path: ~/go/pkg/mod
          key: go-mod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
          restore-keys: go-mod-${{ runner.os }}-${{ runner.arch }}-

      # Fetch the pre-built MinIO image from the build job
      - name: Fetch built MinIO Image
        uses: actions/cache/restore@v6
        id: minio-cache
        with:
          path: ./minio-image.tar
          key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }}
      - name: Load MinIO Image
        run: |
          echo "Loading MinIO image..."
          docker load < ./minio-image.tar
      - name: Install MinIO
        run: |
          docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local
      - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1 + fix: add curl retry flags (https://github.com/helm/kind-action/pull/165)
        with:
          cluster_name: "kind"
          version: "v0.32.0"
          node_image: "kindest/node:v${{ matrix.k8s }}"

      # Match kubectl to the server version before the installer uses it
      - name: Install kubectl
        run: |
          curl -fLO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
      - name: Install CSI snapshot support
        run: ./hack/install-csi-hostpath.sh
        timeout-minutes: 10

      - name: Fetch built CLI
        id: cli-cache
        uses: actions/cache/restore@v6
        with:
          path: ./_output/bin/linux/amd64/velero
          key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Fetch built Image
        id: image-cache
        uses: actions/cache/restore@v6
        with:
          path: ./velero.tar
          key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }}
      - name: Load Velero Image
        run:
          kind load image-archive velero.tar
      - name: Run CSI E2E test
        run: |
          cat << EOF > /tmp/credential
          [default]
          aws_access_key_id=minio
          aws_secret_access_key=minio123
          EOF

          git init -q /tmp/kibishii
          git -C /tmp/kibishii remote add origin https://github.com/vmware-tanzu-experiments/distributed-data-generator.git
          git -C /tmp/kibishii fetch --depth 1 origin "${KIBISHII_COMMIT}"
          git -C /tmp/kibishii checkout -q "${KIBISHII_COMMIT}"

          GOPATH=~/go \
              CLOUD_PROVIDER=kind \
              OBJECT_STORE_PROVIDER=aws \
              FEATURES=EnableCSI \
              BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              CREDS_FILE=/tmp/credential \
              BSL_BUCKET=bucket \
              ADDITIONAL_OBJECT_STORE_PROVIDER=aws \
              ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              ADDITIONAL_CREDS_FILE=/tmp/credential \
              ADDITIONAL_BSL_BUCKET=additional-bucket \
              VELERO_IMAGE=velero:pr-test-linux-amd64 \
              PLUGINS=velero/velero-plugin-for-aws:latest \
              STORAGE_CLASS_FILE=../testdata/storage-class/kind-csi.yaml \
              GINKGO_LABELS="${{ matrix.labels }}" \
              KIBISHII_DIRECTORY=/tmp/kibishii/kubernetes/yaml/ \
              make -C test/ run-e2e
        timeout-minutes: 30
      - name: Upload debug bundle
        if: ${{ failure() }}
        uses: actions/upload-artifact@v7
        with:
          name: CSIDebugBundle-k8s-${{ matrix.k8s }}-job-${{ strategy.job-index }}
          path: /home/runner/work/velero/velero/test/e2e/debug-bundle*