From 300edd2c6994f8c45814abd669b5a5e66979c85e Mon Sep 17 00:00:00 2001 From: Shashank Singh <63052147+Shashank1306s@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:38:21 +0530 Subject: [PATCH] [release-1.17] Fix ResourceDeletionStatusTracker key Kind mismatch in terminating-namespace wait (#9987) (#10163) * Fix ResourceDeletionStatusTracker key Kind mismatch in terminating-namespace wait (#9987) EnsureNamespaceExistsAndIsReady wrote the tracker key with namespace.Kind (getNamespace() sets Kind=Namespace) but read it with clusterNS.Kind (client.Get strips TypeMeta -> Kind=empty). The keys never matched, so the skip-path never fired and every item in a terminating namespace paid the full --terminating-resource-timeout wait (per-resource instead of per-namespace). Use the passed-in namespace object for Contains so Add/Contains keys match. Add a regression test that reproduces the production Kind divergence. Signed-off-by: Shashank1306s Co-authored-by: Shashank1306s Co-authored-by: Priyansh Choudhary (cherry picked from commit de32d93b8ee44d65cc6202385aa42830c62acbb5) * Rename changelog to match backport PR number 10163 hack/changelog-check.sh requires changelogs/unreleased/-*, so the backport needs the backport PR's number rather than the original PR's (#9987). Signed-off-by: Shashank1306s * Pin e2e third-party clones to reviewed commits Pin bitnami/containers and distributed-data-generator to fixed SHAs instead of building default-branch HEAD, and add a minimal permissions block. Signed-off-by: chlins (cherry picked from commit 02fe822860ed02ddff044dafd9d1a61a8bb1b85f) * bump the setup kind action update kind setup action to helm/kind-action@v1 and use FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true to suppress warning. Also set cluster_name: "kind" (helm/kind-action defaults to "chart-testing"), matching the follow-up made on main. Signed-off-by: Adam Zhang (cherry picked from commit 7627223d0f24d4720cd635ccec0dd3d22b71b9aa) --------- Signed-off-by: Shashank1306s Signed-off-by: chlins Signed-off-by: Adam Zhang Co-authored-by: Shashank1306s Co-authored-by: Priyansh Choudhary Co-authored-by: chlins Co-authored-by: Adam Zhang --- .github/workflows/e2e-test-kind.yaml | 46 ++++++++++++++--------- changelogs/unreleased/10163-Shashank1306s | 1 + pkg/util/kube/utils.go | 5 ++- pkg/util/kube/utils_test.go | 33 ++++++++++++++++ 4 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 changelogs/unreleased/10163-Shashank1306s diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 662fe0488..577e5d786 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -1,4 +1,14 @@ name: "Run the E2E test on kind" +permissions: + contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + # Reviewed commit pins for third-party sources this workflow clones and executes. + # Bump them deliberately after reviewing the upstream changes. + # bitnami/containers: [bitnami/minio] Release 2026.7.17-debian-12-r0 + BITNAMI_CONTAINERS_COMMIT: 19fb570e551f15ab0c8264aafa93774266761b8d + # vmware-tanzu-experiments/distributed-data-generator: main as of 2025-07-15 + KIBISHII_COMMIT: bce0469e5f9dd33f31432fab22ff90ad6f2b45ca on: push: pull_request: @@ -11,8 +21,6 @@ jobs: # Build the Velero CLI and image once for all Kubernetes versions, and cache it so the fan-out workers can get it. build: runs-on: ubuntu-latest - outputs: - minio-dockerfile-sha: ${{ steps.minio-version.outputs.dockerfile_sha }} steps: - name: Check out the code uses: actions/checkout@v5 @@ -46,24 +54,23 @@ jobs: run: | IMAGE=velero VERSION=pr-test BUILD_OUTPUT_TYPE=docker make container docker save velero:pr-test-linux-amd64 -o ./velero.tar - # Check and build MinIO image once for all e2e tests - - name: Check Bitnami MinIO Dockerfile version - id: minio-version - run: | - DOCKERFILE_SHA=$(curl -s https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2025/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha') - echo "dockerfile_sha=${DOCKERFILE_SHA}" >> $GITHUB_OUTPUT + # Build the MinIO image once for all e2e tests, from the reviewed bitnami/containers commit. - name: Cache MinIO Image uses: actions/cache@v4 id: minio-cache with: path: ./minio-image.tar - key: minio-bitnami-${{ steps.minio-version.outputs.dockerfile_sha }} + key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }} - name: Build MinIO Image from Bitnami Dockerfile if: steps.minio-cache.outputs.cache-hit != 'true' run: | - echo "Building MinIO image from Bitnami Dockerfile..." - git clone --depth 1 https://github.com/bitnami/containers.git /tmp/bitnami-containers - cd /tmp/bitnami-containers/bitnami/minio/2025/debian-12 + set -euo pipefail + echo "Building MinIO image from Bitnami Dockerfile at ${BITNAMI_CONTAINERS_COMMIT}..." + git init -q /tmp/bitnami-containers + git -C /tmp/bitnami-containers remote add origin https://github.com/bitnami/containers.git + git -C /tmp/bitnami-containers fetch --depth 1 origin "${BITNAMI_CONTAINERS_COMMIT}" + git -C /tmp/bitnami-containers checkout -q "${BITNAMI_CONTAINERS_COMMIT}" + cd /tmp/bitnami-containers/bitnami/minio/2026/debian-12 docker build -t bitnami/minio:local . docker save bitnami/minio:local > ${{ github.workspace }}/minio-image.tar # Create json of k8s versions to test @@ -114,7 +121,7 @@ jobs: id: minio-cache with: path: ./minio-image.tar - key: minio-bitnami-${{ needs.build.outputs.minio-dockerfile-sha }} + key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }} - name: Load MinIO Image run: | echo "Loading MinIO image..." @@ -122,11 +129,11 @@ jobs: - name: Install MinIO run: | docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local - - uses: engineerd/setup-kind@v0.6.2 + - uses: helm/kind-action@v1 with: - skipClusterLogsExport: true - version: "v0.27.0" - image: "kindest/node:v${{ matrix.k8s }}" + cluster_name: "kind" + version: "v0.32.0" + node_image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache uses: actions/cache@v4 @@ -154,7 +161,10 @@ jobs: curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - git clone https://github.com/vmware-tanzu-experiments/distributed-data-generator.git -b main /tmp/kibishii + git init -q /tmp/kibishii + git -C /tmp/kibishii remote add origin https://github.com/vmware-tanzu-experiments/distributed-data-generator.git + git -C /tmp/kibishii fetch --depth 1 origin "${KIBISHII_COMMIT}" + git -C /tmp/kibishii checkout -q "${KIBISHII_COMMIT}" GOPATH=~/go \ CLOUD_PROVIDER=kind \ diff --git a/changelogs/unreleased/10163-Shashank1306s b/changelogs/unreleased/10163-Shashank1306s new file mode 100644 index 000000000..4a975b5da --- /dev/null +++ b/changelogs/unreleased/10163-Shashank1306s @@ -0,0 +1 @@ +Fix ResourceDeletionStatusTracker key mismatch so restore into a terminating namespace waits once per namespace instead of once per resource diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 5e5e97603..f69a28ec2 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -102,7 +102,10 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core return true, err } if clusterNS != nil && (clusterNS.GetDeletionTimestamp() != nil || clusterNS.Status.Phase == corev1api.NamespaceTerminating) { - if resourceDeletionStatusTracker.Contains(clusterNS.Kind, clusterNS.Name, clusterNS.Name) { + // Use namespace.Kind (not clusterNS.Kind) so this key matches the one Add() + // writes below: client.Get() strips TypeMeta (Kind=""), but getNamespace() + // sets Kind="Namespace". Mismatched keys made Contains never match. + if resourceDeletionStatusTracker.Contains(namespace.Kind, namespace.Name, namespace.Name) { namespaceAlreadyInDeletionTracker = true return true, errors.Errorf("namespace %s is already present in the polling set, skipping execution", namespace.Name) } diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index df23903a0..b25e40d4c 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -154,6 +154,39 @@ func TestEnsureNamespaceExistsAndIsReady(t *testing.T) { } } +// TestEnsureNamespaceExistsAndIsReadyTerminatingTrackerKindMismatch verifies the +// tracker skip-path fires when Add and Contains see different Kind values, as they +// do in production: getNamespace() sets Kind="Namespace" but client.Get() strips it. +func TestEnsureNamespaceExistsAndIsReadyTerminatingTrackerKindMismatch(t *testing.T) { + // Passed-in namespace mirrors getNamespace(): Kind is set. + namespace := &corev1api.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + } + + // clusterNS mirrors client.Get(): Kind stripped, phase Terminating. + clusterNS := &corev1api.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Status: corev1api.NamespaceStatus{Phase: corev1api.NamespaceTerminating}, + } + + nsClient := &velerotest.FakeNamespaceClient{} + defer nsClient.AssertExpectations(t) + nsClient.On("Get", "test", metav1.GetOptions{}).Return(clusterNS, nil) + + // Seed the tracker as production Add() does. + tracker := NewResourceDeletionStatusTracker() + tracker.Add(namespace.Kind, namespace.Name, namespace.Name) + + result, nsCreated, err := EnsureNamespaceExistsAndIsReady(namespace, nsClient, time.Millisecond, tracker) + + assert.False(t, result) + assert.False(t, nsCreated) + // Skip-path must fire, not the full terminating-resource-timeout wait. + require.ErrorContains(t, err, "skipping polling for terminating namespace") + assert.NotContains(t, err.Error(), "timed out waiting for terminating namespace") +} + // TestGetVolumeDirectorySuccess tests that the GetVolumeDirectory function // returns a volume's name or a volume's name plus '/mount' when a PVC is present. func TestGetVolumeDirectorySuccess(t *testing.T) {