Dump the github context content

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>

.github/auto-assignees.yml

@@ -16,6 +16,7 @@ reviewers:
       - shubham-pampattiwar
       - Lyndon-Li
       - anshulahuja98
+      - kaovilai
 
     tech-writer:
       - sseago

.github/workflows/crds-verify-kind.yaml

@@ -1,79 +0,0 @@
-name: "Verify Velero CRDs across k8s versions"
-on:
-  pull_request:
-    # Do not run when the change only includes these directories.
-    paths-ignore:
-      - "site/**"
-      - "design/**"
-
-jobs:
-  # Build the Velero CLI once for all Kubernetes versions, and cache it so the fan-out workers can get it.
-  build-cli:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-      # Look for a CLI that's made for this PR
-      - name: Fetch built CLI
-        id: cache
-        uses: actions/cache@v4
-        env:
-          cache-name: cache-velero-cli
-        with:
-          path: ./_output/bin/linux/amd64/velero
-          # The cache key a combination of the current PR number, and a SHA256 hash of the Velero binary
-          key: velero-${{ github.event.pull_request.number }}-${{ hashFiles('./_output/bin/linux/amd64/velero') }}
-          # This key controls the prefixes that we'll look at in the cache to restore from
-          restore-keys: |
-            velero-${{ github.event.pull_request.number }}-
-      # If no binaries were built for this PR, build it now.
-      - name: Build Velero CLI
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          make local
-  # Check the common CLI against all Kubernetes versions
-  crd-check:
-    needs: build-cli
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # Latest k8s versions. There's no series-based tag, nor is there a latest tag.
-        k8s:
-          - 1.23.17
-          - 1.24.17
-          - 1.25.16
-          - 1.26.13
-          - 1.27.10
-          - 1.28.6
-          - 1.29.1
-    # All steps run in parallel unless otherwise specified.
-    # See https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#creating-dependent-jobs
-    steps:
-      - name: Fetch built CLI
-        id: cache
-        uses: actions/cache@v4
-        env:
-          cache-name: cache-velero-cli
-        with:
-          path: ./_output/bin/linux/amd64/velero
-          # The cache key a combination of the current PR number, and a SHA256 hash of the Velero binary
-          key: velero-${{ github.event.pull_request.number }}-${{ hashFiles('./_output/bin/linux/amd64/velero') }}
-          # This key controls the prefixes that we'll look at in the cache to restore from
-          restore-keys: |
-            velero-${{ github.event.pull_request.number }}-
-      - uses: engineerd/setup-kind@v0.5.0
-        with:
-          version: "v0.21.0"
-          image: "kindest/node:v${{ matrix.k8s }}"
-      - name: Install CRDs
-        run: |
-          kubectl cluster-info
-          kubectl get pods -n kube-system
-          kubectl version
-          echo "current-context:" $(kubectl config current-context)
-          echo "environment-kubeconfig:" ${KUBECONFIG}
-          ./_output/bin/linux/amd64/velero install --crds-only --dry-run -oyaml | kubectl apply -f -

.github/workflows/e2e-test-kind.yaml

@@ -6,17 +6,28 @@ on:
     paths-ignore:
       - "site/**"
       - "design/**"
+      - "**/*.md"
 jobs:
+  get-go-version:
+    uses: ./.github/workflows/get-go-version.yaml
+    with:
+      ref: ${{ github.event.pull_request.base.ref }}
+
   # Build the Velero CLI and image once for all Kubernetes versions, and cache it so the fan-out workers can get it.
   build:
     runs-on: ubuntu-latest
+    needs: get-go-version
+    outputs:
+      minio-dockerfile-sha: ${{ steps.minio-version.outputs.dockerfile_sha }}
     steps:
       - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/checkout@v5
+      
+      - name: Set up Go version
+        uses: actions/setup-go@v6
         with:
-          go-version-file: 'go.mod'
+          go-version: ${{ needs.get-go-version.outputs.version }}
+
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
         id: cli-cache
@@ -41,42 +52,91 @@ jobs:
       - name: Build Velero Image
         if: steps.image-cache.outputs.cache-hit != 'true'
         run: |
-          IMAGE=velero VERSION=pr-test make container
-          docker save velero:pr-test -o ./velero.tar
+          IMAGE=velero VERSION=pr-test BUILD_OUTPUT_TYPE=docker make container
+          docker save velero:pr-test-linux-amd64 -o ./velero.tar
+      # Check and build MinIO image once for all e2e tests
+      - name: Check Bitnami MinIO Dockerfile version
+        id: minio-version
+        run: |
+          DOCKERFILE_SHA=$(curl -s https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2025/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha')
+          echo "dockerfile_sha=${DOCKERFILE_SHA}" >> $GITHUB_OUTPUT
+      - name: Cache MinIO Image
+        uses: actions/cache@v4
+        id: minio-cache
+        with:
+          path: ./minio-image.tar
+          key: minio-bitnami-${{ steps.minio-version.outputs.dockerfile_sha }}
+      - name: Build MinIO Image from Bitnami Dockerfile
+        if: steps.minio-cache.outputs.cache-hit != 'true'
+        run: |
+          echo "Building MinIO image from Bitnami Dockerfile..."
+          git clone --depth 1 https://github.com/bitnami/containers.git /tmp/bitnami-containers
+          cd /tmp/bitnami-containers/bitnami/minio/2025/debian-12
+          docker build -t bitnami/minio:local .
+          docker save bitnami/minio:local > ${{ github.workspace }}/minio-image.tar
+  # Create json of k8s versions to test
+  # from guide: https://stackoverflow.com/a/65094398/4590470
+  setup-test-matrix:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Set k8s versions
+        id: set-matrix
+        # everything excluding older tags. limits needs to be high enough to cover all latest versions
+        # and test labels
+        # grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" filters for v1.25 to v9.99
+        # and removes older patches of the same minor version
+        # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}'
+        run: |
+          echo "matrix={\
+            \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\
+            \"labels\":[\
+              \"Basic && (ClusterResource || NodePort || StorageClass)\", \
+              \"ResourceFiltering && !Restic\", \
+              \"ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources\", \
+              \"(NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)\"\
+            ]}" >> $GITHUB_OUTPUT
+
   # Run E2E test against all Kubernetes versions on kind
   run-e2e-test:
-    needs: build
+    needs:
+      - build
+      - setup-test-matrix
+      - get-go-version
     runs-on: ubuntu-latest
     strategy:
-      matrix:
-        k8s:
-          - 1.23.17
-          - 1.24.17
-          - 1.25.16
-          - 1.26.13
-          - 1.27.10
-          - 1.28.6
-          - 1.29.1
-        labels:
-          # labels are used to filter running E2E cases
-          - Basic && (ClusterResource || NodePort || StorageClass)
-          - ResourceFiltering && !Restic
-          - ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources
-          - (NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)
+      matrix: ${{fromJson(needs.setup-test-matrix.outputs.matrix)}}
       fail-fast: false
     steps:
       - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/checkout@v5
+
+      - name: Set up Go version
+        uses: actions/setup-go@v6
         with:
-          go-version-file: 'go.mod'
+          go-version: ${{ needs.get-go-version.outputs.version }}
+
+      # Fetch the pre-built MinIO image from the build job
+      - name: Fetch built MinIO Image
+        uses: actions/cache@v4
+        id: minio-cache
+        with:
+          path: ./minio-image.tar
+          key: minio-bitnami-${{ needs.build.outputs.minio-dockerfile-sha }}
+      - name: Load MinIO Image
+        run: |
+          echo "Loading MinIO image..."
+          docker load < ./minio-image.tar
       - name: Install MinIO
-        run:
-          docker run -d --rm -p 9000:9000 -e "MINIO_ACCESS_KEY=minio" -e "MINIO_SECRET_KEY=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:2021.6.17-debian-10-r7
-      - uses: engineerd/setup-kind@v0.5.0
+        run: |
+          docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local
+      - uses: engineerd/setup-kind@v0.6.2
         with:
-          version: "v0.21.0"
+          skipClusterLogsExport: true
+          version: "v0.27.0"
           image: "kindest/node:v${{ matrix.k8s }}"
       - name: Fetch built CLI
         id: cli-cache
@@ -105,6 +165,8 @@ jobs:
           curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
           sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
 
+          git clone https://github.com/vmware-tanzu-experiments/distributed-data-generator.git -b main /tmp/kibishii
+
           GOPATH=~/go \
               CLOUD_PROVIDER=kind \
               OBJECT_STORE_PROVIDER=aws \
@@ -115,8 +177,10 @@ jobs:
               ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
               ADDITIONAL_CREDS_FILE=/tmp/credential \
               ADDITIONAL_BSL_BUCKET=additional-bucket \
-              VELERO_IMAGE=velero:pr-test \
+              VELERO_IMAGE=velero:pr-test-linux-amd64 \
+              PLUGINS=velero/velero-plugin-for-aws:latest \
               GINKGO_LABELS="${{ matrix.labels }}" \
+              KIBISHII_DIRECTORY=/tmp/kibishii/kubernetes/yaml/ \
               make -C test/ run-e2e
         timeout-minutes: 30
       - name: Upload debug bundle

.github/workflows/get-go-version.yaml

@@ -0,0 +1,38 @@
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: "The target branch's ref"
+        required: true
+        type: string
+    outputs:
+      version: 
+        description: "The expected Go version"
+        value: ${{ jobs.extract.outputs.version }}
+
+jobs:
+  extract:
+      runs-on: ubuntu-latest
+      outputs:
+        version: ${{ steps.pick-version.outputs.version }}
+      steps:
+        - name: Dump github context
+          env:
+            GITHUB_CONTEXT: ${{ toJson(github) }}
+          run: echo "$GITHUB_CONTEXT"
+
+        - name: Check out the code
+          uses: actions/checkout@v5
+
+        - id: pick-version
+          run: |
+            if [ "${{ inputs.ref }}" == "main" ]; then
+              version=$(grep '^go ' go.mod | awk '{print $2}' | cut -d. -f1-2)
+            else
+              goDirectiveVersion=$(grep '^go ' go.mod | awk '{print $2}')
+              toolChainVersion=$(grep '^toolchain ' go.mod | awk '{print $2}')
+              version=$(printf "%s\n%s\n" "$goDirectiveVersion" "$toolChainVersion" | sort -V | tail -n1)
+            fi
+
+            echo "version=$version"
+            echo "version=$version" >> $GITHUB_OUTPUT

.github/workflows/nightly-trivy-scan.yml

@@ -13,13 +13,13 @@ jobs:
         # maintain the versions of Velero those need security scan
         versions: [main]
         # list of images that need scan
-        images: [velero, velero-restore-helper]
+        images: [velero, velero-plugin-for-aws, velero-plugin-for-gcp, velero-plugin-for-microsoft-azure]
     permissions:
       security-events: write  # for github/codeql-action/upload-sarif to upload SARIF results
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master

.github/workflows/pr-changelog-check.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
 
     - name: Check out the code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     - name: Changelog check
       if: ${{ !(contains(github.event.pull_request.labels.*.name, 'kind/changelog-not-required') || contains(github.event.pull_request.labels.*.name, 'Design') || contains(github.event.pull_request.labels.*.name, 'Website') || contains(github.event.pull_request.labels.*.name, 'Documentation'))}}

.github/workflows/pr-ci-check.yml

@@ -1,22 +1,30 @@
 name: Pull Request CI Check
 on: [pull_request]
 jobs:
+  get-go-version:
+    uses: ./.github/workflows/get-go-version.yaml
+    with:
+      ref: ${{ github.event.pull_request.base.ref }}
+
   build:
     name: Run CI
+    needs: get-go-version
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
       - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/checkout@v5
+
+      - name: Set up Go version
+        uses: actions/setup-go@v6
         with:
-          go-version-file: 'go.mod'
+          go-version: ${{ needs.get-go-version.outputs.version }}      
+
       - name: Make ci
         run: make ci
       - name: Upload test coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.out

.github/workflows/pr-codespell.yml

@@ -8,14 +8,14 @@ jobs:
     steps:
 
     - name: Check out the code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     - name: Codespell
       uses: codespell-project/actions-codespell@master
       with:
-        # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere.
+        # ignore the config/.../crd.go file as it's generated binary data that is edited elsewhere.
         skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE
-        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin
+        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin,sie
         check_filenames: true
         check_hidden: true
 

.github/workflows/pr-containers.yml

@@ -13,7 +13,7 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       name: Checkout
 
     - name: Set up QEMU

.github/workflows/pr-goreleaser.yml

@@ -14,7 +14,7 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       name: Checkout
 
     - name: Verify .goreleaser.yml and try a dryrun release.

.github/workflows/pr-linter-check.yml

@@ -1,19 +1,32 @@
 name: Pull Request Linter Check
-on: [pull_request]
+on:
+  pull_request:
+    # Do not run when the change only includes these directories.
+    paths-ignore:
+      - "site/**"
+      - "design/**"
+      - "**/*.md"
 jobs:
+  get-go-version:
+    uses: ./.github/workflows/get-go-version.yaml
+    with:
+      ref: ${{ github.event.pull_request.base.ref }}
 
   build:
     name: Run Linter Check
     runs-on: ubuntu-latest
+    needs: get-go-version
     steps:
       - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/checkout@v5
+
+      - name: Set up Go version
+        uses: actions/setup-go@v6
         with:
-          go-version-file: 'go.mod'
+          go-version: ${{ needs.get-go-version.outputs.version }}
+
       - name: Linter check
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v8
         with:
-          version: v1.57.2
+          version: v2.1.1
           args: --verbose

.github/workflows/push-builder.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         # The default value is "1" which fetches only a single commit. If we merge PR without squash or rebase,
         # there are at least two commits: the first one is the merge commit and the second one is the real commit

.github/workflows/push.yml

@@ -9,26 +9,24 @@ on:
       - '*'
 
 jobs:
+  get-go-version:
+    uses: ./.github/workflows/get-go-version.yaml
+    with:
+      ref: ${{ github.ref }}
 
   build:
     name: Build
     runs-on: ubuntu-latest
+    needs: get-go-version
     steps:
       - name: Check out the code
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/checkout@v5
+
+      - name: Set up Go version
+        uses: actions/setup-go@v6
         with:
-          go-version-file: 'go.mod'
-      - id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GCS_SA_KEY }}'
-      - name: 'set up GCloud SDK'
-        uses: google-github-actions/setup-gcloud@v2
-      - name: 'use gcloud CLI'
-        run: |
-          gcloud info
+          go-version: ${{ needs.get-go-version.outputs.version }}
+
       - name: Set up QEMU
         id: qemu
         uses: docker/setup-qemu-action@v3
@@ -47,17 +45,11 @@ jobs:
       - name: Test
         run: make test
       - name: Upload test coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.out
           verbose: true
-      # Use the JSON key in secret to login gcr.io
-      - uses: 'docker/login-action@v3'
-        with:
-          registry: 'gcr.io' # or REGION.docker.pkg.dev
-          username: '_json_key'
-          password: '${{ secrets.GCR_SA_KEY }}'
       # Only try to publish the container image from the root repo; forks don't have permission to do so and will always get failures.
       - name: Publish container image
         if: github.repository == 'vmware-tanzu/velero'
@@ -68,24 +60,4 @@ jobs:
               
           # Build and push Velero image to docker registry
           docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }}
-          VERSION=$(./hack/docker-push.sh | grep 'VERSION:' | awk -F: '{print $2}' | xargs)
-
-          # Upload Velero image package to GCS
-          source hack/ci/build_util.sh
-          BIN=velero
-          RESTORE_HELPER_BIN=velero-restore-helper
-          GCS_BUCKET=velero-builds
-          VELERO_IMAGE=${BIN}-${VERSION}
-          VELERO_RESTORE_HELPER_IMAGE=${RESTORE_HELPER_BIN}-${VERSION}
-          VELERO_IMAGE_FILE=${VELERO_IMAGE}.tar.gz
-          VELERO_RESTORE_HELPER_IMAGE_FILE=${VELERO_RESTORE_HELPER_IMAGE}.tar.gz
-          VELERO_IMAGE_BACKUP_FILE=${VELERO_IMAGE}-'build.'${GITHUB_RUN_NUMBER}.tar.gz
-          VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE=${VELERO_RESTORE_HELPER_IMAGE}-'build.'${GITHUB_RUN_NUMBER}.tar.gz
-
-          cp ${VELERO_IMAGE_FILE} ${VELERO_IMAGE_BACKUP_FILE}
-          cp ${VELERO_RESTORE_HELPER_IMAGE_FILE} ${VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE}
-
-          uploader ${VELERO_IMAGE_FILE} ${GCS_BUCKET}
-          uploader ${VELERO_RESTORE_HELPER_IMAGE_FILE} ${GCS_BUCKET}
-          uploader ${VELERO_IMAGE_BACKUP_FILE} ${GCS_BUCKET}
-          uploader ${VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE} ${GCS_BUCKET}
+          ./hack/docker-push.sh

.github/workflows/rebase.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout the latest code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
       with:
         fetch-depth: 0
     - name: Automatic Rebase

.github/workflows/stale-issues.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v10.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands."

.gitignore

@@ -53,4 +53,13 @@ tilt-resources/cloud
 # test generated files
 test/e2e/report.xml
 coverage.out
-__debug_bin*
+__debug_bin*
+debug.test*
+
+# make lint cache
+.cache/
+
+# Go telemetry directory created when container sets HOME to working directory
+# This happens because Makefile uses 'docker run -w /github.com/vmware-tanzu/velero'
+# and Go's os.UserConfigDir() falls back to $HOME/.config when XDG_CONFIG_HOME is unset
+.config/

.golangci.yaml

@@ -6,17 +6,12 @@ run:
   # default concurrency is a available CPU number
   concurrency: 4
 
-  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  # timeout for analysis, e.g. 30s, 5m, default is 0
   timeout: 20m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
 
-
-  # default is true. Enables skipping of directories:
-  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs-use-default: true
-
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -32,362 +27,403 @@ run:
   # If false (default) - golangci-lint acquires file lock on start.
   allow-parallel-runners: false
 
-
 # output configuration options
 output:
-  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
   formats:
-    - format: colored-line-number
+    text:
       path: stdout
 
-  # print lines of code with issue, default is true
-  print-issued-lines: true
+      # print lines of code with issue, default is true
+      print-issued-lines: true
 
-  # print linter name in the end of issue text, default is true
-  print-linter-name: true
+      # print linter name in the end of issue text, default is true
+      print-linter-name: true
 
-  # make issues output unique by line, default is true
-  uniq-by-line: true
-
-
-# all available settings of specific linters
-linters-settings:
-  dogsled:
-    # checks assignments with too many blank identifiers; default is 2
-    max-blank-identifiers: 2
-  dupl:
-    # tokens count to trigger issue, 150 by default
-    threshold: 100
-  errcheck:
-    # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
-    # default is false: such cases aren't reported by default.
-    check-type-assertions: false
-
-    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
-    # default is false: such cases aren't reported by default.
-    check-blank: false
-
-    # [deprecated] comma-separated list of pairs of the form pkg:regex
-    # the regex is used to ignore names within pkg. (default "fmt:.*").
-    # see https://github.com/kisielk/errcheck#the-deprecated-method for details
-    # ignore: fmt:.*,io/ioutil:^Read.*
-
-    # path to a file containing a list of functions to exclude from checking
-    # see https://github.com/kisielk/errcheck#excluding-functions for details
-    # exclude: /path/to/file.txt
-  exhaustive:
-    # indicates that switch statements are to be considered exhaustive if a
-    # 'default' case is present, even if all enum members aren't listed in the
-    # switch
-    default-signifies-exhaustive: false
-  funlen:
-    lines: 60
-    statements: 40
-  gocognit:
-    # minimal code complexity to report, 30 by default (but we recommend 10-20)
-    min-complexity: 10
-  nestif:
-    # minimal complexity of if statements to report, 5 by default
-    min-complexity: 4
-  goconst:
-    # minimal length of string constant, 3 by default
-    min-len: 3
-    # minimal occurrences count to trigger, 3 by default
-    min-occurrences: 5
-  gocritic:
-    # Which checks should be enabled; can't be combined with 'disabled-checks';
-    # See https://go-critic.github.io/overview#checks-overview
-    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
-    # By default list of stable checks is used.
-    # enabled-checks:
-    #  - rangeValCopy
-
-    # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
-    # disabled-checks:
-    #  - regexpMust
-
-    # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks.
-    # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
-    # enabled-tags:
-    #  - performance
-    # disabled-tags:
-    #  - experimental
-
-    settings: # settings passed to gocritic
-      captLocal: # must be valid enabled check name
-        paramsOnly: true
-    #  rangeValCopy:
-    #    sizeThreshold: 32
-  gocyclo:
-    # minimal code complexity to report, 30 by default (but we recommend 10-20)
-    min-complexity: 10
-  godot:
-    # check all top-level comments, not only declarations
-    check-all: false
-  godox:
-    # report any comments starting with keywords, this is useful for TODO or FIXME comments that
-    # might be left in the code accidentally and should be resolved before merging
-    keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
-      - NOTE
-      - OPTIMIZE # marks code that should be optimized before merging
-      - HACK # marks hack-arounds that should be removed before merging
-  gofmt:
-    # simplify code: gofmt with `-s` option, true by default
-    simplify: true
-  goimports:
-    # put imports beginning with prefix after 3rd-party packages;
-    # it's a comma-separated list of prefixes
-    local-prefixes: github.com/org/project
-  golint:
-    # minimal confidence for issues, default is 0.8
-    min-confidence: 0.8
-  gomnd:
-    # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description.
-    checks: argument,case,condition,operation,return,assign
-  gomodguard:
-    allowed:
-      modules:                                                        # List of allowed modules
-        # - gopkg.in/yaml.v2
-      domains:                                                        # List of allowed module domains
-        # - golang.org
-    blocked:
-      modules:                                                        # List of blocked modules
-        # - github.com/uudashr/go-module:                             # Blocked module
-        #     recommendations:                                        # Recommended modules that should be used instead (Optional)
-        #       - golang.org/x/mod
-        #     reason: "`mod` is the official go.mod parser library."  # Reason why the recommended module should be used (Optional)
-      versions:                                                       # List of blocked module version constraints
-        # - github.com/mitchellh/go-homedir:                          # Blocked module with version constraint
-        #     version: "< 1.1.0"                                      # Version constraint, see https://github.com/Masterminds/semver#basic-comparisons
-        #     reason: "testing if blocked version constraint works."  # Reason why the version constraint exists. (Optional)
-  govet:
-    # report about shadowed variables
-    # check-shadowing: true
-
-    # settings per analyzer
-    settings:
-      printf: # analyzer name, run `go tool vet help` to see all analyzers
-        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
-
-    # enable or disable analyzers by name
-    enable:
-      - atomicalign
-    enable-all: false
-    disable:
-      - shadow
-    disable-all: false
-  depguard:
-    list-type: blacklist # Velero.io word list : ignore
-    include-go-root: false
-    packages:
-      - github.com/sirupsen/logrus
-    packages-with-error-message:
-      # specify an error message to output when a denylisted package is used
-      - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log"
-  lll:
-    # max line length, lines longer will be reported. Default is 120.
-    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
-    line-length: 120
-    # tab width in spaces. Default to 1.
-    tab-width: 1
-  maligned:
-    # print struct with more effective memory layout or not, false by default
-    suggest-new: true
-  misspell:
-    # Correct spellings using locale preferences for US or UK.
-    # Default is to use a neutral variety of English.
-    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
-    locale: US
-    ignore-words:
-      - someword
-  nakedret:
-    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
-    max-func-lines: 30
-  prealloc:
-    # XXX: we don't recommend using this linter before doing performance profiling.
-    # For most programs usage of prealloc will be a premature optimization.
-
-    # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
-    # True by default.
-    simple: true
-    range-loops: true # Report preallocation suggestions on range loops, true by default
-    for-loops: false # Report preallocation suggestions on for loops, false by default
-  nolintlint:
-    # Enable to ensure that nolint directives are all used. Default is true.
-    allow-unused: false
-    # Disable to ensure that nolint directives don't have a leading space. Default is true.
-    allow-leading-space: true
-    # Exclude following linters from requiring an explanation.  Default is [].
-    allow-no-explanation: []
-    # Enable to require an explanation of nonzero length after each nolint directive. Default is false.
-    require-explanation: true
-    # Enable to require nolint directives to mention the specific linter being suppressed. Default is false.
-    require-specific: true
-  revive:
-    rules:
-      - name: unexported-return
-        disabled: true
-    
-  rowserrcheck:
-    packages:
-      - github.com/jmoiron/sqlx
-  testifylint:
-      # TODO: enable them all
-      disable:
-        - go-require
-        - float-compare
-        - require-error
-      enable-all: true
-  testpackage:
-    # regexp pattern to skip files
-    skip-regexp: (export|internal)_test\.go
-  unparam:
-    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
-    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
-    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
-    # with golangci-lint call it on a directory with the changed file.
-    check-exported: false
-  unused:
-    # treat code as a program (not a library) and report unused exported identifiers; default is false.
-    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
-    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
-    # with golangci-lint call it on a directory with the changed file.
-    check-exported: false
-  whitespace:
-    multi-if: false   # Enforces newlines (or comments) after every multi-line if statement
-    multi-func: false # Enforces newlines (or comments) after every multi-line function signature
-  wsl:
-    # If true append is only allowed to be cuddled if appending value is
-    # matching variables, fields or types on line above. Default is true.
-    strict-append: true
-    # Allow calls and assignments to be cuddled as long as the lines have any
-    # matching variables, fields or types. Default is true.
-    allow-assign-and-call: true
-    # Allow multiline assignments to be cuddled. Default is true.
-    allow-multiline-assign: true
-    # Allow declarations (var) to be cuddled.
-    allow-cuddle-declarations: false
-    # Allow trailing comments in ending of blocks
-    allow-trailing-comment: false
-    # Force newlines in end of case at this limit (0 = never).
-    force-case-trailing-whitespace: 0
-    # Force cuddling of err checks with err var assignment
-    force-err-cuddling: false
-    # Allow leading comments to be separated with empty liens
-    allow-separated-leading-comment: false
+  # Show statistics per linter.      
+  show-stats: false
 
 linters:
-  disable-all: true
+  # all available settings of specific linters
+  settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            # specify an error message to output when a denylisted package is used
+            - pkg: github.com/sirupsen/logrus
+              desc: "logging is allowed only by logutils.Log"
+
+    dogsled:
+      # checks assignments with too many blank identifiers; default is 2
+      max-blank-identifiers: 2
+
+    dupl:
+      # tokens count to trigger issue, 150 by default
+      threshold: 100
+
+    errcheck:
+      # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
+      # default is false: such cases aren't reported by default.
+      check-type-assertions: false
+
+      # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+      # default is false: such cases aren't reported by default.
+      check-blank: false
+
+
+    exhaustive:
+      # indicates that switch statements are to be considered exhaustive if a
+      # 'default' case is present, even if all enum members aren't listed in the
+      # switch
+      default-signifies-exhaustive: false
+
+    funlen:
+      lines: 60
+      statements: 40
+
+    gocognit:
+      # minimal code complexity to report, 30 by default (but we recommend 10-20)
+      min-complexity: 10
+
+    nestif:
+      # minimal complexity of if statements to report, 5 by default
+      min-complexity: 4
+
+    goconst:
+      # minimal length of string constant, 3 by default
+      min-len: 3
+      # minimal occurrences count to trigger, 3 by default
+      min-occurrences: 5
+
+    gocritic:
+      # Which checks should be enabled; can't be combined with 'disabled-checks';
+      # See https://go-critic.github.io/overview#checks-overview
+      # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+      # By default list of stable checks is used.
+      settings: # settings passed to gocritic
+        captLocal: # must be valid enabled check name
+          paramsOnly: true
+
+    gocyclo:
+      # minimal code complexity to report, 30 by default (but we recommend 10-20)
+      min-complexity: 10
+
+    godot:
+      # check all top-level comments, not only declarations
+      check-all: false
+
+    godox:
+      # report any comments starting with keywords, this is useful for TODO or FIXME comments that
+      # might be left in the code accidentally and should be resolved before merging
+      keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
+        - NOTE
+        - OPTIMIZE # marks code that should be optimized before merging
+        - HACK # marks hack-arounds that should be removed before merging
+
+    gosec:
+      excludes:
+        - G115
+
+    govet:
+      # enable or disable analyzers by name
+      enable:
+        - atomicalign
+      enable-all: false
+      disable:
+        - shadow
+      disable-all: false
+  
+    importas:
+       alias:
+        - alias: appsv1api
+          pkg: k8s.io/api/apps/v1
+        - alias: corev1api
+          pkg: k8s.io/api/core/v1
+        - alias: rbacv1
+          pkg: k8s.io/api/rbac/v1
+        - alias: apierrors
+          pkg: k8s.io/apimachinery/pkg/api/errors
+        - alias: apiextv1
+          pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
+        - alias: metav1
+          pkg: k8s.io/apimachinery/pkg/apis/meta/v1
+        - alias: storagev1api
+          pkg: k8s.io/api/storage/v1
+        - alias: batchv1api
+          pkg: k8s.io/api/batch/v1
+
+    lll:
+    # max line length, lines longer will be reported. Default is 120.
+    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+      line-length: 120
+      # tab width in spaces. Default to 1.
+      tab-width: 1
+
+    misspell:
+      # Correct spellings using locale preferences for US or UK.
+      # Default is to use a neutral variety of English.
+      # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+      locale: US
+      ignore-rules:
+        - someword
+
+    nakedret:
+      # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
+      max-func-lines: 30
+
+    prealloc:
+      # XXX: we don't recommend using this linter before doing performance profiling.
+      # For most programs usage of prealloc will be a premature optimization.
+
+      # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
+      # True by default.
+      simple: true
+      range-loops: true # Report preallocation suggestions on range loops, true by default
+      for-loops: false # Report preallocation suggestions on for loops, false by default
+
+    nolintlint:
+      # Enable to ensure that nolint directives are all used. Default is true.
+      allow-unused: false
+      # Exclude following linters from requiring an explanation.  Default is [].
+      allow-no-explanation: []
+      # Enable to require an explanation of nonzero length after each nolint directive. Default is false.
+      require-explanation: true
+      # Enable to require nolint directives to mention the specific linter being suppressed. Default is false.
+      require-specific: true
+
+    perfsprint:
+      strconcat: false
+      sprintf1: false
+      errorf: false
+      int-conversion: true
+
+    revive:
+      rules:
+        - name: blank-imports
+          disabled: true
+        - name: context-as-argument
+          disabled: true
+        - name: context-keys-type
+        - name: dot-imports
+          disabled: true
+        - name: early-return
+          disabled: true
+          arguments:
+            - "preserveScope"
+        - name: empty-block
+          disabled: true
+        - name: error-naming
+          disabled: true
+        - name: error-return
+          disabled: true
+        - name: error-strings
+          disabled: true
+        - name: errorf
+          disabled: true
+        - name: increment-decrement
+        - name: indent-error-flow
+          disabled: true
+        - name: range
+        - name: receiver-naming
+          disabled: true
+        - name: redefines-builtin-id
+          disabled: true
+        - name: superfluous-else
+          disabled: true
+          arguments:
+            - "preserveScope"
+        - name: time-naming
+        - name: unexported-return
+          disabled: true
+        - name: unnecessary-stmt
+        - name: unreachable-code
+        - name: unused-parameter
+          disabled: true
+        - name: use-any
+        - name: var-declaration
+        - name: var-naming
+          disabled: true
+
+    rowserrcheck:
+      packages:
+        - github.com/jmoiron/sqlx
+
+    staticcheck:
+      checks:
+        - all
+        - -QF1001 # FIXME
+        - -QF1003 # FIXME
+        - -QF1004 # FIXME
+        - -QF1007 # FIXME
+        - -QF1008 # FIXME
+        - -QF1009 # FIXME
+        - -QF1012 # FIXME
+
+    testifylint:
+      # TODO: enable them all
+      disable:
+        - float-compare
+        - go-require
+      enable-all: true
+
+    testpackage:
+      # regexp pattern to skip files
+      skip-regexp: (export|internal)_test\.go
+    unparam:
+      # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+      # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+      # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+      # with golangci-lint call it on a directory with the changed file.
+      check-exported: false
+
+    usetesting:
+      os-setenv: false
+
+    whitespace:
+      multi-if: false # Enforces newlines (or comments) after every multi-line if statement
+      multi-func: false # Enforces newlines (or comments) after every multi-line function signature
+
+    wsl:
+      # If true append is only allowed to be cuddled if appending value is
+      # matching variables, fields or types on line above. Default is true.
+      strict-append: true
+      # Allow calls and assignments to be cuddled as long as the lines have any
+      # matching variables, fields or types. Default is true.
+      allow-assign-and-call: true
+      # Allow multiline assignments to be cuddled. Default is true.
+      allow-multiline-assign: true
+      # Allow declarations (var) to be cuddled.
+      allow-cuddle-declarations: false
+      # Allow trailing comments in ending of blocks
+      allow-trailing-comment: false
+      # Force newlines in end of case at this limit (0 = never).
+      force-case-trailing-whitespace: 0
+      # Force cuddling of err checks with err var assignment
+      force-err-cuddling: false
+      # Allow leading comments to be separated with empty lines
+      allow-separated-leading-comment: false
+
+  default: none
   enable:
     - asasalint
     - asciicheck
     - bidichk
     - bodyclose
+    - copyloopvar
     - dogsled
-    - durationcheck
     - dupword
+    - durationcheck
     - errcheck
-    - exportloopref
     - errchkjson
+    - exptostd
+    - ginkgolinter
     - goconst
-    - gofmt
     - goheader
-    - goimports
     - goprintffuncname
     - gosec
-    - gosimple
     - govet
-    - ginkgolinter
     - importas
     - ineffassign
     - misspell
     - nakedret
-    - nosprintfhostport
     - nilerr
     - noctx
     - nolintlint
+    - nosprintfhostport
+    - perfsprint
     - revive
     - staticcheck
-    - stylecheck
     - testifylint
     - thelper
-    - typecheck
     - unconvert
     - unparam
     - unused
     - usestdlibvars
+    - usetesting
     - whitespace
-  fast: false
 
+  exclusions:
+    # which dirs to skip: issues from them won't be reported;
+    # can use regexp here: generated.*, regexp is applied on full path;
+    # default value is empty list, but default dirs are skipped independently
+    # from this option's value (see skip-dirs-use-default).
+    # "/" will be replaced by current OS file path separator to properly work
+    # on Windows.
+    paths:
+      - pkg/plugin/generated/*
+      - third_party
+
+    rules:
+      - linters:
+          - staticcheck
+        text: "DefaultVolumesToRestic" # No need to report deprecate for DefaultVolumesToRestic.
+      - path: ".*_test.go$"
+        linters:
+          - errcheck
+          - goconst
+          - gosec
+          - govet
+          - staticcheck
+          - unparam
+          - unused
+      - path: test/
+        linters:
+          - errcheck
+          - goconst
+          - gosec
+          - nilerr
+          - staticcheck
+          - unparam
+          - unused
+      - path: ".*data_upload_controller_test.go$"
+        linters:
+          - dupword
+        text: "type"
+      - path: ".*config_test.go$"
+        linters:
+          - dupword
+        text: "bucket"
+
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
 
 issues:
-  exclude-rules:
-    - linters:
-        - staticcheck
-      text: "DefaultVolumesToRestic" # No need to report deprecate for DefaultVolumesToRestic.
-    - path: ".*_test.go$"
-      linters:
-        - dupword
-        - errcheck
-        - goconst
-        - gosec
-        - govet
-        - staticcheck
-        - stylecheck
-        - unparam
-        - unused
-    - path: test/
-      linters:
-        - dupword
-        - errcheck
-        - goconst
-        - gosec
-        - nilerr
-        - staticcheck
-        - stylecheck
-        - unparam
-        - unused
-
-  # The list of ids of default excludes to include or disable. By default it's empty.
-  include:
-    - EXC0002 # disable excluding of issues about comments from golint
-
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
   max-issues-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
-  # Show only new issues created after git revision `REV`
-  # new-from-rev: origin/main
+  # make issues output unique by line, default is true
+  uniq-by-line: true
 
-  # which dirs to skip: issues from them won't be reported;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but default dirs are skipped independently
-  # from this option's value (see skip-dirs-use-default).
-  # "/" will be replaced by current OS file path separator to properly work
-  # on Windows.
-  exclude-dirs:
-    - pkg/plugin/generated/*
+# This file contains all available configuration options
+# with their default values.
+formatters:
+  enable:
+    - gofmt
+    - goimports
+
+  exclusions:
+    generated: lax
+    paths:
+      - pkg/plugin/generated/*
+      - third_party
+
+  settings:
+    gofmt:
+      # simplify code: gofmt with `-s` option, true by default
+      simplify: true
+    goimports:
+      local-prefixes:
+        - github.com/vmware-tanzu/velero
 
 severity:
-  # Default value is empty string.
-  # Set the default severity for issues. If severity rules are defined and the issues 
-  # do not match or no severity is provided to the rule this will be the default 
-  # severity applied. Severities should match the supported severity names of the 
-  # selected out format.
-  # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity
-  # -   Checkstyle: https://checkstyle.sourceforge.io/property_types.html#severity
-  # -       Github: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message
-  default-severity: error
-
-  # The default value is false. 
-  # If set to true severity-rules regular expressions become case sensitive.
-  case-sensitive: false
+  default: error
 
   # Default value is empty list.
   # When a list of severity rules are provided, severity information will be added to lint
@@ -396,5 +432,7 @@ severity:
   # Only affects out formats that support setting severity information.
   rules:
     - linters:
-      - dupl
+        - dupl
       severity: info
+
+version: "2"

.goreleaser.yml

@@ -26,18 +26,23 @@ builds:
       - arm
       - arm64
       - ppc64le
+      - s390x
     ignore:
       # don't build arm for darwin and arm/arm64 for windows
       - goos: darwin
         goarch: arm
       - goos: darwin
         goarch: ppc64le
+      - goos: darwin
+        goarch: s390x
       - goos: windows
         goarch: arm
       - goos: windows
         goarch: arm64
       - goos: windows
         goarch: ppc64le
+      - goos: windows
+        goarch: s390x
     ldflags:
       - -X "github.com/vmware-tanzu/velero/pkg/buildinfo.Version={{ .Tag }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.GitSHA={{ .FullCommit }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.GitTreeState={{ .Env.GIT_TREE_STATE }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.ImageRegistry={{ .Env.REGISTRY }}"
 archives:
@@ -60,4 +65,4 @@ git:
   # tags if there are more than one tag in the same commit.
   #
   # Default: `-version:refname`
-  tag_sort: -version:creatordate
+  tag_sort: -version:creatordate

Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Velero binary build section
-FROM --platform=$BUILDPLATFORM golang:1.22-bookworm AS velero-builder
+FROM --platform=$BUILDPLATFORM golang:1.24-bookworm AS velero-builder
 
 ARG GOPROXY
 ARG BIN
@@ -42,13 +42,16 @@ RUN mkdir -p /output/usr/bin && \
     export GOARM=$( echo "${GOARM}" | cut -c2-) && \
     go build -o /output/${BIN} \
     -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \
+    go build -o /output/velero-restore-helper \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-restore-helper && \
     go build -o /output/velero-helper \
     -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \
     go clean -modcache -cache
 
 # Restic binary build section
-FROM --platform=$BUILDPLATFORM golang:1.22-bookworm AS restic-builder
+FROM --platform=$BUILDPLATFORM golang:1.24-bookworm AS restic-builder
 
+ARG GOPROXY
 ARG BIN
 ARG TARGETOS
 ARG TARGETARCH

Dockerfile-Windows

@@ -0,0 +1,57 @@
+# Copyright the Velero contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG OS_VERSION=1809
+
+# Velero binary build section
+FROM --platform=$BUILDPLATFORM golang:1.24-bookworm AS velero-builder
+
+ARG GOPROXY
+ARG BIN
+ARG PKG
+ARG VERSION
+ARG REGISTRY
+ARG GIT_SHA
+ARG GIT_TREE_STATE
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+ENV CGO_ENABLED=0 \
+    GO111MODULE=on \
+    GOPROXY=${GOPROXY} \
+    GOOS=${TARGETOS} \
+    GOARCH=${TARGETARCH} \
+    GOARM=${TARGETVARIANT} \
+    LDFLAGS="-X ${PKG}/pkg/buildinfo.Version=${VERSION} -X ${PKG}/pkg/buildinfo.GitSHA=${GIT_SHA} -X ${PKG}/pkg/buildinfo.GitTreeState=${GIT_TREE_STATE} -X ${PKG}/pkg/buildinfo.ImageRegistry=${REGISTRY}"
+
+WORKDIR /go/src/github.com/vmware-tanzu/velero
+
+COPY . /go/src/github.com/vmware-tanzu/velero
+
+RUN mkdir -p /output/usr/bin && \
+    export GOARM=$( echo "${GOARM}" | cut -c2-) && \
+    go build -o /output/${BIN}.exe \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \
+    go build -o /output/velero-restore-helper.exe \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-restore-helper && \    
+    go build -o /output/velero-helper.exe \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \
+    go clean -modcache -cache
+
+# Velero image packing section
+FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION}
+COPY --from=velero-builder /output /
+
+USER ContainerUser

MAINTAINERS.md

@@ -10,10 +10,10 @@
 | Daniel Jiang        | [reasonerjt](https://github.com/reasonerjt)                   | [VMware](https://www.github.com/vmware/)         |
 | Wenkai Yin          | [ywk253100](https://github.com/ywk253100)                     | [VMware](https://www.github.com/vmware/)         |
 | Xun Jiang           | [blackpiglet](https://github.com/blackpiglet)                 | [VMware](https://www.github.com/vmware/)         |
-| Ming Qiu            | [qiuming-best](https://github.com/qiuming-best)               | [VMware](https://www.github.com/vmware/)         |
 | Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift)        |
 | Yonghui Li          | [Lyndon-Li](https://github.com/Lyndon-Li)                     | [VMware](https://www.github.com/vmware/)         |
 | Anshul Ahuja        | [anshulahuja98](https://github.com/anshulahuja98)             | [Microsoft Azure](https://www.github.com/azure/) |
+| Tiger Kaovilai      | [kaovilai](https://github.com/kaovilai)                       | [OpenShift](https://github.com/openshift)        |
 
 ## Emeritus Maintainers
 * Adnan Abdulhussein ([prydonius](https://github.com/prydonius))
@@ -26,7 +26,8 @@
 * Bridget McErlean ([zubron](https://github.com/zubron))
 * JenTing Hsiao ([jenting](https://github.com/jenting))
 * Dave Smith-Uchida ([dsu-igeek](https://github.com/dsu-igeek))
-  
+* Ming Qiu ([qiuming-best](https://github.com/qiuming-best))
+
 ## Velero Contributors & Stakeholders
 
 | Feature Area           |                                         Lead                                         |

Makefile

@@ -22,15 +22,26 @@ PKG := github.com/vmware-tanzu/velero
 
 # Where to push the docker image.
 REGISTRY ?= velero
-GCR_REGISTRY ?= gcr.io/velero-gcp
+# In order to push images to an insecure registry, follow the two steps:
+#   1. Set "INSECURE_REGISTRY=true" 
+#   2. Provide your own buildx builder instance by setting "BUILDX_INSTANCE=your-own-builder-instance"
+#      The builder can be created with the following command:
+#        cat << EOF > buildkitd.toml
+#        [registry."insecure-registry-ip:port"]
+#        http = true
+#        insecure = true
+#        EOF
+#        docker buildx create --name=velero-builder --driver=docker-container --bootstrap --use --config ./buildkitd.toml
+#      Refer to https://github.com/docker/buildx/issues/1370#issuecomment-1288516840 for more details
+INSECURE_REGISTRY ?= false
 
 # Image name
 IMAGE ?= $(REGISTRY)/$(BIN)
-GCR_IMAGE ?= $(GCR_REGISTRY)/$(BIN)
 
 # We allow the Dockerfile to be configurable to enable the use of custom Dockerfiles
 # that pull base images from different registries.
 VELERO_DOCKERFILE ?= Dockerfile
+VELERO_DOCKERFILE_WINDOWS ?= Dockerfile-Windows
 BUILDER_IMAGE_DOCKERFILE ?= hack/build-image/Dockerfile
 
 # Calculate the realpath of the build-image Dockerfile as we `cd` into the hack/build
@@ -54,7 +65,7 @@ endif
 BUILDER_IMAGE := $(REGISTRY)/build-image:$(BUILDER_IMAGE_TAG)
 BUILDER_IMAGE_CACHED := $(shell docker images -q ${BUILDER_IMAGE} 2>/dev/null )
 
-HUGO_IMAGE := hugo-builder
+HUGO_IMAGE := ghcr.io/gohugoio/hugo
 
 # Which architecture to build - see $(ALL_ARCH) for options.
 # if the 'local' rule is being run, detect the ARCH from 'go env'
@@ -68,10 +79,8 @@ TAG_LATEST ?= false
 
 ifeq ($(TAG_LATEST), true)
 	IMAGE_TAGS ?= $(IMAGE):$(VERSION) $(IMAGE):latest
-	GCR_IMAGE_TAGS ?= $(GCR_IMAGE):$(VERSION) $(GCR_IMAGE):latest
 else
 	IMAGE_TAGS ?= $(IMAGE):$(VERSION)
-	GCR_IMAGE_TAGS ?= $(GCR_IMAGE):$(VERSION)
 endif
 
 # check buildx is enabled only if docker is in path
@@ -94,13 +103,32 @@ define BUILDX_ERROR
 buildx not enabled, refusing to run this recipe
 see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-velero for more info
 endef
-
+# comma cannot be escaped and can only be used in Make function arguments by putting into variable
+comma=,
 # The version of restic binary to be downloaded
 RESTIC_VERSION ?= 0.15.0
 
-CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le
-BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH))
-BUILDX_OUTPUT_TYPE ?= docker
+CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le linux-s390x
+BUILD_OUTPUT_TYPE ?= docker
+BUILD_OS ?= linux
+BUILD_ARCH ?= amd64
+BUILD_WINDOWS_VERSION ?= ltsc2022
+
+ifeq ($(BUILD_OUTPUT_TYPE), docker)
+	ALL_OS = linux
+	ALL_ARCH.linux = $(word 2, $(subst -, ,$(shell go env GOOS)-$(shell go env GOARCH)))
+else
+	ALL_OS = $(subst $(comma), ,$(BUILD_OS))
+	ALL_ARCH.linux = $(subst $(comma), ,$(BUILD_ARCH))
+endif
+
+ALL_ARCH.windows = $(if $(filter windows,$(ALL_OS)),amd64,)
+ALL_OSVERSIONS.windows = $(if $(filter windows,$(ALL_OS)),$(BUILD_WINDOWS_VERSION),)
+ALL_OS_ARCH.linux =  $(foreach os, $(filter linux,$(ALL_OS)), $(foreach arch, ${ALL_ARCH.linux}, ${os}-$(arch)))
+ALL_OS_ARCH.windows = $(foreach os, $(filter windows,$(ALL_OS)), $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, ${os}-${osversion}-${arch})))
+ALL_OS_ARCH = $(ALL_OS_ARCH.linux)$(ALL_OS_ARCH.windows)
+
+ALL_IMAGE_TAGS = $(IMAGE_TAGS)
 
 # set git sha and tree state
 GIT_SHA = $(shell git rev-parse HEAD)
@@ -124,17 +152,14 @@ GOBIN=$$(pwd)/.go/bin
 # If you want to build all containers, see the 'all-containers' rule.
 all:
 	@$(MAKE) build
-	@$(MAKE) build BIN=velero-restore-helper
 
 build-%:
 	@$(MAKE) --no-print-directory ARCH=$* build
-	@$(MAKE) --no-print-directory ARCH=$* build BIN=velero-restore-helper
 
 all-build: $(addprefix build-, $(CLI_PLATFORMS))
 
 all-containers:
 	@$(MAKE) --no-print-directory container
-	@$(MAKE) --no-print-directory container BIN=velero-restore-helper
 
 local: build-dirs
 # Add DEBUG=1 to enable debug locally
@@ -196,11 +221,38 @@ container:
 ifneq ($(BUILDX_ENABLED), true)
 	$(error $(BUILDX_ERROR))
 endif
+
+ifeq ($(BUILDX_INSTANCE),)
+	@echo creating a buildx instance
+	-docker buildx rm velero-builder || true
+	@docker buildx create --use --name=velero-builder
+else
+	@echo using a specified buildx instance $(BUILDX_INSTANCE)
+	@docker buildx use $(BUILDX_INSTANCE)
+endif
+
+	@mkdir -p _output
+
+	@for osarch in $(ALL_OS_ARCH); do \
+		$(MAKE) container-$${osarch}; \
+	done
+
+ifeq ($(BUILD_OUTPUT_TYPE), registry)
+	@for tag in $(ALL_IMAGE_TAGS); do \
+		IMAGE_TAG=$${tag} $(MAKE) push-manifest; \
+	done
+endif
+
+container-linux-%:
+	@BUILDX_ARCH=$* $(MAKE) container-linux
+
+container-linux:
+	@echo "building container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)"
+
 	@docker buildx build --pull \
-	--output=type=$(BUILDX_OUTPUT_TYPE) \
-	--platform $(BUILDX_PLATFORMS) \
-	$(addprefix -t , $(IMAGE_TAGS)) \
-	$(addprefix -t , $(GCR_IMAGE_TAGS)) \
+	--output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-linux-$(BUILDX_ARCH).tar,)" \
+	--platform="linux/$(BUILDX_ARCH)" \
+	$(addprefix -t , $(addsuffix "-linux-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \
 	--build-arg=GOPROXY=$(GOPROXY) \
 	--build-arg=PKG=$(PKG) \
 	--build-arg=BIN=$(BIN) \
@@ -209,14 +261,54 @@ endif
 	--build-arg=GIT_TREE_STATE=$(GIT_TREE_STATE) \
 	--build-arg=REGISTRY=$(REGISTRY) \
 	--build-arg=RESTIC_VERSION=$(RESTIC_VERSION) \
+	--provenance=false \
+	--sbom=false \
 	-f $(VELERO_DOCKERFILE) .
-	@echo "container: $(IMAGE):$(VERSION)"
-ifeq ($(BUILDX_OUTPUT_TYPE)_$(REGISTRY), registry_velero)
-	docker pull $(IMAGE):$(VERSION)
-	rm -f $(BIN)-$(VERSION).tar
-	docker save $(IMAGE):$(VERSION) -o $(BIN)-$(VERSION).tar
-	gzip -f $(BIN)-$(VERSION).tar
-endif
+
+	@echo "built container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)"
+
+container-windows-%:
+	@BUILDX_OSVERSION=$(firstword $(subst -, ,$*)) BUILDX_ARCH=$(lastword $(subst -, ,$*)) $(MAKE) container-windows
+
+container-windows:
+	@echo "building container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)"
+
+	@docker buildx build --pull \
+	--output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH).tar,)" \
+	--platform="windows/$(BUILDX_ARCH)" \
+	$(addprefix -t , $(addsuffix "-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \
+	--build-arg=GOPROXY=$(GOPROXY) \
+	--build-arg=PKG=$(PKG) \
+	--build-arg=BIN=$(BIN) \
+	--build-arg=VERSION=$(VERSION) \
+	--build-arg=OS_VERSION=$(BUILDX_OSVERSION) \
+	--build-arg=GIT_SHA=$(GIT_SHA) \
+    --build-arg=GIT_TREE_STATE=$(GIT_TREE_STATE) \
+	--build-arg=REGISTRY=$(REGISTRY) \
+	--provenance=false \
+	--sbom=false \
+	-f $(VELERO_DOCKERFILE_WINDOWS) .
+
+	@echo "built container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)"
+
+push-manifest:
+	@echo "building manifest: $(IMAGE_TAG) for $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})"
+	@docker manifest create --amend --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
+
+	@set -x; \
+	for arch in $(ALL_ARCH.windows); do \
+		for osversion in $(ALL_OSVERSIONS.windows); do \
+			BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \
+			full_version=`docker manifest inspect --insecure=$(INSECURE_REGISTRY) $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \
+			docker manifest annotate --os windows --arch $${arch} --os-version $${full_version} $(IMAGE_TAG) $(IMAGE_TAG)-windows-$${osversion}-$${arch}; \
+		done; \
+	done
+
+	@echo "pushing manifest $(IMAGE_TAG)"
+	@docker manifest push --purge --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG)
+
+	@echo "pushed manifest $(IMAGE_TAG):"
+	@docker manifest inspect --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG)
 
 SKIP_TESTS ?=
 test: build-dirs
@@ -359,7 +451,7 @@ release:
 serve-docs: build-image-hugo
 	docker run \
 	--rm \
-	-v "$$(pwd)/site:/srv/hugo" \
+	-v "$$(pwd)/site:/project" \
 	-it -p 1313:1313 \
 	$(HUGO_IMAGE) \
 	server --bind=0.0.0.0 --enableGitInfo=false
@@ -387,7 +479,7 @@ go-generate:
 # make new-changelog CHANGELOG_BODY="Changes you have made"
 new-changelog: GH_LOGIN ?= $(shell gh pr view --json author --jq .author.login 2> /dev/null)
 new-changelog: GH_PR_NUMBER ?= $(shell gh pr view --json number --jq .number 2> /dev/null)
-new-changelog: CHANGELOG_BODY ?= "$(shell gh pr view --json title --jq .title)"
+new-changelog: CHANGELOG_BODY ?= '$(shell gh pr view --json title --jq .title)'
 new-changelog:
 	@if [ "$(GH_LOGIN)" = "" ]; then \
 		echo "branch does not have PR or cli not logged in, try 'gh auth login' or 'gh pr create'"; \
@@ -395,4 +487,4 @@ new-changelog:
 	fi
 	@mkdir -p ./changelogs/unreleased/ && \
 	echo $(CHANGELOG_BODY) > ./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN) && \
-	echo "\"$(CHANGELOG_BODY)\" added to ./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN)"
+	echo \"$(CHANGELOG_BODY)\" added to "./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN)"

README.md

@@ -42,6 +42,8 @@ The following is a list of the supported Kubernetes versions for each Velero ver
 
 | Velero version | Expected Kubernetes version compatibility | Tested on Kubernetes version        |
 |----------------|-------------------------------------------|-------------------------------------|
+| 1.17           | 1.18-latest                               | 1.31.7, 1.32.3, 1.33.1, and 1.34.0          |
+| 1.16           | 1.18-latest                               | 1.31.4, 1.32.3, and 1.33.0          |
 | 1.15           | 1.18-latest                               | 1.28.8, 1.29.8, 1.30.4 and 1.31.1   |
 | 1.14           | 1.18-latest                               | 1.27.9, 1.28.9, and 1.29.4          |
 | 1.13           | 1.18-latest                               | 1.26.5, 1.27.3, 1.27.8, and 1.28.3  |

SECURITY.md

@@ -12,13 +12,13 @@ The Velero project maintains the following [governance document](https://github.
 
 Security is of the highest importance and all security vulnerabilities or suspected security vulnerabilities should be reported to Velero privately, to minimize attacks against current users of Velero before they are fixed. Vulnerabilities will be investigated and patched on the next patch (or minor) release as soon as possible. This information could be kept entirely internal to the project.  
 
-If you know of a publicly disclosed security vulnerability for Velero, please **IMMEDIATELY** contact the VMware Security Team (security@vmware.com).
+If you know of a publicly disclosed security vulnerability for Velero, please **IMMEDIATELY** contact the Security Team (velero-security.pdl@broadcom.com).
 
  
 
 **IMPORTANT: Do not file public issues on GitHub for security vulnerabilities**
 
-To report a vulnerability or a security-related issue, please contact the VMware email address with the details of the vulnerability. The email will be fielded by the VMware Security Team and then shared with the Velero maintainers who have committer and release permissions. Emails will be addressed within 3 business days, including a detailed plan to investigate the issue and any potential workarounds to perform in the meantime. Do not report non-security-impacting bugs through this channel. Use [GitHub issues](https://github.com/vmware-tanzu/velero/issues/new/choose) instead.
+To report a vulnerability or a security-related issue, please contact the email address with the details of the vulnerability. The email will be fielded by the Security Team and then shared with the Velero maintainers who have committer and release permissions. Emails will be addressed within 3 business days, including a detailed plan to investigate the issue and any potential workarounds to perform in the meantime. Do not report non-security-impacting bugs through this channel. Use [GitHub issues](https://github.com/vmware-tanzu/velero/issues/new/choose) instead.
 
 
 ## Proposed Email Content
@@ -29,7 +29,7 @@ Provide a descriptive subject line and in the body of the email include the foll
 
 *   Basic identity information, such as your name and your affiliation or company.
 *   Detailed steps to reproduce the vulnerability  (POC scripts, screenshots, and logs are all helpful to us).
-*   Description of the effects of the vulnerability on Velero and the related hardware and software configurations, so that the VMware Security Team can reproduce it.
+*   Description of the effects of the vulnerability on Velero and the related hardware and software configurations, so that the Security Team can reproduce it.
 *   How the vulnerability affects Velero usage and an estimation of the attack surface, if there is one.
 *   List other projects or dependencies that were used in conjunction with Velero to produce the vulnerability.
 
@@ -49,7 +49,7 @@ Provide a descriptive subject line and in the body of the email include the foll
 
 ## Patch, Release, and Disclosure
 
-The VMware Security Team will respond to vulnerability reports as follows:
+The Security Team will respond to vulnerability reports as follows:
 
  
 
@@ -62,7 +62,7 @@ The VMware Security Team will respond to vulnerability reports as follows:
 5. The Security Team will also create a [CVSS](https://www.first.org/cvss/specification-document) using the [CVSS Calculator](https://www.first.org/cvss/calculator/3.0). The Security Team makes the final call on the calculated CVSS; it is better to move quickly than making the CVSS perfect. Issues may also be reported to [Mitre](https://cve.mitre.org/) using this [scoring calculator](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator). The CVE will initially be set to private.
 6. The Security Team will work on fixing the vulnerability and perform internal testing before preparing to roll out the fix.
 7. The Security Team will provide early disclosure of the vulnerability by emailing the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list. Distributors can initially plan for the vulnerability patch ahead of the fix, and later can test the fix and provide feedback to the Velero team. See the section **Early Disclosure to Velero Distributors List** for details about how to join this mailing list. 
-8. A public disclosure date is negotiated by the VMware SecurityTeam, the bug submitter, and the distributors list. We prefer to fully disclose the bug as soon as possible once a user mitigation or patch is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for distributor coordination. The timeframe for disclosure is from immediate (especially if it’s already publicly known) to a few weeks. For a critical vulnerability with a straightforward mitigation, we expect the report date for the public disclosure date to be on the order of 14 business days. The VMware Security Team holds the final say when setting a public disclosure date.
+8. A public disclosure date is negotiated by the SecurityTeam, the bug submitter, and the distributors list. We prefer to fully disclose the bug as soon as possible once a user mitigation or patch is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for distributor coordination. The timeframe for disclosure is from immediate (especially if it’s already publicly known) to a few weeks. For a critical vulnerability with a straightforward mitigation, we expect the report date for the public disclosure date to be on the order of 14 business days. The Security Team holds the final say when setting a public disclosure date.
 9. Once the fix is confirmed, the Security Team will patch the vulnerability in the next patch or minor release, and backport a patch release into all earlier supported releases. Upon release of the patched version of Velero, we will follow the **Public Disclosure Process**.
 
 
@@ -79,7 +79,7 @@ The Security Team will also publish any mitigating steps users can take until th
 
 
 
-*   Use security@vmware.com to report security concerns to the VMware Security Team, who uses the list to privately discuss security issues and fixes prior to disclosure.
+*   Use velero-security.pdl@broadcom.com to report security concerns to the Security Team, who uses the list to privately discuss security issues and fixes prior to disclosure.
 *   Join the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list for early private information and vulnerability disclosure. Early disclosure may include mitigating steps and additional information on security patch releases. See below for information on how Velero distributors or vendors can apply to join this list.
 
 
@@ -107,11 +107,11 @@ To be eligible to join the [Velero Distributors](https://groups.google.com/u/1/g
 
 ## Embargo Policy
 
-The information that members receive on the Velero Distributors mailing list must not be made public, shared, or even hinted at anywhere beyond those who need to know within your specific team, unless you receive explicit approval to do so from the VMware Security Team. This remains true until the public disclosure date/time agreed upon by the list. Members of the list and others cannot use the information for any reason other than to get the issue fixed for your respective distribution's users.
+The information that members receive on the Velero Distributors mailing list must not be made public, shared, or even hinted at anywhere beyond those who need to know within your specific team, unless you receive explicit approval to do so from the Security Team. This remains true until the public disclosure date/time agreed upon by the list. Members of the list and others cannot use the information for any reason other than to get the issue fixed for your respective distribution's users.
 
 Before you share any information from the list with members of your team who are required to fix the issue, these team members must agree to the same terms, and only be provided with information on a need-to-know basis.
 
-In the unfortunate event that you share information beyond what is permitted by this policy, you must urgently inform the VMware Security Team (security@vmware.com) of exactly what information was leaked and to whom. If you continue to leak information and break the policy outlined here, you will be permanently removed from the list.
+In the unfortunate event that you share information beyond what is permitted by this policy, you must urgently inform the Security Team (velero-security.pdl@broadcom.com) of exactly what information was leaked and to whom. If you continue to leak information and break the policy outlined here, you will be permanently removed from the list.
 
  
 
@@ -123,6 +123,6 @@ Send new membership requests to projectvelero-distributors@googlegroups.com. In
 
 ## Confidentiality, integrity and availability
 
-We consider vulnerabilities leading to the compromise of data confidentiality, elevation of privilege, or integrity to be our highest priority concerns. Availability, in particular in areas relating to DoS and resource exhaustion, is also a serious security concern. The VMware Security Team takes all vulnerabilities, potential vulnerabilities, and suspected vulnerabilities seriously and will investigate them in an urgent and expeditious manner.
+We consider vulnerabilities leading to the compromise of data confidentiality, elevation of privilege, or integrity to be our highest priority concerns. Availability, in particular in areas relating to DoS and resource exhaustion, is also a serious security concern. The Security Team takes all vulnerabilities, potential vulnerabilities, and suspected vulnerabilities seriously and will investigate them in an urgent and expeditious manner.
 
 Note that we do not currently consider the default settings for Velero to be secure-by-default. It is necessary for operators to explicitly configure settings, role based access control, and other resource related features in Velero to provide a hardened Velero environment. We will not act on any security disclosure that relates to a lack of safe defaults. Over time, we will work towards improved safe-by-default configuration, taking into account backwards compatibility.

Tiltfile

@@ -52,7 +52,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip(
 
 tilt_helper_dockerfile_header = """
 # Tilt image
-FROM golang:1.22 as tilt-helper
+FROM golang:1.24 as tilt-helper
 
 # Support live reloading with Tilt
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \

changelogs/CHANGELOG-1.16.md

@@ -0,0 +1,156 @@
+## v1.16
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.16.0
+
+### Container Image
+`velero/velero:v1.16.0`
+
+### Documentation
+https://velero.io/docs/v1.16/
+
+### Upgrading
+https://velero.io/docs/v1.16/upgrade-to-1.16/
+
+### Highlights
+#### Windows cluster support
+In v1.16, Velero supports to run in Windows clusters and backup/restore Windows workloads, either stateful or stateless:
+ * Hybrid build and all-in-one image: the build process is enhanced to build an all-in-one image for hybrid CPU architecture and hybrid platform. For more information, check the design https://github.com/vmware-tanzu/velero/blob/main/design/multiple-arch-build-with-windows.md
+ * Deployment in Windows clusters: Velero node-agent, data mover pods and maintenance jobs now support to run in both linux and Windows nodes
+ * Data mover backup/restore Windows workloads: Velero built-in data mover supports Windows workloads throughout its full cycle, i.e., discovery, backup, restore, pre/post hook, etc. It automatically identifies Windows workloads and schedules data mover pods to the right group of nodes
+
+Check the epic issue https://github.com/vmware-tanzu/velero/issues/8289 for more information.  
+
+#### Parallel Item Block backup
+v1.16 now supports to back up item blocks in parallel. Specifically, during backup, correlated resources are grouped in item blocks and Velero backup engine creates a thread pool to back up the item blocks in parallel. This significantly improves the backup throughput, especially when there are large scale of resources.  
+Pre/post hooks also belongs to item blocks, so will also run in parallel along with the item blocks.  
+Users are allowed to configure the parallelism through the `--item-block-worker-count` Velero server parameter. If not configured, the default parallelism is 1.  
+
+For more information, check issue https://github.com/vmware-tanzu/velero/issues/8334.  
+
+#### Data mover restore enhancement in scalability
+In previous releases, for each volume of WaitForFirstConsumer mode, data mover restore is only allowed to happen in the node that the volume is attached. This severely degrades the parallelism and the balance of node resource(CPU, memory, network bandwidth) consumption for data mover restore (https://github.com/vmware-tanzu/velero/issues/8044).  
+
+In v1.16, users are allowed to configure data mover restores running and spreading evenly across all nodes in the cluster. The configuration is done through a new flag `ignoreDelayBinding` in node-agent configuration (https://github.com/vmware-tanzu/velero/issues/8242).  
+
+#### Data mover enhancements in observability 
+In 1.16, some observability enhancements are added:
+ * Output various statuses of intermediate objects for failures of data mover backup/restore (https://github.com/vmware-tanzu/velero/issues/8267)
+ * Output the errors when Velero fails to delete intermediate objects during clean up (https://github.com/vmware-tanzu/velero/issues/8125)
+
+The outputs are in the same node-agent log and enabled automatically.  
+
+#### CSI snapshot backup/restore enhancement in usability
+In previous releases, a unnecessary VolumeSnapshotContent object is retained for each backup and synced to other clusters sharing the same backup storage location. And during restore, the retained VolumeSnapshotContent is also restored unnecessarily.  
+
+In 1.16, the retained VolumeSnapshotContent is removed from the backup, so no unnecessary CSI objects are synced or restored.  
+
+For more information, check issue https://github.com/vmware-tanzu/velero/issues/8725.  
+
+#### Backup Repository Maintenance enhancement in resiliency and observability
+In v1.16, some enhancements of backup repository maintenance are added to improve the observability and resiliency:
+ * A new backup repository maintenance history section, called `RecentMaintenance`, is added to the BackupRepository CR. Specifically, for each BackupRepository, including start/completion time, completion status and error message. (https://github.com/vmware-tanzu/velero/issues/7810)
+ * Running maintenance jobs are now recaptured after Velero server restarts. (https://github.com/vmware-tanzu/velero/issues/7753)
+ * The maintenance job will not be launched for readOnly BackupStorageLocation. (https://github.com/vmware-tanzu/velero/issues/8238)
+ * The backup repository will not try to initialize a new repository for readOnly BackupStorageLocation. (https://github.com/vmware-tanzu/velero/issues/8091)
+ * Users now are allowed to configure the intervals of an effective maintenance in the way of `normalGC`, `fastGC` and `eagerGC`, through the `fullMaintenanceInterval` parameter in backupRepository configuration. (https://github.com/vmware-tanzu/velero/issues/8364)
+
+#### Volume Policy enhancement of filtering volumes by PVC labels
+In v1.16, Volume Policy is extended to support filtering volumes by PVC labels. (https://github.com/vmware-tanzu/velero/issues/8256).   
+
+#### Resource Status restore per object
+In v1.16, users are allowed to define whether to restore resource status per object through an annotation `velero.io/restore-status` set on the object. (https://github.com/vmware-tanzu/velero/issues/8204).  
+
+#### Velero Restore Helper binary is merged into Velero image 
+In v1.16, Velero banaries, i.e., velero, velero-helper and velero-restore-helper, are all included into the single Velero image. (https://github.com/vmware-tanzu/velero/issues/8484).  
+
+### Runtime and dependencies
+Golang runtime: 1.23.7  
+kopia: 0.19.0
+
+### Limitations/Known issues
+#### Limitations of Windows support
+  * fs-backup is not supported for Windows workloads and so fs-backup runs only in linux nodes for linux workloads
+  * Backup/restore of NTFS extended attributes/advanced features are not supported, i.e., Security Descriptors, System/Hidden/ReadOnly attributes, Creation Time, NTFS Streams, etc.
+
+### All Changes
+  * Add third party annotation support for maintenance job, so that the declared third party annotations could be added to the maintenance job pods (#8812, @Lyndon-Li)
+  * Fix issue #8803, use deterministic name to create backupRepository (#8808, @Lyndon-Li)
+  * Refactor restoreItem and related functions to differentiate the backup resource name and the restore target resource name. (#8797, @blackpiglet)
+  * ensure that PV is removed before VS is deleted (#8777, @ix-rzi)
+  * host_pods should not be mandatory to node-agent (#8774, @mpryc)
+  * Log doesn't show pv name, but displays %!s(MISSING) instead (#8771, @hu-keyu)
+  * Fix issue #8754, add third party annotation support for data mover (#8770, @Lyndon-Li)
+  * Add docs for volume policy with labels as a criteria (#8759, @shubham-pampattiwar)
+  * Move pvc annotation removal from CSI RIA to regular PVC RIA (#8755, @sseago)
+  * Add doc for maintenance history (#8747, @Lyndon-Li)
+  * Fix issue #8733, add doc for restorePVC (#8737, @Lyndon-Li)
+  * Fix issue #8426, add doc for Windows support (#8736, @Lyndon-Li)
+  * Fix issue #8475, refactor build-from-source doc for hybrid image build (#8729, @Lyndon-Li)
+  * Return directly if no pod volme backup are tracked (#8728, @ywk253100)
+  * Fix issue #8706, for immediate volumes, there is no selected-node annotation on PVC, so deduce the attached node from VolumeAttachment CRs (#8715, @Lyndon-Li)
+  * Add labels as a criteria for volume policy (#8713, @shubham-pampattiwar)
+  * Copy SecurityContext from Containers[0] if present for PVR (#8712, @sseago)
+  * Support pushing images to an insecure registry (#8703, @ywk253100)
+  * Modify golangci configuration to make it work. (#8695, @blackpiglet)
+  * Run backup post hooks inside ItemBlock synchronously (#8694, @ywk253100)
+  * Add docs for object level status restore (#8693, @shubham-pampattiwar)
+  * Clean artifacts generated during CSI B/R. (#8684, @blackpiglet)
+  * Don't run maintenance on the ReadOnly BackupRepositories. (#8681, @blackpiglet)
+  * Fix #8657: WaitGroup panic issue (#8679, @ywk253100)
+  * Fixes issue #8214, validate `--from-schedule` flag in create backup command to prevent empty or whitespace-only values. (#8665, @aj-2000)
+  * Implement parallel ItemBlock processing via backup_controller goroutines (#8659, @sseago)
+  * Clean up leaked CSI snapshot for incomplete backup (#8637, @raesonerjt)
+  * Handle update conflict when restoring the status (#8630, @ywk253100)
+  * Fix issue #8419, support repo maintenance job to run on Windows nodes (#8626, @Lyndon-Li)
+  * Always create DataUpload configmap in restore namespace (#8621, @sseago)
+  * Fix issue #8091, avoid to create new repo when BSL is readonly (#8615, @Lyndon-Li)
+  * Fix issue #8242, distribute dd evenly across nodes (#8611, @Lyndon-Li)
+  * Fix issue #8497, update du/dd progress on completion (#8608, @Lyndon-Li)
+  * Fix issue #8418, add Windows toleration to data mover pods (#8606, @Lyndon-Li)
+  * Check the PVB status via podvolume Backupper rather than calling API server to avoid API server issue (#8603, @ywk253100)
+  * Fix issue #8067, add tmp folder (/tmp for linux, C:\Windows\Temp for Windows) as an alternative of udmrepo's config file location (#8602, @Lyndon-Li)
+  * Data mover restore for Windows (#8594, @Lyndon-Li)
+  * Skip patching the PV in finalization for failed operation (#8591, @reasonerjt)
+  * Fix issue #8579, set event burst to block event broadcaster from filtering events (#8590, @Lyndon-Li)
+  * Configurable Kopia Maintenance Interval. backup-repository-configmap adds an option for configurable`fullMaintenanceInterval` where fastGC (12 hours), and eagerGC (6 hours) allowing for faster removal of deleted velero backups from kopia repo. (#8581, @kaovilai)
+  * Fix issue #7753, recall repo maintenance history on Velero server restart (#8580, @Lyndon-Li)
+  * Clear validation errors when schedule is valid (#8575, @ywk253100)
+  * Merge restore helper image into Velero server image (#8574, @ywk253100)
+  * Don't include excluded items in ItemBlocks (#8572, @sseago)
+  * fs uploader and block uploader support Windows nodes (#8569, @Lyndon-Li)
+  * Fix issue #8418, support data mover backup for Windows nodes (#8555, @Lyndon-Li)
+  * Fix issue #8044, allow users to ignore delay binding the restorePVC of data mover when it is in WaitForFirstConsumer mode (#8550, @Lyndon-Li)
+  * Fix issue #8539, validate uploader types when o.CRDsOnly is set to false only since CRD installation doesn't rely on uploader types (#8538, @Lyndon-Li)
+  * Fix issue #7810, add maintenance history for backupRepository CRs (#8532, @Lyndon-Li)
+  * Make fs-backup work on linux nodes with the new Velero deployment and disable fs-backup if the source/target pod is running in non-linux node (#8424) (#8518, @Lyndon-Li)
+  * Fix issue: backup schedule pause/unpause doesn't work (#8512, @ywk253100)
+  * Fix backup post hook issue #8159 (caused by #7571): always execute backup post hooks after PVBs are handled (#8509, @ywk253100)
+  * Fix issue #8267, enhance the error message when expose fails (#8508, @Lyndon-Li)
+  * Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env (#8504, @Lyndon-Li)
+  * Design to add label selector as a criteria for volume policy (#8503, @shubham-pampattiwar)
+  * Related to issue #8485, move the acceptedByNode and acceptedTimestamp to Status of DU/DD CRD (#8498, @Lyndon-Li)
+  * Add SecurityContext to restore-helper (#8491, @reasonerjt)
+  * Fix issue #8433, add third party labels to data mover pods when the same labels exist in node-agent pods (#8487, @Lyndon-Li)
+  * Fix issue #8485, add an accepted time so as to count the prepare timeout (#8486, @Lyndon-Li)
+  * Fix issue #8125, log diagnostic info for data mover exposers when expose timeout (#8482, @Lyndon-Li)
+  * Fix issue #8415, implement multi-arch build and Windows build (#8476, @Lyndon-Li)
+  * Pin kopia to 0.18.2 (#8472, @Lyndon-Li)
+  * Add nil check for updating DataUpload VolumeInfo in finalizing phase (#8471, @blackpiglet)
+  * Allowing Object-Level Resource Status Restore (#8464, @shubham-pampattiwar)
+  * For issue #8429. Add the design for multi-arch build and windows build (#8459, @Lyndon-Li)
+  * Upgrade go.mod k8s.io/ go.mod to v0.31.3 and implemented proper logger configuration for both client-go and controller-runtime libraries. This change ensures that logging format and level settings are properly applied throughout the codebase. The update improves logging consistency and control across the Velero system. (#8450, @kaovilai)
+  * Add Design for Allowing Object-Level Resource Status Restore (#8403, @shubham-pampattiwar)
+  * Fix issue #8391, check ErrCancelled from suffix of data mover pod's termination message (#8396, @Lyndon-Li)
+  * Fix issue #8394, don't call closeDataPath in VGDP callbacks, otherwise, the VGDP cleanup will hang (#8395, @Lyndon-Li)
+  * Adding support in velero Resource Policies for filtering PVs based on additional VolumeAttributes properties under CSI PVs (#8383, @mayankagg9722)
+  * Add --item-block-worker-count flag to velero install and server (#8380, @sseago)
+  * Make BackedUpItems thread safe (#8366, @sseago)
+  * Include --annotations flag in backup and restore create commands (#8354, @alromeros)
+  * Use aggregated discovery API to discovery API groups and resources (#8353, @ywk253100)
+  * Copy "envFrom" from Velero server when creating maintenance jobs (#8343, @evhan)
+  * Set hinting region to use for GetBucketRegion() in pkg/repository/config/aws.go (#8297, @kaovilai)
+  * Bump up version of client-go and controller-runtime (#8275, @ywk253100)
+  * fix(pkg/repository/maintenance): don't panic when there's no container statuses (#8271, @mcluseau)
+  * Add Backup warning for inclusion of NS managed by ArgoCD (#8257, @shubham-pampattiwar)
+  * Added tracking for deleted namespace status check in restore flow. (#8233, @sangitaray2021)

changelogs/CHANGELOG-1.17.md

@@ -0,0 +1,143 @@
+## v1.17
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.17.0
+
+### Container Image
+`velero/velero:v1.17.0`
+
+### Documentation
+https://velero.io/docs/v1.17/
+
+### Upgrading
+https://velero.io/docs/v1.17/upgrade-to-1.17/
+
+### Highlights
+#### Modernized fs-backup
+In v1.17, Velero fs-backup is modernized to the micro-service architecture, which brings below benefits:  
+- Many features that were absent to fs-backup are now available, i.e., load concurrency control, cancel, resume on restart, etc.
+- fs-backup is more robust, the running backup/restore could survive from node-agent restart; and the resource allocation is in a more granular manner, the failure of one backup/restore won't impact others.  
+- The resource usage of node-agent is steady, especially, the node-agent pods won't request huge memory and hold it for a long time.  
+
+Check design https://github.com/vmware-tanzu/velero/blob/main/design/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md for more details.  
+
+#### fs-backup support Windows cluster
+In v1.17, Velero fs-backup supports to backup/restore Windows workloads. By leveraging the new micro-service architecture for fs-backup, data mover pods could run in Windows nodes and backup/restore Windows volumes. Together with CSI snapshot data movement for Windows which is delivered in 1.16, Velero now supports Windows workload backup/restore in full scenarios.  
+Check design https://github.com/vmware-tanzu/velero/blob/main/design/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md for more details.  
+
+#### Volume group snapshot support
+In v1.17, Velero supports [volume group snapshots](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/) which is a beta feature in Kubernetes upstream, for both CSI snapshot backup and CSI snapshot data movement. This allows a snapshot to be taken from multiple volumes at the same point-in-time to achieve write order consistency, which is helpful to achieve better data consistency when multiple volumes being backed up are correlated.  
+Check the document https://velero.io/docs/main/volume-group-snapshots/ for more details.  
+
+#### Priority class support
+In v1.17, [Kubernetes priority class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) is supported for all modules across Velero. Specifically, users are allowed to configure priority class to Velero server, node-agent, data mover pods, backup repository maintenance jobs separately.  
+Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/priority-class-name-support_design.md for more details.  
+
+#### Scalability and Resiliency improvements of data movers
+##### Reduce excessive number of data mover pods in Pending state
+In v1.17, Velero allows users to set a `PrepareQueueLength` in the node-agent configuration, data mover pods and volumes out of this number won't be created until data path quota is available, so that excessive number cluster resources won't  be taken unnecessarily, which is particularly helpful for large scale environments. This improvement applies to all kinds of data movements, including fs-backup and CSI snapshot data movement.  
+Check design https://github.com/vmware-tanzu/velero/blob/main/design/node-agent-load-soothing.md for more details.  
+
+##### Enhancement on node-agent restart handling for data movements
+In v1.17, data movements in all phases could survive from node-agent restart and resume themselves; when a data movement gets orphaned in special cases, e.g., cluster node absent, it could also be canceled appropriately after the restart. This improvement applies to all kinds of data movements, including fs-backup and CSI snapshot data movement.  
+Check issue https://github.com/vmware-tanzu/velero/issues/8534 for more details.  
+
+##### CSI snapshot data movement restore node-selection and node-selection by storage class
+In v1.17, CSI snapshot data movement restore acquires the same node-selection capability as backup, that is, users could specify which nodes can/cannot run data mover pods for both backup and restore now. And users are also allowed to configure the node-selection per storage class, which is particularly helpful to the environments where a storage class are not usable by all cluster nodes.  
+Check issue https://github.com/vmware-tanzu/velero/issues/8186 and https://github.com/vmware-tanzu/velero/issues/8223 for more details.  
+
+#### Include/exclude policy support for resource policy
+In v1.17, Velero resource policy supports `includeExcludePolicy` besides the existing `volumePolicy`. This allows users to set include/exclude filters for resources in a resource policy configmap, so that these filters are reusable among multiple backups.  
+Check the document https://velero.io/docs/main/resource-filtering/#creating-resource-policies:~:text=resources%3D%22*%22-,Resource%20policies,-Velero%20provides%20resource for more details.  
+
+### Runtime and dependencies
+Golang runtime: 1.24.6  
+kopia: 0.21.1  
+
+### Limitations/Known issues
+
+### Breaking changes
+#### Deprecation of Restic
+According to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md#deprecation-policy), backup of fs-backup under Restic path is removed in v1.17, so `--uploader-type=restic` is not a valid installation configuration anymore. This means you cannot create a backup under Restic path, but you can still restore from the previous backups under Restic path until v1.19.  
+
+#### Repository maintenance job configurations are removed from Velero server parameter
+Since the repository maintenance job configurations are moved to repository maintenance job configMap, in v1.17 below Velero sever parameters are removed:
+- --keep-latest-maintenance-jobs
+- --maintenance-job-cpu-request
+- --maintenance-job-mem-request
+- --maintenance-job-cpu-limit
+- --maintenance-job-mem-limit
+
+### All Changes
+  * Add ConfigMap parameters validation for install CLI and server start. (#9200, @blackpiglet)
+  * Add priorityclasses to high priority restore list (#9175, @kaovilai)
+  * Introduced context-based logger for backend implementations (Azure, GCS, S3, and Filesystem) (#9168, @priyansh17)
+  * Fix issue #9140, add os=windows:NoSchedule toleration for Windows pods (#9165, @Lyndon-Li)
+  * Remove the repository maintenance job parameters from velero server. (#9147, @blackpiglet)
+  * Add include/exclude policy to resources policy (#9145, @reasonerjt)
+  * Add ConfigMap support for keepLatestMaintenanceJobs with CLI parameter fallback (#9135, @shubham-pampattiwar)
+  * Fix the dd and du's node affinity issue. (#9130, @blackpiglet)
+  * Remove the WaitUntilVSCHandleIsReady from vs BIA. (#9124, @blackpiglet)
+  * Add comprehensive Volume Group Snapshots documentation with workflow diagrams and examples (#9123, @shubham-pampattiwar)
+  * Fix issue #9065, add doc for node-agent prepare queue length (#9118, @Lyndon-Li)
+  * Fix issue #9095, update restore doc for PVC selected-node (#9117, @Lyndon-Li)
+  * Update CSI Snapshot Data Movement doc for issue #8534, #8185 (#9113, @Lyndon-Li)
+  * Fix issue #8986, refactor fs-backup doc after VGDP Micro Service for fs-backup (#9112, @Lyndon-Li)
+  * Return error if timeout when checking server version (#9111, @ywk253100)
+  * Update "Default Volumes to Fs Backup" to "File System Backup (Default)" (#9105, @shubham-pampattiwar)
+  * Fix issue #9077, don't block backup deletion on list VS error (#9100, @Lyndon-Li)
+  * Bump up Kopia to v0.21.1 (#9098, @Lyndon-Li)
+  * Add imagePullSecrets inheritance for VGDP pod and maintenance job. (#9096, @blackpiglet)
+  * Avoid checking the VS and VSC status in the backup finalizing phase. (#9092, @blackpiglet)
+  * Fix issue #9053, Always remove selected-node annotation during PVC restore when no node mapping exists. Breaking change: Previously, the annotation was preserved if the node existed. (#9076, @Lyndon-Li)
+  * Enable parameterized kubelet mount path during node-agent installation (#9074, @longxiucai)
+  * Fix issue #8857, support third party tolerations for data mover pods (#9072, @Lyndon-Li)
+  * Fix issue #8813, remove restic from the valid uploader type (#9069, @Lyndon-Li)
+  * Fix issue #8185, allow users to disable pod volume host path mount for node-agent (#9068, @Lyndon-Li)
+  * Fix #8344, add the design for a mechanism to soothe creation of data mover pods for DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore (#9067, @Lyndon-Li)
+  * Fix #8344, add a mechanism to soothe creation of data mover pods for DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore (#9064, @Lyndon-Li)
+  * Add Gauge metric for BSL availability (#9059, @reasonerjt)
+  * Fix missing defaultVolumesToFsBackup flag output in Velero describe backup cmd (#9056, @shubham-pampattiwar)
+  * Allow for proper tracking of multiple hooks per container (#9048, @sseago)
+  * Make the backup repository controller doesn't invalidate the BSL on restart (#9046, @blackpiglet)
+  * Removed username/password credential handling from newConfigCredential as azidentity.UsernamePasswordCredentialOptions is reported as deprecated. (#9041, @priyansh17)
+  * Remove dependency with VolumeSnapshotClass in DataUpload. (#9040, @blackpiglet)
+  * Fix issue #8961, cancel PVB/PVR on Velero server restart (#9031, @Lyndon-Li)
+  * Fix issue #8962, resume PVB/PVR during node-agent restarts (#9030, @Lyndon-Li)
+  * Bump kopia v0.20.1 (#9027, @Lyndon-Li)
+  * Fix issue #8965, support PVB/PVR's cancel state in the backup/restore (#9026, @Lyndon-Li)
+  * Fix Issue 8816 When specifying LabelSelector on restore, related items such as PVC and VolumeSnapshot are not included (#9024, @amastbau)
+  * Fix issue #8963, add legacy PVR controller for Restic path (#9022, @Lyndon-Li)
+  * Fix issue #8964, add Windows support for VGDP MS for fs-backup (#9021, @Lyndon-Li)
+  * Accommodate VGS workflows in PVC CSI plugin (#9019, @shubham-pampattiwar)
+  * Fix issue #8958, add VGDP MS PVB controller (#9015, @Lyndon-Li)
+  * Fix issue #8959, add VGDP MS PVR controller (#9014, @Lyndon-Li)
+  * Fix issue #8988, add data path for VGDP ms PVR (#9005, @Lyndon-Li)
+  * Fix issue #8988, add data path for VGDP ms pvb (#8998, @Lyndon-Li)
+  * Skip VS and VSC not created by backup. (#8990, @blackpiglet)
+  * Make ResticIdentifier optional for kopia BackupRepositories (#8987, @kaovilai)
+  * Fix issue #8960, implement PodVolume exposer for PVB/PVR (#8985, @Lyndon-Li)
+  * fix: update mc command in minio-deployment example (#8982, @vishal-chdhry)
+  * Fix issue #8957, add design for VGDP MS for fs-backup (#8979, @Lyndon-Li)
+  * Add BSL status check for backup/restore operations. (#8976, @blackpiglet)
+  * Mark BackupRepository not ready when BSL changed (#8975, @ywk253100)
+  * Add support for [distributed snapshotting](https://github.com/kubernetes-csi/external-snapshotter/tree/4cedb3f45790ac593ebfa3324c490abedf739477?tab=readme-ov-file#distributed-snapshotting) (#8969, @flx5)
+  * Fix issue #8534, refactor dm controllers to tolerate cancel request in more cases, e.g., node restart, node drain (#8952, @Lyndon-Li)
+  * The backup and restore VGDP affinity enhancement implementation. (#8949, @blackpiglet)
+  * Remove CSI VS and VSC metadata from backup. (#8946, @blackpiglet)
+  * Extend PVCAction itemblock plugin to support grouping PVCs under VGS label key (#8944, @shubham-pampattiwar)
+  * Copy security context from origin pod (#8943, @farodin91)
+  * Add support for configuring VGS label key (#8938, @shubham-pampattiwar)
+  * Add VolumeSnapshotContent into the RIA and the mustHave resource list. (#8924, @blackpiglet)
+  * Mounted cloud credentials should not be world-readable (#8919, @sseago)
+  * Warn for not found error in patching managed fields (#8902, @sseago)
+  * Fix issue 8878, relief node os deduction error checks (#8891, @Lyndon-Li)
+  * Skip namespace in terminating state in backup resource collection. (#8890, @blackpiglet)
+  * Implement PriorityClass Support (#8883, @kaovilai)
+  * Fix Velero adding restore-wait init container when not needed. (#8880, @kaovilai)
+  * Pass the logger in kopia related operations. (#8875, @hu-keyu)
+  * Inherit the dnsPolicy and dnsConfig from the node agent pod. This is done so that the kopia task uses the same configuration. (#8845, @flx5)
+  * Add design for VolumeGroupSnapshot support (#8778, @shubham-pampattiwar)
+  * Inherit k8s default volumeSnapshotClass. (#8719, @hu-keyu)
+  * CLI automatically discovers and uses cacert from BSL for download requests (#8557, @kaovilai)
+  * This PR aims to add s390x support to Velero binary. (#7505, @pandurangkhandeparker)

changelogs/unreleased/9173-clementnuss

@@ -0,0 +1 @@
+feat: Permit specifying annotations for the BackupPVC

changelogs/unreleased/9226-sseago

@@ -0,0 +1 @@
+Get pod list once per namespace in pvc IBA

changelogs/unreleased/9233-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #9229, don't attach backupPVC to the source node

changelogs/unreleased/9244-priyansh17

@@ -0,0 +1 @@
+Update AzureAD Microsoft Authentication Library to v1.5.0

changelogs/unreleased/9248-0xLeo258

@@ -0,0 +1 @@
+Protect VolumeSnapshot field from race condition during multi-thread backup

changelogs/unreleased/9256-shubham-pampattiwar

@@ -0,0 +1 @@
+Fix repository maintenance jobs to inherit allowlisted tolerations from Velero deployment

changelogs/unreleased/9264-shubham-pampattiwar

@@ -0,0 +1 @@
+Fix schedule controller to prevent backup queue accumulation during extended blocking scenarios by properly handling empty backup phases

changelogs/unreleased/9281-0xLeo258

@@ -0,0 +1 @@
+Implement concurrency control for cache of native VolumeSnapshotter plugin.

changelogs/unreleased/9295-sseago

@@ -0,0 +1 @@
+Add option for privileged fs-backup pod

changelogs/unreleased/9302-blackpiglet

@@ -0,0 +1 @@
+VerifyJSONConfigs verify every elements in Data.

config/crd/v1/bases/velero.io_backuprepositories.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: backuprepositories.velero.io
 spec:
   group: velero.io
@@ -71,7 +71,7 @@ spec:
               resticIdentifier:
                 description: |-
                   ResticIdentifier is the full restic-compatible string for identifying
-                  this repository.
+                  this repository. This field is only used when RepositoryType is "restic".
                 type: string
               volumeNamespace:
                 description: |-
@@ -81,15 +81,14 @@ spec:
             required:
             - backupStorageLocation
             - maintenanceFrequency
-            - resticIdentifier
             - volumeNamespace
             type: object
           status:
             description: BackupRepositoryStatus is the current status of a BackupRepository.
             properties:
               lastMaintenanceTime:
-                description: LastMaintenanceTime is the last time maintenance was
-                  run.
+                description: LastMaintenanceTime is the last time repo maintenance
+                  succeeded.
                 format: date-time
                 nullable: true
                 type: string
@@ -104,6 +103,33 @@ spec:
                 - Ready
                 - NotReady
                 type: string
+              recentMaintenance:
+                description: RecentMaintenance is status of the recent repo maintenance.
+                items:
+                  properties:
+                    completeTimestamp:
+                      description: CompleteTimestamp is the completion time of the
+                        repo maintenance.
+                      format: date-time
+                      nullable: true
+                      type: string
+                    message:
+                      description: Message is a message about the current status of
+                        the repo maintenance.
+                      type: string
+                    result:
+                      description: Result is the result of the repo maintenance.
+                      enum:
+                      - Succeeded
+                      - Failed
+                      type: string
+                    startTimestamp:
+                      description: StartTimestamp is the start time of the repo maintenance.
+                      format: date-time
+                      nullable: true
+                      type: string
+                  type: object
+                type: array
             type: object
         type: object
     served: true

config/crd/v1/bases/velero.io_backups.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: backups.velero.io
 spec:
   group: velero.io
@@ -63,7 +63,6 @@ spec:
                   DefaultVolumesToRestic specifies whether restic should be used to take a
                   backup of all pod volumes by default.
 
-
                   Deprecated: this field is no longer used and will be removed entirely in future. Use DefaultVolumesToFsBackup instead.
                 nullable: true
                 type: boolean
@@ -176,11 +175,13 @@ spec:
                                     items:
                                       type: string
                                     type: array
+                                    x-kubernetes-list-type: atomic
                                 required:
                                 - key
                                 - operator
                                 type: object
                               type: array
+                              x-kubernetes-list-type: atomic
                             matchLabels:
                               additionalProperties:
                                 type: string
@@ -364,11 +365,13 @@ spec:
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
@@ -425,11 +428,13 @@ spec:
                             items:
                               type: string
                             type: array
+                            x-kubernetes-list-type: atomic
                         required:
                         - key
                         - operator
                         type: object
                       type: array
+                      x-kubernetes-list-type: atomic
                     matchLabels:
                       additionalProperties:
                         type: string
@@ -502,6 +507,10 @@ spec:
                       uploads to perform when using the uploader.
                     type: integer
                 type: object
+              volumeGroupSnapshotLabelKey:
+                description: VolumeGroupSnapshotLabelKey specifies the label key to
+                  group PVCs under a VGS.
+                type: string
               volumeSnapshotLocations:
                 description: VolumeSnapshotLocations is a list containing names of
                   VolumeSnapshotLocations associated with this backup.

config/crd/v1/bases/velero.io_backupstoragelocations.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: backupstoragelocations.velero.io
 spec:
   group: velero.io
@@ -86,10 +86,13 @@ spec:
                       valid secret key.
                     type: string
                   name:
+                    default: ""
                     description: |-
                       Name of the referent.
+                      This field is effectively required, but due to backwards compatibility is
+                      allowed to be empty. Instances of this type with an empty value here are
+                      almost certainly wrong.
                       More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                      TODO: Add other useful fields. apiVersion, kind, uid?
                     type: string
                   optional:
                     description: Specify whether the Secret or its key must be defined
@@ -141,7 +144,6 @@ spec:
                 description: |-
                   AccessMode is an unused field.
 
-
                   Deprecated: there is now an AccessMode field on the Spec and this field
                   will be removed entirely as of v2.0.
                 enum:
@@ -153,7 +155,6 @@ spec:
                   LastSyncedRevision is the value of the `metadata/revision` file in the backup
                   storage location the last time the BSL's contents were synced into the cluster.
 
-
                   Deprecated: this field is no longer updated or used for detecting changes to
                   the location's contents and will be removed entirely in v2.0.
                 type: string

config/crd/v1/bases/velero.io_deletebackuprequests.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: deletebackuprequests.velero.io
 spec:
   group: velero.io

config/crd/v1/bases/velero.io_downloadrequests.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: downloadrequests.velero.io
 spec:
   group: velero.io

config/crd/v1/bases/velero.io_podvolumebackups.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: podvolumebackups.velero.io
 spec:
   group: velero.io
@@ -15,38 +15,41 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - description: Pod Volume Backup status such as New/InProgress
+    - description: PodVolumeBackup status such as New/InProgress
       jsonPath: .status.phase
       name: Status
       type: string
-    - description: Time when this backup was started
+    - description: Time duration since this PodVolumeBackup was started
       jsonPath: .status.startTimestamp
-      name: Created
+      name: Started
       type: date
-    - description: Namespace of the pod containing the volume to be backed up
-      jsonPath: .spec.pod.namespace
-      name: Namespace
-      type: string
-    - description: Name of the pod containing the volume to be backed up
-      jsonPath: .spec.pod.name
-      name: Pod
-      type: string
-    - description: Name of the volume to be backed up
-      jsonPath: .spec.volume
-      name: Volume
-      type: string
-    - description: The type of the uploader to handle data transfer
-      jsonPath: .spec.uploaderType
-      name: Uploader Type
-      type: string
+    - description: Completed bytes
+      format: int64
+      jsonPath: .status.progress.bytesDone
+      name: Bytes Done
+      type: integer
+    - description: Total bytes
+      format: int64
+      jsonPath: .status.progress.totalBytes
+      name: Total Bytes
+      type: integer
     - description: Name of the Backup Storage Location where this backup should be
         stored
       jsonPath: .spec.backupStorageLocation
       name: Storage Location
       type: string
-    - jsonPath: .metadata.creationTimestamp
+    - description: Time duration since this PodVolumeBackup was created
+      jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: Name of the node where the PodVolumeBackup is processed
+      jsonPath: .status.node
+      name: Node
+      type: string
+    - description: The type of the uploader to handle data transfer
+      jsonPath: .spec.uploaderType
+      name: Uploader
+      type: string
     name: v1
     schema:
       openAPIV3Schema:
@@ -76,6 +79,11 @@ spec:
                   BackupStorageLocation is the name of the backup storage location
                   where the backup repository is stored.
                 type: string
+              cancel:
+                description: |-
+                  Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set
+                  when the PodVolumeBackup is in InProgress phase
+                type: boolean
               node:
                 description: Node is the name of the node that the Pod is running
                   on.
@@ -96,7 +104,6 @@ spec:
                       the event) or if no container name is specified "spec.containers[2]" (container with
                       index 2 in this pod). This syntax is chosen only to have some well-defined way of
                       referencing a part of an object.
-                      TODO: this design is not final and this field is subject to change in the future.
                     type: string
                   kind:
                     description: |-
@@ -166,6 +173,13 @@ spec:
           status:
             description: PodVolumeBackupStatus is the current status of a PodVolumeBackup.
             properties:
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the pod volume backup is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
               completionTimestamp:
                 description: |-
                   CompletionTimestamp records the time a backup was completed.
@@ -186,7 +200,11 @@ spec:
                 description: Phase is the current state of the PodVolumeBackup.
                 enum:
                 - New
+                - Accepted
+                - Prepared
                 - InProgress
+                - Canceling
+                - Canceled
                 - Completed
                 - Failed
                 type: string

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: podvolumerestores.velero.io
 spec:
   group: velero.io
@@ -15,39 +15,40 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - description: Namespace of the pod containing the volume to be restored
-      jsonPath: .spec.pod.namespace
-      name: Namespace
+    - description: PodVolumeRestore status such as New/InProgress
+      jsonPath: .status.phase
+      name: Status
       type: string
-    - description: Name of the pod containing the volume to be restored
-      jsonPath: .spec.pod.name
-      name: Pod
+    - description: Time duration since this PodVolumeRestore was started
+      jsonPath: .status.startTimestamp
+      name: Started
+      type: date
+    - description: Completed bytes
+      format: int64
+      jsonPath: .status.progress.bytesDone
+      name: Bytes Done
+      type: integer
+    - description: Total bytes
+      format: int64
+      jsonPath: .status.progress.totalBytes
+      name: Total Bytes
+      type: integer
+    - description: Name of the Backup Storage Location where the backup data is stored
+      jsonPath: .spec.backupStorageLocation
+      name: Storage Location
+      type: string
+    - description: Time duration since this PodVolumeRestore was created
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Name of the node where the PodVolumeRestore is processed
+      jsonPath: .status.node
+      name: Node
       type: string
     - description: The type of the uploader to handle data transfer
       jsonPath: .spec.uploaderType
       name: Uploader Type
       type: string
-    - description: Name of the volume to be restored
-      jsonPath: .spec.volume
-      name: Volume
-      type: string
-    - description: Pod Volume Restore status such as New/InProgress
-      jsonPath: .status.phase
-      name: Status
-      type: string
-    - description: Pod Volume Restore status such as New/InProgress
-      format: int64
-      jsonPath: .status.progress.totalBytes
-      name: TotalBytes
-      type: integer
-    - description: Pod Volume Restore status such as New/InProgress
-      format: int64
-      jsonPath: .status.progress.bytesDone
-      name: BytesDone
-      type: integer
-    - jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
     name: v1
     schema:
       openAPIV3Schema:
@@ -77,6 +78,11 @@ spec:
                   BackupStorageLocation is the name of the backup storage location
                   where the backup repository is stored.
                 type: string
+              cancel:
+                description: |-
+                  Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
+                  when the PodVolumeRestore is in InProgress phase
+                type: boolean
               pod:
                 description: Pod is a reference to the pod containing the volume to
                   be restored.
@@ -93,7 +99,6 @@ spec:
                       the event) or if no container name is specified "spec.containers[2]" (container with
                       index 2 in this pod). This syntax is chosen only to have some well-defined way of
                       referencing a part of an object.
-                      TODO: this design is not final and this field is subject to change in the future.
                     type: string
                   kind:
                     description: |-
@@ -163,6 +168,13 @@ spec:
           status:
             description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore.
             properties:
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the pod volume restore is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
               completionTimestamp:
                 description: |-
                   CompletionTimestamp records the time a restore was completed.
@@ -174,11 +186,19 @@ spec:
               message:
                 description: Message is a message about the pod volume restore's status.
                 type: string
+              node:
+                description: Node is name of the node where the pod volume restore
+                  is processed.
+                type: string
               phase:
                 description: Phase is the current state of the PodVolumeRestore.
                 enum:
                 - New
+                - Accepted
+                - Prepared
                 - InProgress
+                - Canceling
+                - Canceled
                 - Completed
                 - Failed
                 type: string

config/crd/v1/bases/velero.io_restores.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: restores.velero.io
 spec:
   group: velero.io
@@ -138,11 +138,13 @@ spec:
                                     items:
                                       type: string
                                     type: array
+                                    x-kubernetes-list-type: atomic
                                 required:
                                 - key
                                 - operator
                                 type: object
                               type: array
+                              x-kubernetes-list-type: atomic
                             matchLabels:
                               additionalProperties:
                                 type: string
@@ -291,11 +293,13 @@ spec:
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
@@ -354,11 +358,13 @@ spec:
                             items:
                               type: string
                             type: array
+                            x-kubernetes-list-type: atomic
                         required:
                         - key
                         - operator
                         type: object
                       type: array
+                      x-kubernetes-list-type: atomic
                     matchLabels:
                       additionalProperties:
                         type: string

config/crd/v1/bases/velero.io_schedules.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: schedules.velero.io
 spec:
   group: velero.io
@@ -102,7 +102,6 @@ spec:
                       DefaultVolumesToRestic specifies whether restic should be used to take a
                       backup of all pod volumes by default.
 
-
                       Deprecated: this field is no longer used and will be removed entirely in future. Use DefaultVolumesToFsBackup instead.
                     nullable: true
                     type: boolean
@@ -215,11 +214,13 @@ spec:
                                         items:
                                           type: string
                                         type: array
+                                        x-kubernetes-list-type: atomic
                                     required:
                                     - key
                                     - operator
                                     type: object
                                   type: array
+                                  x-kubernetes-list-type: atomic
                                 matchLabels:
                                   additionalProperties:
                                     type: string
@@ -405,11 +406,13 @@ spec:
                               items:
                                 type: string
                               type: array
+                              x-kubernetes-list-type: atomic
                           required:
                           - key
                           - operator
                           type: object
                         type: array
+                        x-kubernetes-list-type: atomic
                       matchLabels:
                         additionalProperties:
                           type: string
@@ -466,11 +469,13 @@ spec:
                                 items:
                                   type: string
                                 type: array
+                                x-kubernetes-list-type: atomic
                             required:
                             - key
                             - operator
                             type: object
                           type: array
+                          x-kubernetes-list-type: atomic
                         matchLabels:
                           additionalProperties:
                             type: string
@@ -544,6 +549,10 @@ spec:
                           uploads to perform when using the uploader.
                         type: integer
                     type: object
+                  volumeGroupSnapshotLabelKey:
+                    description: VolumeGroupSnapshotLabelKey specifies the label key
+                      to group PVCs under a VGS.
+                    type: string
                   volumeSnapshotLocations:
                     description: VolumeSnapshotLocations is a list containing names
                       of VolumeSnapshotLocations associated with this backup.

config/crd/v1/bases/velero.io_serverstatusrequests.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: serverstatusrequests.velero.io
 spec:
   group: velero.io

config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: volumesnapshotlocations.velero.io
 spec:
   group: velero.io
@@ -57,10 +57,13 @@ spec:
                       valid secret key.
                     type: string
                   name:
+                    default: ""
                     description: |-
                       Name of the referent.
+                      This field is effectively required, but due to backwards compatibility is
+                      allowed to be empty. Instances of this type with an empty value here are
+                      almost certainly wrong.
                       More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                      TODO: Add other useful fields. apiVersion, kind, uid?
                     type: string
                   optional:
                     description: Specify whether the Secret or its key must be defined

config/crd/v2alpha1/bases/velero.io_datadownloads.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: datadownloads.velero.io
 spec:
   group: velero.io
@@ -92,6 +92,13 @@ spec:
                   DataMover specifies the data mover to be used by the backup.
                   If DataMover is "" or "velero", the built-in data mover will be used.
                 type: string
+              nodeOS:
+                description: NodeOS is OS of the node where the DataDownload is processed.
+                enum:
+                - auto
+                - linux
+                - windows
+                type: string
               operationTimeout:
                 description: |-
                   OperationTimeout specifies the time used to wait internal operations,
@@ -136,6 +143,16 @@ spec:
           status:
             description: DataDownloadStatus is the current status of a DataDownload.
             properties:
+              acceptedByNode:
+                description: Node is name of the node where the DataUpload is prepared.
+                type: string
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the DataUpload is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
               completionTimestamp:
                 description: |-
                   CompletionTimestamp records the time a restore was completed.

config/crd/v2alpha1/bases/velero.io_datauploads.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: datauploads.velero.io
 spec:
   group: velero.io
@@ -87,6 +87,9 @@ spec:
                   of the CSI snapshot.
                 nullable: true
                 properties:
+                  driver:
+                    description: Driver is the driver used by the VolumeSnapshotContent
+                    type: string
                   snapshotClass:
                     description: SnapshotClass is the name of the snapshot class that
                       the volume snapshot is created with
@@ -143,6 +146,17 @@ spec:
           status:
             description: DataUploadStatus is the current status of a DataUpload.
             properties:
+              acceptedByNode:
+                description: AcceptedByNode is name of the node where the DataUpload
+                  is prepared.
+                type: string
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the DataUpload is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
               completionTimestamp:
                 description: |-
                   CompletionTimestamp records the time a backup was completed.
@@ -165,6 +179,13 @@ spec:
               node:
                 description: Node is name of the node where the DataUpload is processed.
                 type: string
+              nodeOS:
+                description: NodeOS is OS of the node where the DataUpload is processed.
+                enum:
+                - auto
+                - linux
+                - windows
+                type: string
               path:
                 description: Path is the full path of the snapshot volume being backed
                   up.

config/rbac/role.yaml

@@ -8,17 +8,7 @@ rules:
   - ""
   resources:
   - persistentvolumerclaims
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
   - persistentvolumes
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
   - pods
   verbs:
   - get
@@ -26,6 +16,18 @@ rules:
   - velero.io
   resources:
   - backuprepositories
+  - backups
+  - backupstoragelocations
+  - datadownloads
+  - datauploads
+  - deletebackuprequests
+  - downloadrequests
+  - podvolumebackups
+  - podvolumerestores
+  - restores
+  - schedules
+  - serverstatusrequests
+  - volumesnapshotlocations
   verbs:
   - create
   - delete
@@ -38,239 +40,18 @@ rules:
   - velero.io
   resources:
   - backuprepositories/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - backups
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - backups/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - backupstoragelocations
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - backupstoragelocations/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - datadownloads
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - datadownloads/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - datauploads
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - datauploads/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - deletebackuprequests
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - deletebackuprequests/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - downloadrequests
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - downloadrequests/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - podvolumebackups
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - podvolumebackups/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - podvolumerestores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - podvolumerestores/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - restores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - restores/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - schedules
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - schedules/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - velero.io
-  resources:
-  - serverstatusrequests
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - velero.io
-  resources:
   - serverstatusrequests/status
   verbs:
   - get
   - patch
   - update
-- apiGroups:
-  - velero.io
-  resources:
-  - volumesnapshotlocations
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch

design/Implemented/Extend-VolumePolicies-to-support-more-actions.md

@@ -276,7 +276,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group
 
 	if !boolptr.IsSetToFalse(v.snapshotVolumes) {
 		// If the backup.Spec.SnapshotVolumes is not set, or set to true, then should take the snapshot.
-		v.logger.Infof("performing snapshot action for pv %s as the snapshotVolumes is not set to false")
+		v.logger.Infof("performing snapshot action for pv %s as the snapshotVolumes is not set to false", pv.Name)
 		return true, nil
 	}
 

design/Implemented/backup-performance-improvements.md

@@ -191,25 +191,25 @@ type ItemBlockWorkerPool struct {
 }
 
 type ItemBlockInput struct {
-    itemBlock  ItemBlock
+    itemBlock  *BackupItemBlock
     returnChan chan ItemBlockReturn
 }
 
 type ItemBlockReturn struct {
-    itemBlock  ItemBlock
+    itemBlock *BackupItemBlock
     resources []schema.GroupResource
     err       error
 }
 
 func (*p ItemBlockWorkerPool) getInputChannel() chan ItemBlockInput
-func RunItemBlockWorkers(context context.Context, workers int)
-func processItemBlocksWorker(context context.Context, itemBlockChannel chan ItemBlockInput, logger logrus.FieldLogger, wg *sync.WaitGroup)
+func StartItemBlockWorkerPool(context context.Context, workers int, logger logrus.FieldLogger) ItemBlockWorkerPool
+func processItemBlockWorker(context context.Context, itemBlockChannel chan ItemBlockInput, logger logrus.FieldLogger, wg *sync.WaitGroup)
 ```
 
-The worker pool will be started by calling `RunItemBlockWorkers` in `backupReconciler.SetupWithManager`, passing in the worker count and reconciler context.
-`SetupWithManager` will also add the input channel to the `itemBackupper` so that it will be available during backup processing.
-The func `RunItemBlockWorkers` will create the `ItemBlockWorkerPool` with a shared buffered input channel (fixed buffer size) and start `workers` gororoutines which will each call `processItemBlocksWorker`.
-The `processItemBlocksWorker` func (run by the worker goroutines) will read from `itemBlockChannel`, call `BackupItemBlock` on the retrieved `ItemBlock`, and then send the return value to the retrieved `returnChan`, and then process the next block.
+The worker pool will be started by calling `StartItemBlockWorkerPool` in `NewBackupReconciler()`, passing in the worker count and reconciler context.
+`backupreconciler.prepareBackupRequest` will also add the input channel to the `backupRequest` so that it will be available during backup processing.
+The func `StartItemBlockWorkerPool` will create the `ItemBlockWorkerPool` with a shared buffered input channel (fixed buffer size) and start `workers` gororoutines which will each call `processItemBlockWorker`.
+The `processItemBlockWorker` func (run by the worker goroutines) will read from `itemBlockChannel`, call `BackupItemBlock` on the retrieved `ItemBlock`, and then send the return value to the retrieved `returnChan`, and then process the next block.
 
 #### Modify ItemBlock processing loop to send ItemBlocks to the worker pool rather than backing them up directly
 

design/Implemented/clean_artifacts_in_csi_flow.md

@@ -0,0 +1,374 @@
+# Design to clean the artifacts generated in the CSI backup and restore workflows
+
+## Terminology
+
+* VSC: VolumeSnapshotContent
+* VS: VolumeSnapshot
+
+## Abstract
+* The design aims to delete the unnecessary VSs and VSCs generated during CSI backup and restore process. 
+* The design stop creating related VSCs during backup syncing.
+
+## Background
+In the current CSI backup and restore workflows, please notice the CSI B/R workflows means only using the CSI snapshots in the B/R, not including the CSI snapshot data movement workflows, some generated artifacts are kept after the backup or the restore process completion.
+
+Some of them are kept due to design, for example, the VolumeSnapshotContents generated during the backup are kept to make sure the backup deletion can clean the snapshots in the storage providers.
+
+Some of them are kept by accident, for example, after restore, two VolumeSnapshotContents are generated for the same VolumeSnapshot. One is from the backup content, and one is dynamically generated from the restore's VolumeSnapshot.
+
+The design aims to clean the unnecessary artifacts, and make the CSI B/R workflow more concise and reliable.
+
+## Goals
+- Clean the redundant VSC generated during CSI backup and restore.
+- Remove the VSCs in the backup sync process.
+
+## Non Goals
+- There were some discussion about whether Velero backup should include VSs and VSCs not generated in during the backup. By far, the conclusion is not including them is a better option. Although that is a useful enhancement, that is not included this design.
+- Delete all the CSI-related metadata files in the BSL is not the aim of this design. 
+
+## Detailed Design
+### Backup
+During backup, the main change is the backup-generated VSCs should not kept anymore.
+
+The reasons is we don't need them to ensure the snapshots clean up during backup deletion. Please reference to the [Backup Deletion section](#backup-deletion) section for detail.
+
+As a result, we can simplify the VS deletion logic in the backup. Before, we need to not only delete the VS, but also recreate a static VSC pointing a non-exiting VS.
+
+The deletion code in VS BackupItemAction can be simplify to the following:
+
+``` go
+	if backup.Status.Phase == velerov1api.BackupPhaseFinalizing ||
+		backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed {
+		p.log.
+			WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)).
+			WithField("BackupPhase", backup.Status.Phase).Debugf("Cleaning VolumeSnapshots.")
+
+		if vsc == nil {
+			vsc = &snapshotv1api.VolumeSnapshotContent{}
+		}
+
+		csi.DeleteReadyVolumeSnapshot(*vs, *vsc, p.crClient, p.log)
+		return item, nil, "", nil, nil
+	}
+
+
+func DeleteReadyVolumeSnapshot(
+	vs snapshotv1api.VolumeSnapshot,
+	vsc snapshotv1api.VolumeSnapshotContent,
+	client crclient.Client,
+	logger logrus.FieldLogger,
+) {
+	logger.Infof("Deleting Volumesnapshot %s/%s", vs.Namespace, vs.Name)
+	if vs.Status == nil ||
+		vs.Status.BoundVolumeSnapshotContentName == nil ||
+		len(*vs.Status.BoundVolumeSnapshotContentName) <= 0 {
+		logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.",
+			vs.Namespace, vs.Name)
+		return
+	}
+
+	if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil {
+		// Patch the DeletionPolicy of the VolumeSnapshotContent to set it to Retain.
+		// This ensures that the volume snapshot in the storage provider is kept.
+		if err := SetVolumeSnapshotContentDeletionPolicy(
+			vsc.Name,
+			client,
+			snapshotv1api.VolumeSnapshotContentRetain,
+		); err != nil {
+			logger.Warnf("Failed to patch DeletionPolicy of volume snapshot %s/%s",
+				vs.Namespace, vs.Name)
+			return
+		}
+
+		if err := client.Delete(context.TODO(), &vsc); err != nil {
+			logger.Warnf("Failed to delete the VSC %s: %s", vsc.Name, err.Error())
+		}
+	}
+	if err := client.Delete(context.TODO(), &vs); err != nil {
+		logger.Warnf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err)
+	} else {
+		logger.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s",
+			vs.Namespace, vs.Name)
+	}
+}
+```
+
+### Restore
+
+#### Restore the VolumeSnapshotContent
+The current behavior of VSC restoration is that the VSC from the backup is restore, and the restored VS also triggers creating a new VSC dynamically.
+
+Two VSCs created for the same VS in one restore seems not right.
+
+Skip restore the VSC from the backup is not a viable alternative, because VSC may reference to a [snapshot create secret](https://kubernetes-csi.github.io/docs/secrets-and-credentials-volume-snapshot-class.html?highlight=snapshotter-secret-name#createdelete-volumesnapshot-secret).
+
+If the `SkipRestore` is set true in the restore action's result, the secret returned in the additional items is ignored too.
+
+As a result, restore the VSC from the backup, and setup the VSC and the VS's relation is a better choice.
+
+Another consideration is the VSC name should not be the same as the backed-up VSC's, because the older version Velero's restore and backup keep the VSC after completion.
+
+There's high possibility that the restore will fail due to the VSC already exists in the cluster.
+
+Multiple restores of the same backup will also meet the same problem.
+
+The proposed solution is using the restore's UID and the VS's name to generate sha256 hash value as the new VSC name. Both the VS and VSC RestoreItemAction can access those UIDs, and it will avoid the conflicts issues.
+
+The restored VS name also shares the same generated name.
+
+The VS-referenced VSC name and the VSC's snapshot handle name are in their status.
+
+Velero restore process purges the restore resources' metadata and status before running the RestoreItemActions.
+
+As a result, we cannot read these information in the VS and VSC RestoreItemActions.
+
+Fortunately, RestoreItemAction input parameters includes the `ItemFromBackup`. The status is intact in `ItemFromBackup`.
+
+``` go
+func (p *volumeSnapshotRestoreItemAction) Execute(
+	input *velero.RestoreItemActionExecuteInput,
+) (*velero.RestoreItemActionExecuteOutput, error) {
+	p.log.Info("Starting VolumeSnapshotRestoreItemAction")
+
+	if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) {
+		p.log.Infof("Restore %s/%s did not request for PVs to be restored.",
+			input.Restore.Namespace, input.Restore.Name)
+		return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil
+	}
+
+	var vs snapshotv1api.VolumeSnapshot
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(
+		input.Item.UnstructuredContent(), &vs); err != nil {
+		return &velero.RestoreItemActionExecuteOutput{},
+			errors.Wrapf(err, "failed to convert input.Item from unstructured")
+	}
+
+	var vsFromBackup snapshotv1api.VolumeSnapshot
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(
+		input.ItemFromBackup.UnstructuredContent(), &vsFromBackup); err != nil {
+		return &velero.RestoreItemActionExecuteOutput{},
+			errors.Wrapf(err, "failed to convert input.Item from unstructured")
+	}
+
+	// If cross-namespace restore is configured, change the namespace
+	// for VolumeSnapshot object to be restored
+	newNamespace, ok := input.Restore.Spec.NamespaceMapping[vs.GetNamespace()]
+	if !ok {
+		// Use original namespace
+		newNamespace = vs.Namespace
+	}
+
+	if csiutil.IsVolumeSnapshotExists(newNamespace, vs.Name, p.crClient) {
+		p.log.Debugf("VolumeSnapshot %s already exists in the cluster. Return without change.", vs.Namespace+"/"+vs.Name)
+		return &velero.RestoreItemActionExecuteOutput{UpdatedItem: input.Item}, nil
+	}
+
+	newVSCName := generateSha256FromRestoreAndVsUID(string(input.Restore.UID), string(vsFromBackup.UID))
+	// Reset Spec to convert the VolumeSnapshot from using
+	// the dynamic VolumeSnapshotContent to the static one.
+	resetVolumeSnapshotSpecForRestore(&vs, &newVSCName)
+
+	// Reset VolumeSnapshot annotation. By now, only change
+	// DeletionPolicy to Retain.
+	resetVolumeSnapshotAnnotation(&vs)
+
+	vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs)
+	if err != nil {
+		p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name)
+		return nil, errors.WithStack(err)
+	}
+
+	p.log.Infof(`Returning from VolumeSnapshotRestoreItemAction with 
+		no additionalItems`)
+
+	return &velero.RestoreItemActionExecuteOutput{
+		UpdatedItem:     &unstructured.Unstructured{Object: vsMap},
+		AdditionalItems: []velero.ResourceIdentifier{},
+	}, nil
+}
+
+// generateSha256FromRestoreAndVsUID Use the restore UID and the VS UID to generate the new VSC name.
+// By this way, VS and VSC RIA action can get the same VSC name.
+func generateSha256FromRestoreAndVsUID(restoreUID string, vsUID string) string {
+	sha256Bytes := sha256.Sum256([]byte(restoreUID + "/" + vsUID))
+	return "vsc-" + hex.EncodeToString(sha256Bytes[:])
+}
+```
+
+#### Restore the VolumeSnapshot
+``` go
+// Execute restores a VolumeSnapshotContent object without modification
+// returning the snapshot lister secret, if any, as additional items to restore.
+func (p *volumeSnapshotContentRestoreItemAction) Execute(
+	input *velero.RestoreItemActionExecuteInput,
+) (*velero.RestoreItemActionExecuteOutput, error) {
+	if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) {
+		p.log.Infof("Restore did not request for PVs to be restored %s/%s",
+			input.Restore.Namespace, input.Restore.Name)
+		return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil
+	}
+
+	p.log.Info("Starting VolumeSnapshotContentRestoreItemAction")
+
+	var vsc snapshotv1api.VolumeSnapshotContent
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(
+		input.Item.UnstructuredContent(), &vsc); err != nil {
+		return &velero.RestoreItemActionExecuteOutput{},
+			errors.Wrapf(err, "failed to convert input.Item from unstructured")
+	}
+
+	var vscFromBackup snapshotv1api.VolumeSnapshotContent
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(
+		input.ItemFromBackup.UnstructuredContent(), &vscFromBackup); err != nil {
+		return &velero.RestoreItemActionExecuteOutput{},
+			errors.Errorf(err.Error(), "failed to convert input.ItemFromBackup from unstructured")
+	}
+
+	// If cross-namespace restore is configured, change the namespace
+	// for VolumeSnapshot object to be restored
+	newNamespace, ok := input.Restore.Spec.NamespaceMapping[vsc.Spec.VolumeSnapshotRef.Namespace]
+	if ok {
+		// Update the referenced VS namespace to the mapping one.
+		vsc.Spec.VolumeSnapshotRef.Namespace = newNamespace
+	}
+
+	// Reset VSC name to align with VS.
+	vsc.Name = generateSha256FromRestoreAndVsUID(string(input.Restore.UID), string(vscFromBackup.Spec.VolumeSnapshotRef.UID))
+
+	// Reset the ResourceVersion and UID of referenced VolumeSnapshot.
+	vsc.Spec.VolumeSnapshotRef.ResourceVersion = ""
+	vsc.Spec.VolumeSnapshotRef.UID = ""
+
+	// Set the DeletionPolicy to Retain to avoid VS deletion will not trigger snapshot deletion
+	vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain
+
+	if vscFromBackup.Status != nil && vscFromBackup.Status.SnapshotHandle != nil {
+		vsc.Spec.Source.VolumeHandle = nil
+		vsc.Spec.Source.SnapshotHandle = vscFromBackup.Status.SnapshotHandle
+	} else {
+		p.log.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name)
+		return nil, errors.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name)
+	}
+
+	additionalItems := []velero.ResourceIdentifier{}
+	if csi.IsVolumeSnapshotContentHasDeleteSecret(&vsc) {
+		additionalItems = append(additionalItems,
+			velero.ResourceIdentifier{
+				GroupResource: schema.GroupResource{Group: "", Resource: "secrets"},
+				Name:          vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation],
+				Namespace:     vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation],
+			},
+		)
+	}
+
+	vscMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vsc)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	p.log.Infof("Returning from VolumeSnapshotContentRestoreItemAction with %d additionalItems",
+		len(additionalItems))
+	return &velero.RestoreItemActionExecuteOutput{
+		UpdatedItem:     &unstructured.Unstructured{Object: vscMap},
+		AdditionalItems: additionalItems,
+	}, nil
+}
+```
+
+
+### Backup Sync
+csi-volumesnapshotclasses.json, csi-volumesnapshotcontents.json, and csi-volumesnapshots.json are CSI-related metadata files in the BSL for each backup.
+
+csi-volumesnapshotcontents.json and csi-volumesnapshots.json are not needed anymore, but csi-volumesnapshotclasses.json is still needed.
+
+One concrete scenario is that a backup is created in cluster-A, then the backup is synced to cluster-B, and the backup is deleted in the cluster-B. In this case, we don't have a chance to create the VS and VSC needed VolumeSnapshotClass.
+
+The VSC deletion workflow proposed by this design needs to create the VSC first. If the VSC's referenced VolumeSnapshotClass doesn't exist in cluster, the creation of VSC will fail.
+
+As a result, the VolumeSnapshotClass should still be synced in the backup sync process.
+
+### Backup Deletion
+Two factors are worthy for consideration for the backup deletion change:
+* Because the VSCs generated by the backup are not synced anymore, and the VSCs generated during the backup will not be kept too. The backup deletion needs to generate a VSC, then deletes it to make sure the snapshots in the storage provider are clean too.
+* The VSs generated by the backup are already deleted in the backup process, we don't need a DeleteItemAction for the VS anymore. As a result, the `velero.io/csi-volumesnapshot-delete` plugin is unneeded.
+
+For the VSC DeleteItemAction, we need to generate a VSC. Because we only care about the snapshot deletion, we don't need to create a VS associated with the VSC.
+
+Create a static VSC, then point it to a pseudo VS, and reference to the snapshot handle should be enough.
+
+To avoid the created VSC conflict with older version Velero B/R generated ones, the VSC name is set to `vsc-uuid`.
+
+The following is an example of the implementation.
+``` go
+	uuid, err := uuid.NewRandom()
+	if err != nil {
+		p.log.WithError(err).Errorf("Fail to generate the UUID to create VSC %s", snapCont.Name)
+		return errors.Wrapf(err, "Fail to generate the UUID to create VSC %s", snapCont.Name)
+	}
+	snapCont.Name = "vsc-" + uuid.String()
+
+	snapCont.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete
+
+	snapCont.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{
+		SnapshotHandle: snapCont.Status.SnapshotHandle,
+	}
+
+	snapCont.Spec.VolumeSnapshotRef = corev1api.ObjectReference{
+		APIVersion: snapshotv1api.SchemeGroupVersion.String(),
+		Kind:       "VolumeSnapshot",
+		Namespace:  "ns-" + string(snapCont.UID),
+		Name:       "name-" + string(snapCont.UID),
+	}
+
+	snapCont.ResourceVersion = ""
+
+	if err := p.crClient.Create(context.TODO(), &snapCont); err != nil {
+		return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", snapCont.Name)
+	}
+
+	// Read resource timeout from backup annotation, if not set, use default value.
+	timeout, err := time.ParseDuration(
+		input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation])
+	if err != nil {
+		p.log.Warnf("fail to parse resource timeout annotation %s: %s",
+			input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error())
+		timeout = 10 * time.Minute
+	}
+	p.log.Debugf("resource timeout is set to %s", timeout.String())
+
+	interval := 5 * time.Second
+
+	// Wait until VSC created and ReadyToUse is true.
+	if err := wait.PollUntilContextTimeout(
+		context.Background(),
+		interval,
+		timeout,
+		true,
+		func(ctx context.Context) (bool, error) {
+			tmpVSC := new(snapshotv1api.VolumeSnapshotContent)
+			if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(&snapCont), tmpVSC); err != nil {
+				return false, errors.Wrapf(
+					err, "failed to get VolumeSnapshotContent %s", snapCont.Name,
+				)
+			}
+
+			if tmpVSC.Status != nil && boolptr.IsSetToTrue(tmpVSC.Status.ReadyToUse) {
+				return true, nil
+			}
+
+			return false, nil
+		},
+	); err != nil {
+		return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name)
+	}
+```
+
+## Security Considerations
+Security is not relevant to this design.
+
+## Compatibility
+In this design, no new information is added in backup and restore. As a result, this design doesn't have any compatibility issue.
+
+## Open Issues
+Please notice the CSI snapshot backup and restore mechanism not supporting all file-store-based volume, e.g. Azure Files, EFS or vSphere CNS File Volume. Only block-based volumes are supported.
+Refer to [this comment](https://github.com/vmware-tanzu/velero/issues/3151#issuecomment-2623507686) for more details.

design/Implemented/handle-backup-of-volumes-by-resources-filters.md

@@ -86,7 +86,7 @@ volumePolicies:
     # capacity condition matches the volumes whose capacity falls into the range
     capacity: "0,100Gi"
     csi:
-      driver: aws.ebs.csi.driver
+      driver: ebs.csi.aws.com
       fsType: ext4
     storageClass:
     - gp2
@@ -174,7 +174,7 @@ data:
   - conditions:
       capacity: "0,100Gi"
       csi:
-        driver: aws.ebs.csi.driver
+        driver: ebs.csi.aws.com
         fsType: ext4
       storageClass:
       - gp2

design/Implemented/include-exclude-in-resource-policy.md

@@ -0,0 +1,82 @@
+# Proposal to add include exclude policy to resource policy
+
+This enhancement will allow the user to set include and exclude filters for resources in a resource policy configmap, so that
+these filters are reusable and the user will not need to set them each time they create a backup.
+
+## Background
+As mentioned in issue [#8610](https://github.com/vmware-tanzu/velero/issues/8610).  When there's a long list of resources 
+to include or exclude in a backup, it can be cumbersome to set them each time a backup is created.  There's a requirement to
+set these filters in a separate data structure so that they can be reused in multiple backups.
+
+## High-Level Design
+We may extend the data structure of resource policy to add `includeExcludePolicy`, which include the include and exclude filters 
+in the BackupSpec.  When the user creates a backup which references the resource policy config `velero backup create --resource-policies-configmap <configmap-name>`,
+the filters in "includeExcludePolicy" will take effect to filter the resources when velero collects the resources to backup.
+
+## Detailed Design
+
+### Data Structure
+The map `includeExcludePolicy` contains four fields `includedClusterScopedResources`, `excludedClusterScopedResources`, 
+`includedNamespaceScopedResources`,`excludedNamespaceScopedResources`.  These filters work exactly as the filters defined BackupSpec with
+the same names.  An example of the policy looks like:
+```yaml
+#omitted other irrelevant fields like 'version', 'volumePolicies'
+includeExcludePolicy:
+  includedClusterScopedResources:
+    - "cr"
+    - "crd"
+    - "pv"
+  excludedClusterScopedResources:
+    - "volumegroupsnapshotclass"
+    - "ingressclass"
+  includedNamespaceScopedResources:
+    - "pod"
+    - "service"
+    - "deployment"
+    - "pvc"
+  excludedNamespaceScopedResources:
+    - "configmap"
+```
+These filters are in the form of scoped include/exclude filters, which by design will not work with the "old" resource filters.
+Therefore, when a Backup references a resource policy configmap which has `includeExcludePolicy`, and at the same time it has 
+the "old" resource filters, i.e. `includedResources`, `excludedResources`, `includeClusterResources` set in the BackupSpec, the
+Backup will fail with a validation error.
+
+### Priorities 
+A user may set the include/exclude filters in Backupspec and also in the resource policy configmap.  In this case, the filters 
+in both the Backupspec and the resource policy configmap will take effect.  When there's a conflict, the filters in the Backupspec 
+will take precedence.  For example, if resource X is in the list of `includedNamespaceScopedResources` filter in the Backupspec, but 
+it's also in the list of `excludedClusterScopedResources` in the resource policy configmap, then resource X will be included in the backup.
+In this way, users can set the filters in the resource policy configmap to cover most of their use cases, and then override them 
+in the Backupspec when needed.
+
+### Implementation
+In addition to the data structure change, we will need to implement the following changes:
+1. A new function `CombineWithPolicy` will be added to the struct `ScopeIncludesExcludes`, which will combine the include/exclude filters
+in the resource policy configmap with the include/exclude filters in the Backupspec:  
+```go
+func (ie *ScopeIncludesExcludes) CombineWithPolicy(policy resourcepolicies.IncludeExcludePolicy) {
+	mapFunc := scopeResourceMapFunc(ie.helper)
+	for _, item := range policy.ExcludedNamespaceScopedResources {
+		resolvedItem := mapFunc(item, true)
+		if resolvedItem == "" {
+			continue
+		}
+		if !ie.ShouldInclude(resolvedItem) && !ie.ShouldExclude(resolvedItem) {
+			// The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter
+			// when the struct does not include this item and this item is not yet in the excludes filter.
+			ie.namespaceScopedResourceFilter.excludes.Insert(resolvedItem)
+		}
+		
+	}
+.....
+```
+This function will be called in the `kubernetesBackupper.BackupWithResolvers` function, to make sure the combined `ScopeIncludesExcludes` 
+filter will be assigned to the `ResourceIncludesExcludes` filter of the Backup request.
+
+2. Extra validation code will be added to the function `prepareBackupRequest` of `BackupReconciler` to check if there are "old"
+Resource filters in the BackupSpec when the Backup references a resource policy configmap which has `includeExcludePolicy`.
+
+## Alternatives Considered
+We may put `includeExcludePolicy` in a separate configmap, but it will require adding extra field to BackupSpec to reference the configmap,
+which is not necessary.

design/Implemented/multiple-arch-build-with-windows.md

@@ -0,0 +1,122 @@
+# Multi-arch Build and Windows Build Support
+
+## Background
+
+At present, Velero images could be built for linux-amd64 and linux-arm64. We need to support other platforms, i.e., windows-amd64.  
+At present, for linux image build, we leverage Buildkit's `--platform` option to create the image manifest list in one build call. However, it is a limited way and doesn't fully support all multi-arch scenarios. Specifically, since the build is done in one call with the same parameters, it is impossbile to build images with different configurations (e.g., Windows build requires a different Dockerfile).   
+At present, Velero by default build images locally, or no image or manifest is pushed to registry. However, docker doesn't support multi-arch build locally. We need to clarify the behavior of local build.    
+
+## Goals
+- Refactor the `make container` process to fully support multi-arch build
+- Add Windows build to the existing build process
+- Clarify the behavior of local build with multi-arch build capabilities
+- Don't change the pattern of the final image tag to be used by users
+
+## Non-Goals
+- There may be some workarounds to make the multi-arch image/manifest fully available locally. These workarounds will not be adopted, so local build always build single-arch images
+
+## Local Build
+
+For local build, two values of `--output` parameter for `docker buildx build` are supported:
+- `docker`: a docker format image is built, but the image is only built for the platform (`<os>/<arch>`) as same as the building env. E.g., when building from linux-amd64 env, a single manifest of linux-amd64 is created regardless how the input parameters are configured.  
+- `tar`: one or more images are built as tarballs according to the input platform (`<os>/<arch>`) parameters. Specifically, one tarball is generated for each platform. The build process is the same with the `Build Separate Manifests` of `Push Build` as detailed below. Merely, the `--output` parameter diffs, as `type=tar;dest=<tarball generated path>`. The tarball is generated to the `_output` folder and named with the platform info, e.g., `_output/velero-main-linux-amd64.tar`.  
+
+## Push Build
+
+For push build, the `--output` parameter for `docker buildx build` is always `registry`. And build will go according to the input parameters and create multi-arch manifest lists.    
+
+### Step 1: Build Separate Manifests
+
+Instead of specifying multiple platforms (`<os>/<arch>`) to `--platform` option, we add multiple `container-%` targets in Makefile and each target builds one platform representively.  
+
+The goal here is to build multiple manifests through the multiple targets. However, `docker buildx build` by default creates a manifest list even though there is only one element in `--platform`. Therefore, two flags `--provenance=false` and `--sbom=false` will be set additionally to force `docker buildx build` to create manifests.  
+
+Each manifest has a unique tag, the OS type and arch is added to the tag, in the pattern `$(REGISTRY)/$(BIN):$(VERSION)-$(OS)-$(ARCH)`. For example, `velero/velero:main-linux-amd64`.  
+
+All the created manifests will be pushed to registry so that the all-in-one manifest list could be created.  
+
+### Step 2: Create All-In-One Manifest List
+
+The next step is to create a manifest list to include all the created manifests. This could be done by `docker manifest create` command, the tags created and pushed at Step 1 are passed to this command.  
+A tag is also created for the manifest list, in the pattern `$(REGISTRY)/$(BIN):$(VERSION)`. For example, `velero/velero:main`.  
+
+### Step 3: Push All-In-One Manifest List
+
+The created manifest will be pushed to registry by command `docker manifest push`.  
+
+## Input Parameters
+
+Below are the input parameters that are configurable to meet different build purposes during Dev and release cycle:
+- BUILD_OUTPUT_TYPE: the type of output for the build, i.e., `docker`, `tar`, `registry`, while `docker` and `tar` is for local build; `registry` means push build. Default value is `docker`  
+- BUILD_OS: which types of OS should be built for. Multiple values are accepted, e.g., `linux,windows`. Default value is `linux`  
+- BUILD_ARCH: which types of architecture should be built for. Multiple values are accepted, e.g., `amd64,arm64`. Default value is `amd64`  
+- BUILDX_INSTANCE: an existing buildx instance to be used by the build. Default value is <empty> which indicates the build to create a new buildx instance  
+
+## Windows Build
+
+Windows container images vary from Windows OS versions, e.g., `ltsc2022` for Windows server 2022 and `1809` for Windows server 2019. Images for different OS versions should be built separately.  
+Therefore, separate build targets are added for each OS version, like `container-windows-%`.  
+For the same reason, a new input parameter is added, `BUILD_WINDOWS_VERSION`. The default value is `ltsc2022`. Windows server 2022 is the only base image we will deliver officially, Windows server 2019 is not supported. In future, we may need to support Windows server 2025 base image.  
+For local build to tar, the Windows OS version is also added to the name of the tarball, e.g., `_output/velero-main-windows-ltsc2022-amd64.tar`.  
+
+At present, Windows container image only supports `amd64` as the architecture, so `BUILD_ARCH` is ignored for Windows.  
+
+The Windows manifests need to be annotated with os type, arch, and os version. This will be done through `docker manifest annotate` command.  
+
+## Use Malti-arch Images
+
+In order to use the images, the manifest list's tag should be provided to `velero install` command or helm, the individual manifests are covered by the manifest list. During launch time, the container engine will load the right image to the container according to the platform of the running node.  
+
+## Build Samples
+
+**Local build to docker**
+```
+make container
+```
+The built image could be listed by `docker image ls`.  
+
+**Local build for linux-amd64 and windows-amd64 to tar**
+```
+BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows make container
+```
+Under `_output` directory, below files are generated:  
+```
+velero-main-linux-amd64.tar
+velero-main-windows-ltsc2022-amd64.tar
+``` 
+
+**Local build for linux-amd64, linux-arm64 and windows-amd64 to tar**
+```
+BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
+```
+Under `_output` directory, below files are generated:  
+```
+velero-main-linux-amd64.tar
+velero-main-linux-arm64.tar
+velero-main-windows-ltsc2022-amd64.tar
+```
+
+**Push build for linux-amd64 and windows-amd64**  
+Prerequisite: login to registry, e.g., through `docker login`  
+```
+BUILD_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows make container
+```
+Nothing is available locally, in the registry 3 tags are available:
+```
+velero/velero:main
+velero/velero:main-windows-ltsc2022-amd64
+velero/velero:main-linux-amd64
+```
+
+**Push build for linux-amd64, linux-arm64 and windows-amd64**  
+Prerequisite: login to registry, e.g., through `docker login` 
+```
+BUILD_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
+```
+Nothing is available locally, in the registry 4 tags are available:
+```
+velero/velero:main
+velero/velero:main-windows-ltsc2022-amd64
+velero/velero:main-linux-amd64
+velero/velero:main-linux-arm64
+```

design/Implemented/node-agent-affinity.md

@@ -128,5 +128,5 @@ Once this problem happens, the backupPod stays in `Pending` phase, and the corre
 On the other hand, the backupPod is deleted after the prepare timeout, so there is no way to tell the cause is one of the above problems or others.  
 To help the troubleshooting, we can add some diagnostic mechanism to discover the status of the backupPod and node-agent in the same node before deleting it as a result of the prepare timeout.  
 
-[1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
+[1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
 [2]: volume-snapshot-data-movement/volume-snapshot-data-movement.md

design/Implemented/node-agent-load-soothing.md

@@ -0,0 +1,121 @@
+# Node-agent Load Soothing Design
+
+## Glossary & Abbreviation
+
+**Velero Generic Data Path (VGDP)**: VGDP is the collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository.  
+
+## Background
+
+As mentioned in [node-agent Concurrency design][2], [CSI Snapshot Data Movement design][3], [VGDP Micro Service design][4] and [VGDP Micro Service for fs-backup design][5], all data movement activities for CSI snapshot data movement backups/restores and fs-backup respect the `loadConcurrency` settings configured in the `node-agent-configmap`. Once the number of existing loads exceeds the corresponding `loadConcurrency` setting, the loads will be throttled and some loads will be held until VGDP quotas are available.  
+However, this throttling only happens after the data mover pod is started and gets to `running`. As a result, when there are large number of concurrent volume backups, there may be many data mover pods get created but the VGDP instances inside them are actually on hold because of the VGDP throttling.  
+This could cause below problems:
+- In some environments, there is a pod limit in each node of the cluster or a pod limit throughout the cluster, too many of the inactive data mover pods may block other pods from running
+- In some environments, the system disk for each node of the cluster is limited, while pods also occupy system disk space, etc., many of the inactive data mover pods also take unnecessary space from system disk and cause other critical pods evicted
+- For CSI snapshot data movement backup, before creation of the data mover pod, the volume snapshot has also created, this means excessive number of snapshots may also be created and live for longer time since the VGDP won't start until the quota is available. However, in some environments, large number of snapshots is not allowed or may cause degradation of the storage peroformance
+
+On the other hand, the VGDP throttling mentioned in [node-agent Concurrency design][2] is an accurate controlling mechanism, that is, exactly the required number of data mover pods are throttled.  
+
+Therefore, another mechanism is required to soothe the creation of the data mover pods and volume snapshots before the VGDP throttling. It doesn't need to accurately control these creations but should effectively reduce the excessive number of inactive data mover pods and volume snapshots.  
+It is not practical to make an accurate control as it is almost impossible to predict which group of nodes a data mover pod is scheduled to, under the consideration of many complex factors, i.e., selected node, affinity, node OS, etc.  
+
+
+## Goals
+
+- Allow users to configure the expected number of loads pending on waiting for VGDP load concurrency quota
+- Create a soothing mechanism to prevent new loads from starting if the number of existing loads excceds the expected number
+
+## Non-Goals
+- Accurately controlling the loads from initiation is not a goal  
+
+## Solution
+
+We introduce a new field `prepareQueueLength` in `loadConcurrency` of `node-agent-configmap` as the allowed number of loads that are under preparing (expose). Specifically, loads are in this situation after its CR is in `Accepted` and `Prepared` phase. The `prepareQueueLength` should be a positive number, negative numbers will be ignored.  
+Once the value is set, the soothing mechanism takes effect, as the best effort, only the allowed number of CRs go into `Accepted` or `Prepared` phase, others will wait and stay as `New` state; and thereby only the allowed number of data mover pods, volume snapshots are created.  
+Otherwise, node-agent works the same as the legacy behavior, CRs go to `Accepted` or `Prepared` state as soon as the controllers process them and data mover pods and volume snapshots are also created without any constraints.  
+If users want to constrain the excessive number of pending data mover pods and volume snapshots, they could set a value by considering the VGDP load concurrency; otherwise, if they don't see constrains for pods or volume snapshots in their environment, they don't need to use this feature, in parallel preparing could also be beneficial for increasing the concurrency.  
+
+Node-agent server checks this configuration at startup time and use it to initiate the related VGDP modules. Therefore, users could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted.  
+
+The data structure is as below:
+```go
+type LoadConcurrency struct {
+    // GlobalConfig specifies the concurrency number to all nodes for which per-node config is not specified
+    GlobalConfig int `json:"globalConfig,omitempty"`
+
+    // PerNodeConfig specifies the concurrency number to nodes matched by rules
+    PerNodeConfig []RuledConfigs `json:"perNodeConfig,omitempty"`
+
+    // PrepareQueueLength specifies the max number of loads that are under expose
+	PrepareQueueLength int `json:"prepareQueueLength,omitempty"`    
+}
+```
+
+### Sample
+A sample of the ConfigMap is as below:
+```json
+{
+    "loadConcurrency": {
+        "globalConfig": 2,
+        "perNodeConfig": [
+            {
+                "nodeSelector": {
+                    "matchLabels": {
+                        "kubernetes.io/hostname": "node1"
+                    }
+                },
+                "number": 3
+            },
+            {
+                "nodeSelector": {
+                    "matchLabels": {
+                        "beta.kubernetes.io/instance-type": "Standard_B4ms"
+                    }
+                },
+                "number": 5
+            }
+        ],
+        "prepareQueueLength": 2
+    }
+}
+```
+To create the configMap, users need to save something like the above sample to a json file and then run below command:
+```
+kubectl create cm <ConfigMap name> -n velero --from-file=<json file name>
+```
+
+## Detailed Design
+Changes apply to the DataUpload Controller, DataDownload Controller, PodVolumeBackup Controller and PodVolumeRestore Controller, as below:
+1. The soothe happens to data mover CRs (DataUpload, DataDownload, PodVolumeBackup or PodVolumeRestore) that are in `New` state
+2. Before starting processing the CR, the corresponding controller counts the existing CRs under or pending for expose in the cluster, that is a total number of existing DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore that are in either `Accepted` or `Preparing` state  
+3. If the total number doesn't exceed the allowed number, the controller set the CR's phase to `Accepted`
+4. Once the total number exceeds the allowed number, the controller gives up processing the CR and have it requeued later. The delay for the requeue is 5 seconds
+
+The count happens for all the controllers in all nodes, to prevent the checks drain out the API server, the count happens to controller client caches for those CRs. And the count result is also cached, so that the count only happens whenever necessary. Below shows how it judges the necessity:
+- When one or more CRs' phase change to `Accepted`
+- When one or more CRs' phase change from `Accepted` to one of the terminal phases
+- When one or more CRs' phase change from `Prepared` to one of the terminal phases
+- When one or more CRs' phase change from `Prepared` to `InProgress`
+
+Ideally, 2~3 in the above steps need to be synchornized among controllers in all nodes. However, this synchronization is not implemented, the consideration is as below:    
+1. It is impossible to accurately synchronize the count among controllers in different nodes, because the client cache is not coherrent among nodes.  
+2. It is possible to synchronize the count among controllers in the same node. However, it is too expensive to make this synchronization, because 2~3 are part of the expose workflow, the synchronization impacts the performance and stability of the existing workflow. 
+3. Even without the synchronization, the soothing mechanism still works eventually -- when the controllers see all the discharged loads (expected ones and over-discharged ones), they will stop creating new loads until the quota is available again.  
+4. Step 2~3 that need to be synchronized could complete very quickly.    
+
+This is why we say this mechanism is not an accurate control. Or in another word, it is possible that more loads than the number of `prepareQueueLength` are discharged if controllers make the count and expose in the overlapped time (step 2~3).  
+For example, when multiple controllers of the same type (DataUpload, DataDownload, PodVolumeBackup or PodVolumeRestore) from different nodes make the count:  
+```
+max number of waiting loads = number defined by `prepareQueueLength` + number of nodes in cluster
+```
+As another example, when hybrid loads are running the count concurrently, e.g., mix of data mover backups, data mover restores, pod volume backups or pod volume restores, more loads may be discharged and the number depends on the number of concurrent hybrid loads.  
+In either case, because step 2~3 is short in time, it is less likely to reach the theoretically worset result.  
+
+
+
+
+
+[1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
+[2]: node-agent-concurrency.md
+[3]: volume-snapshot-data-movement/volume-snapshot-data-movement.md
+[4]: vgdp-micro-service/vgdp-micro-service.md
+[5]: vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md

design/Implemented/priority-class-name-support_design.md

@@ -0,0 +1,694 @@
+# PriorityClass Support Design Proposal
+
+## Abstract
+
+This design document outlines the implementation of priority class name support for Velero components, including the Velero server deployment, node agent daemonset, and maintenance jobs. This feature allows users to specify a priority class name for Velero components, which can be used to influence the scheduling and eviction behavior of these components.
+
+## Background
+
+Kubernetes allows users to define priority classes, which can be used to influence the scheduling and eviction behavior of pods. Priority classes are defined as cluster-wide resources, and pods can reference them by name. When a pod is created, the priority admission controller uses the priority class name to populate the priority value for the pod. The scheduler then uses this priority value to determine the order in which pods are scheduled.
+
+Currently, Velero does not provide a way for users to specify a priority class name for its components. This can be problematic in clusters where resource contention is high, as Velero components may be evicted or not scheduled in a timely manner, potentially impacting backup and restore operations.
+
+## Goals
+
+- Add support for specifying priority class names for Velero components
+- Update the Velero CLI to accept priority class name parameters for different components
+- Update the Velero deployment, node agent daemonset, maintenance jobs, and data mover pods to use the specified priority class names
+
+## Non Goals
+
+- Creating or managing priority classes
+- Automatically determining the appropriate priority class for Velero components
+
+## High-Level Design
+
+The implementation will add new fields to the Velero options struct to store the priority class names for the server deployment and node agent daemonset. The Velero CLI will be updated to accept new flags for these components. For data mover pods and maintenance jobs, priority class names will be configured through existing ConfigMap mechanisms (`node-agent-configmap` for data movers and `repo-maintenance-job-configmap` for maintenance jobs). The Velero deployment, node agent daemonset, maintenance jobs, and data mover pods will be updated to use their respective priority class names.
+
+## Detailed Design
+
+### CLI Changes
+
+New flags will be added to the `velero install` command to specify priority class names for different components:
+
+```go
+flags.StringVar(
+    &o.ServerPriorityClassName,
+    "server-priority-class-name",
+    o.ServerPriorityClassName,
+    "Priority class name for the Velero server deployment. Optional.",
+)
+
+flags.StringVar(
+    &o.NodeAgentPriorityClassName,
+    "node-agent-priority-class-name",
+    o.NodeAgentPriorityClassName,
+    "Priority class name for the node agent daemonset. Optional.",
+)
+```
+
+Note: Priority class names for data mover pods and maintenance jobs will be configured through their respective ConfigMaps (`--node-agent-configmap` for data movers and `--repo-maintenance-job-configmap` for maintenance jobs).
+
+### Velero Options Changes
+
+The `VeleroOptions` struct in `pkg/install/resources.go` will be updated to include new fields for priority class names:
+
+```go
+type VeleroOptions struct {
+    // ... existing fields ...
+    ServerPriorityClassName       string
+    NodeAgentPriorityClassName    string
+}
+```
+
+### Deployment Changes
+
+The `podTemplateConfig` struct in `pkg/install/deployment.go` will be updated to include a new field for the priority class name:
+
+```go
+type podTemplateConfig struct {
+    // ... existing fields ...
+    priorityClassName string
+}
+```
+
+A new function, `WithPriorityClassName`, will be added to set this field:
+
+```go
+func WithPriorityClassName(priorityClassName string) podTemplateOption {
+    return func(c *podTemplateConfig) {
+        c.priorityClassName = priorityClassName
+    }
+}
+```
+
+The `Deployment` function will be updated to use the priority class name:
+
+```go
+deployment := &appsv1api.Deployment{
+    // ... existing fields ...
+    Spec: appsv1api.DeploymentSpec{
+        // ... existing fields ...
+        Template: corev1api.PodTemplateSpec{
+            // ... existing fields ...
+            Spec: corev1api.PodSpec{
+                // ... existing fields ...
+                PriorityClassName: c.priorityClassName,
+            },
+        },
+    },
+}
+```
+
+### DaemonSet Changes
+
+The `DaemonSet` function will use the priority class name passed via the podTemplateConfig (from the CLI flag):
+
+```go
+daemonSet := &appsv1api.DaemonSet{
+    // ... existing fields ...
+    Spec: appsv1api.DaemonSetSpec{
+        // ... existing fields ...
+        Template: corev1api.PodTemplateSpec{
+            // ... existing fields ...
+            Spec: corev1api.PodSpec{
+                // ... existing fields ...
+                PriorityClassName: c.priorityClassName,
+            },
+        },
+    },
+}
+```
+
+### Maintenance Job Changes
+
+The `JobConfigs` struct in `pkg/repository/maintenance/maintenance.go` will be updated to include a field for the priority class name:
+
+```go
+type JobConfigs struct {
+    // LoadAffinities is the config for repository maintenance job load affinity.
+    LoadAffinities []*kube.LoadAffinity `json:"loadAffinity,omitempty"`
+
+    // PodResources is the config for the CPU and memory resources setting.
+    PodResources *kube.PodResources `json:"podResources,omitempty"`
+    
+    // PriorityClassName is the priority class name for the maintenance job pod
+    // Note: This is only read from the global configuration, not per-repository
+    PriorityClassName string `json:"priorityClassName,omitempty"`
+}
+```
+
+The `buildJob` function will be updated to use the priority class name from the global job configuration:
+
+```go
+func buildJob(cli client.Client, ctx context.Context, repo *velerov1api.BackupRepository, bslName string, config *JobConfigs,
+    podResources kube.PodResources, logLevel logrus.Level, logFormat *logging.FormatFlag) (*batchv1.Job, error) {
+    // ... existing code ...
+    
+    // Use the priority class name from the global job configuration if available
+    // Note: Priority class is only read from global config, not per-repository
+    priorityClassName := ""
+    if config != nil && config.PriorityClassName != "" {
+        priorityClassName = config.PriorityClassName
+    }
+    
+    // ... existing code ...
+    
+    job := &batchv1.Job{
+        // ... existing fields ...
+        Spec: batchv1.JobSpec{
+            // ... existing fields ...
+            Template: corev1api.PodTemplateSpec{
+                // ... existing fields ...
+                Spec: corev1api.PodSpec{
+                    // ... existing fields ...
+                    PriorityClassName: priorityClassName,
+                },
+            },
+        },
+    }
+    
+    // ... existing code ...
+}
+```
+
+Users will be able to configure the priority class name for all maintenance jobs by creating the repository maintenance job ConfigMap before installation. For example:
+
+```bash
+# Create the ConfigMap before running velero install
+cat <<EOF | kubectl create configmap repo-maintenance-job-config -n velero --from-file=config.json=/dev/stdin
+{
+    "global": {
+        "priorityClassName": "low-priority",
+        "podResources": {
+            "cpuRequest": "100m",
+            "memoryRequest": "128Mi"
+        }
+    }
+}
+EOF
+
+# Then install Velero referencing this ConfigMap
+velero install --provider aws \
+    --repo-maintenance-job-configmap repo-maintenance-job-config \
+    # ... other flags
+```
+
+The ConfigMap can be updated after installation to change the priority class for future maintenance jobs. Note that only the "global" configuration is used for priority class - all maintenance jobs will use the same priority class regardless of which repository they are maintaining.
+
+### Node Agent ConfigMap Changes
+
+We'll update the `Configs` struct in `pkg/nodeagent/node_agent.go` to include a field for the priority class name in the node-agent-configmap:
+
+```go
+type Configs struct {
+    // ... existing fields ...
+    
+    // PriorityClassName is the priority class name for the data mover pods 
+    // created by the node agent
+    PriorityClassName string `json:"priorityClassName,omitempty"`
+}
+```
+
+This will allow users to configure the priority class name for data mover pods through the node-agent-configmap. Note that the node agent daemonset itself gets its priority class from the `--node-agent-priority-class-name` CLI flag during installation, not from this configmap. For example:
+
+```bash
+# Create the ConfigMap before running velero install
+cat <<EOF | kubectl create configmap node-agent-config -n velero --from-file=config.json=/dev/stdin
+{
+    "priorityClassName": "low-priority",
+    "loadAffinity": [
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "node-role.kubernetes.io/worker": "true"
+                }
+            }
+        }
+    ]
+}
+EOF
+
+# Then install Velero referencing this ConfigMap
+velero install --provider aws \
+    --node-agent-configmap node-agent-config \
+    --use-node-agent \
+    # ... other flags
+```
+
+The `createBackupPod` function in `pkg/exposer/csi_snapshot.go` will be updated to accept and use the priority class name:
+
+```go
+func (e *csiSnapshotExposer) createBackupPod(
+    ctx context.Context,
+    ownerObject corev1api.ObjectReference,
+    backupPVC *corev1api.PersistentVolumeClaim,
+    operationTimeout time.Duration,
+    label map[string]string,
+    annotation map[string]string,
+    affinity *kube.LoadAffinity,
+    resources corev1api.ResourceRequirements,
+    backupPVCReadOnly bool,
+    spcNoRelabeling bool,
+    nodeOS string,
+    priorityClassName string, // New parameter
+) (*corev1api.Pod, error) {
+    // ... existing code ...
+    
+    pod := &corev1api.Pod{
+        // ... existing fields ...
+        Spec: corev1api.PodSpec{
+            // ... existing fields ...
+            PriorityClassName: priorityClassName,
+            // ... existing fields ...
+        },
+    }
+    
+    // ... existing code ...
+}
+```
+
+The call to `createBackupPod` in the `Expose` method will be updated to pass the priority class name retrieved from the node-agent-configmap:
+
+```go
+priorityClassName, _ := kube.GetDataMoverPriorityClassName(ctx, namespace, kubeClient, configMapName)
+backupPod, err := e.createBackupPod(
+    ctx,
+    ownerObject,
+    backupPVC,
+    csiExposeParam.OperationTimeout,
+    csiExposeParam.HostingPodLabels,
+    csiExposeParam.HostingPodAnnotations,
+    csiExposeParam.Affinity,
+    csiExposeParam.Resources,
+    backupPVCReadOnly,
+    spcNoRelabeling,
+    csiExposeParam.NodeOS,
+    priorityClassName, // Priority class name from node-agent-configmap
+)
+```
+
+A new function, `GetDataMoverPriorityClassName`, will be added to the `pkg/util/kube` package (in the same file as `ValidatePriorityClass`) to retrieve the priority class name for data mover pods:
+
+```go
+// In pkg/util/kube/priority_class.go
+
+// GetDataMoverPriorityClassName retrieves the priority class name for data mover pods from the node-agent-configmap
+func GetDataMoverPriorityClassName(ctx context.Context, namespace string, kubeClient kubernetes.Interface, configName string) (string, error) {
+    // configData is a minimal struct to parse only the priority class name from the ConfigMap
+    type configData struct {
+        PriorityClassName string `json:"priorityClassName,omitempty"`
+    }
+
+    // Get the ConfigMap
+    cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{})
+    if err != nil {
+        if apierrors.IsNotFound(err) {
+            // ConfigMap not found is not an error, just return empty string
+            return "", nil
+        }
+        return "", errors.Wrapf(err, "error getting node agent config map %s", configName)
+    }
+
+    if cm.Data == nil {
+        // No data in ConfigMap, return empty string
+        return "", nil
+    }
+
+    // Extract the first value from the ConfigMap data
+    jsonString := ""
+    for _, v := range cm.Data {
+        jsonString = v
+        break // Use the first value found
+    }
+
+    if jsonString == "" {
+        // No data to parse, return empty string
+        return "", nil
+    }
+
+    // Parse the JSON to extract priority class name
+    var config configData
+    if err := json.Unmarshal([]byte(jsonString), &config); err != nil {
+        // Invalid JSON is not a critical error for priority class
+        // Just return empty string to use default behavior
+        return "", nil
+    }
+
+    return config.PriorityClassName, nil
+}
+```
+
+This function will get the priority class name from the node-agent-configmap. If it's not found, it will return an empty string.
+
+### Validation and Logging
+
+To improve observability and help with troubleshooting, the implementation will include:
+
+1. **Optional Priority Class Validation**: A helper function to check if a priority class exists in the cluster. This function will be added to the `pkg/util/kube` package alongside other Kubernetes utility functions:
+
+```go
+// In pkg/util/kube/priority_class.go
+
+// ValidatePriorityClass checks if the specified priority class exists in the cluster
+// Returns true if the priority class exists or if priorityClassName is empty
+// Returns false if the priority class doesn't exist or validation fails
+// Logs warnings when the priority class doesn't exist
+func ValidatePriorityClass(ctx context.Context, kubeClient kubernetes.Interface, priorityClassName string, logger logrus.FieldLogger) bool {
+    if priorityClassName == "" {
+        return true
+    }
+    
+    _, err := kubeClient.SchedulingV1().PriorityClasses().Get(ctx, priorityClassName, metav1.GetOptions{})
+    if err != nil {
+        if apierrors.IsNotFound(err) {
+            logger.Warnf("Priority class %q not found in cluster. Pod creation may fail if the priority class doesn't exist when pods are scheduled.", priorityClassName)
+        } else {
+            logger.WithError(err).Warnf("Failed to validate priority class %q", priorityClassName)
+        }
+        return false
+    }
+    logger.Infof("Validated priority class %q exists in cluster", priorityClassName)
+    return true
+}
+```
+
+2. **Debug Logging**: Add debug logs when priority classes are applied:
+
+```go
+// In deployment creation
+if c.priorityClassName != "" {
+    logger.Debugf("Setting priority class %q for Velero server deployment", c.priorityClassName)
+}
+
+// In daemonset creation
+if c.priorityClassName != "" {
+    logger.Debugf("Setting priority class %q for node agent daemonset", c.priorityClassName)
+}
+
+// In maintenance job creation
+if priorityClassName != "" {
+    logger.Debugf("Setting priority class %q for maintenance job %s", priorityClassName, job.Name)
+}
+
+// In data mover pod creation
+if priorityClassName != "" {
+    logger.Debugf("Setting priority class %q for data mover pod %s", priorityClassName, pod.Name)
+}
+```
+
+These validation and logging features will help administrators:
+
+- Identify configuration issues early (validation warnings)
+- Troubleshoot priority class application issues
+- Verify that priority classes are being applied as expected
+
+The `ValidatePriorityClass` function should be called at the following points:
+
+1. **During `velero install`**: Validate the priority classes specified via CLI flags:
+   - After parsing `--server-priority-class-name` flag
+   - After parsing `--node-agent-priority-class-name` flag
+
+2. **When reading from ConfigMaps**: Validate priority classes when loading configurations:
+   - In `GetDataMoverPriorityClassName` when reading from node-agent-configmap
+   - In maintenance job controller when reading from repo-maintenance-job-configmap
+
+3. **During pod/job creation** (optional, for runtime validation):
+   - Before creating data mover pods (PVB/PVR/CSI snapshot data movement)
+   - Before creating maintenance jobs
+
+Example usage:
+
+```go
+// During velero install
+if o.ServerPriorityClassName != "" {
+    _ = kube.ValidatePriorityClass(ctx, kubeClient, o.ServerPriorityClassName, logger.WithField("component", "server"))
+    // For install command, we continue even if validation fails (warnings are logged)
+}
+
+// When reading from ConfigMap in node-agent server
+priorityClassName, err := kube.GetDataMoverPriorityClassName(ctx, namespace, kubeClient, configMapName)
+if err == nil && priorityClassName != "" {
+    // Validate the priority class exists in the cluster
+    if kube.ValidatePriorityClass(ctx, kubeClient, priorityClassName, logger.WithField("component", "data-mover")) {
+        dataMovePriorityClass = priorityClassName
+        logger.WithField("priorityClassName", priorityClassName).Info("Using priority class for data mover pods")
+    } else {
+        logger.WithField("priorityClassName", priorityClassName).Warn("Priority class not found in cluster, data mover pods will use default priority")
+        // Clear the priority class to prevent pod creation failures
+        priorityClassName = ""
+    }
+}
+```
+
+Note: The validation function returns a boolean to allow callers to decide how to handle missing priority classes. For the install command, validation failures are ignored (only warnings are logged) to allow for scenarios where priority classes might be created after Velero installation. For runtime components like the node-agent server, the priority class is cleared if validation fails to prevent pod creation failures.
+
+## Alternatives Considered
+
+1. **Using a single flag for all components**: We could have used a single flag for all components, but this would not allow for different priority classes for different components. Since maintenance jobs and data movers typically require lower priority than the Velero server, separate flags provide more flexibility.
+
+2. **Using a configuration file**: We could have added support for specifying the priority class names in a configuration file. However, this would have required additional changes to the Velero CLI and would have been more complex to implement.
+
+3. **Inheriting priority class from parent components**: We initially considered having maintenance jobs inherit their priority class from the Velero server, and data movers inherit from the node agent. However, this approach doesn't allow for the appropriate prioritization of different components based on their importance and resource requirements.
+
+## Security Considerations
+
+There are no security considerations for this feature.
+
+## Compatibility
+
+This feature is compatible with all Kubernetes versions that support priority classes. The PodPriority feature became stable in Kubernetes 1.14. For more information, see the [Kubernetes documentation on Pod Priority and Preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/).
+
+## ConfigMap Update Strategy
+
+### Static ConfigMap Reading at Startup
+
+The node-agent server reads and parses the ConfigMap once during initialization and passes configurations (like `podResources`, `loadAffinity`, and `priorityClassName`) directly to controllers as parameters. This approach ensures:
+
+- Single ConfigMap read to minimize API calls
+- Consistent configuration across all controllers
+- Validation of priority classes at startup with fallback behavior
+- No need for complex update mechanisms or watchers
+
+ConfigMap changes require a restart of the node-agent to take effect.
+
+### Implementation Approach
+
+1. **Data Mover Controllers**: Receive priority class as a string parameter from node-agent server at initialization
+2. **Maintenance Job Controller**: Read fresh configuration from repo-maintenance-job-configmap at job creation time
+3. ConfigMap changes require restart of components to take effect
+4. Priority class validation happens at startup with automatic fallback to prevent failures
+
+## Implementation
+
+The implementation will involve the following steps:
+
+1. Add the priority class name fields for server and node agent to the `VeleroOptions` struct
+2. Add the priority class name field to the `podTemplateConfig` struct
+3. Add the `WithPriorityClassName` function for the server deployment and daemonset
+4. Update the `Deployment` function to use the server priority class name
+5. Update the `DaemonSet` function to use the node agent priority class name
+6. Update the `JobConfigs` struct to include `PriorityClassName` field
+7. Update the `buildJob` function in maintenance job to use the priority class name from JobConfigs (global config only)
+8. Update the `Configs` struct in node agent to include `PriorityClassName` field for data mover pods
+9. Update the data mover pod creation to use the priority class name from node-agent-configmap
+10. Update the PodVolumeBackup controller to retrieve and apply priority class name from node-agent-configmap
+11. Update the PodVolumeRestore controller to retrieve and apply priority class name from node-agent-configmap
+12. Add the `GetDataMoverPriorityClassName` utility function to retrieve priority class from configmap
+13. Add the priority class name flags for server and node agent to the `velero install` command
+14. Add unit tests for:
+    - `WithPriorityClassName` function
+    - `GetDataMoverPriorityClassName` function
+    - Priority class application in deployment, daemonset, and job specs
+15. Add integration tests to verify:
+    - Priority class is correctly applied to all component pods
+    - ConfigMap updates are reflected in new pods
+    - Empty/missing priority class names are handled gracefully
+16. Update user documentation to include:
+    - How to configure priority classes for each component
+    - Examples of creating ConfigMaps before installation
+    - Expected priority class hierarchy recommendations
+    - Troubleshooting guide for priority class issues
+17. Update CLI documentation for new flags (`--server-priority-class-name` and `--node-agent-priority-class-name`)
+
+Note: The server deployment and node agent daemonset will have CLI flags for priority class. Data mover pods and maintenance jobs will use their respective ConfigMaps for priority class configuration.
+
+This approach ensures that different Velero components can use different priority class names based on their importance and resource requirements:
+
+1. The Velero server deployment can use a higher priority class to ensure it continues running even under resource pressure.
+2. The node agent daemonset can use a medium priority class.
+3. Maintenance jobs can use a lower priority class since they should not run when resources are limited.
+4. Data mover pods can use a lower priority class since they should not run when resources are limited.
+
+### Implementation Considerations
+
+Priority class names are configured through different mechanisms:
+
+1. **Server Deployment**: Uses the `--server-priority-class-name` CLI flag during installation.
+
+2. **Node Agent DaemonSet**: Uses the `--node-agent-priority-class-name` CLI flag during installation.
+
+3. **Data Mover Pods**: Will use the node-agent-configmap (specified via the `--node-agent-configmap` flag). This ConfigMap controls priority class for all data mover pods (including PVB and PVR) created by the node agent.
+
+4. **Maintenance Jobs**: Will use the repository maintenance job ConfigMap (specified via the `--repo-maintenance-job-configmap` flag). Users should create this ConfigMap before running `velero install` with the desired priority class configuration. The ConfigMap can be updated after installation to change priority classes for future maintenance jobs. While the ConfigMap structure supports per-repository configuration for resources and affinity, priority class is intentionally only read from the global configuration to ensure all maintenance jobs have the same priority.
+
+#### ConfigMap Pre-Creation Guide
+
+For components that use ConfigMaps for priority class configuration, the ConfigMaps must be created before running `velero install`. Here's the recommended workflow:
+
+```bash
+# Step 1: Create priority classes in your cluster (if not already existing)
+kubectl apply -f - <<EOF
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: velero-critical
+value: 100
+globalDefault: false
+description: "Critical priority for Velero server"
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: velero-standard
+value: 50
+globalDefault: false
+description: "Standard priority for Velero node agent"
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: velero-low
+value: 10
+globalDefault: false
+description: "Low priority for Velero data movers and maintenance jobs"
+EOF
+
+# Step 2: Create the namespace
+kubectl create namespace velero
+
+# Step 3: Create ConfigMaps for data movers and maintenance jobs
+kubectl create configmap node-agent-config -n velero --from-file=config.json=/dev/stdin <<EOF
+{
+    "priorityClassName": "velero-low"
+}
+EOF
+
+kubectl create configmap repo-maintenance-job-config -n velero --from-file=config.json=/dev/stdin <<EOF
+{
+    "global": {
+        "priorityClassName": "velero-low"
+    }
+}
+EOF
+
+# Step 4: Install Velero with priority class configuration
+velero install \
+    --provider aws \
+    --server-priority-class-name velero-critical \
+    --node-agent-priority-class-name velero-standard \
+    --node-agent-configmap node-agent-config \
+    --repo-maintenance-job-configmap repo-maintenance-job-config \
+    --use-node-agent
+```
+
+#### Recommended Priority Class Hierarchy
+
+When configuring priority classes for Velero components, consider the following hierarchy based on component criticality:
+
+1. **Velero Server (Highest Priority)**:
+   - Example: `velero-critical` with value 100
+   - Rationale: The server must remain running to coordinate backup/restore operations
+
+2. **Node Agent DaemonSet (Medium Priority)**:
+   - Example: `velero-standard` with value 50
+   - Rationale: Node agents need to be available on nodes but are less critical than the server
+
+3. **Data Mover Pods & Maintenance Jobs (Lower Priority)**:
+   - Example: `velero-low` with value 10
+   - Rationale: These are temporary workloads that can be delayed during resource contention
+
+This hierarchy ensures that core Velero components remain operational even under resource pressure, while allowing less critical workloads to be preempted if necessary.
+
+This approach has several advantages:
+
+- Leverages existing configuration mechanisms, minimizing new CLI flags
+- Provides a single point of configuration for related components (node agent and its pods)
+- Allows dynamic configuration updates without requiring Velero reinstallation
+- Maintains backward compatibility with existing installations
+- Enables administrators to set up priority classes during initial deployment
+- Keeps configuration simple by using the same priority class for all maintenance jobs
+
+The priority class name for data mover pods will be determined by checking the node-agent-configmap. This approach provides a centralized way to configure priority class names for all data mover pods. The same approach will be used for PVB (PodVolumeBackup) and PVR (PodVolumeRestore) pods, which will also retrieve their priority class name from the node-agent-configmap.
+
+For PVB and PVR pods specifically, the implementation follows this approach:
+
+1. **Controller Initialization**: Both PodVolumeBackup and PodVolumeRestore controllers are updated to accept a priority class name as a string parameter. The node-agent server reads the priority class from the node-agent-configmap once at startup:
+
+```go
+// In node-agent server startup (pkg/cmd/cli/nodeagent/server.go)
+dataMovePriorityClass := ""
+if s.config.nodeAgentConfig != "" {
+    ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+    defer cancel()
+    priorityClass, err := kube.GetDataMoverPriorityClassName(ctx, s.namespace, s.kubeClient, s.config.nodeAgentConfig)
+    if err != nil {
+        s.logger.WithError(err).Warn("Failed to get priority class name from node-agent-configmap, using empty value")
+    } else if priorityClass != "" {
+        // Validate the priority class exists in the cluster
+        if kube.ValidatePriorityClass(ctx, s.kubeClient, priorityClass, s.logger.WithField("component", "data-mover")) {
+            dataMovePriorityClass = priorityClass
+            s.logger.WithField("priorityClassName", priorityClass).Info("Using priority class for data mover pods")
+        } else {
+            s.logger.WithField("priorityClassName", priorityClass).Warn("Priority class not found in cluster, data mover pods will use default priority")
+        }
+    }
+}
+
+// Pass priority class to controllers
+pvbReconciler := controller.NewPodVolumeBackupReconciler(
+    s.mgr.GetClient(), s.mgr, s.kubeClient, ..., dataMovePriorityClass)
+pvrReconciler := controller.NewPodVolumeRestoreReconciler(
+    s.mgr.GetClient(), s.mgr, s.kubeClient, ..., dataMovePriorityClass)
+```
+
+2. **Controller Structure**: Controllers store the priority class name as a field:
+
+```go
+type PodVolumeBackupReconciler struct {
+    // ... existing fields ...
+    dataMovePriorityClass string
+}
+```
+
+3. **Pod Creation**: The priority class is included in the pod spec when creating data mover pods.
+
+### VGDP Micro-Service Considerations
+
+With the introduction of VGDP micro-services (as described in the VGDP micro-service design), data mover pods are created as dedicated pods for volume snapshot data movement. These pods will also inherit the priority class configuration from the node-agent-configmap. Since VGDP-MS pods (backupPod/restorePod) inherit their configurations from the node-agent, they will automatically use the priority class name specified in the node-agent-configmap.
+
+This ensures that all pods created by Velero for data movement operations (CSI snapshot data movement, PVB, and PVR) use a consistent approach for priority class name configuration through the node-agent-configmap.
+
+### How Exposers Receive Configuration
+
+CSI Snapshot Exposer and Generic Restore Exposer do not directly watch or read ConfigMaps. Instead, they receive configuration through their parent controllers:
+
+1. **Controller Initialization**: Controllers receive the priority class name as a parameter during initialization from the node-agent server.
+
+2. **Configuration Propagation**: During reconciliation of resources:
+   - The controller calls `setupExposeParam()` which includes the `dataMovePriorityClass` value
+   - For CSI operations: `CSISnapshotExposeParam.PriorityClassName` is set
+   - For generic restore: `GenericRestoreExposeParam.PriorityClassName` is set
+   - The controller passes these parameters to the exposer's `Expose()` method
+
+3. **Pod Creation**: The exposer creates pods with the priority class name provided by the controller.
+
+This design keeps exposers stateless and ensures:
+- Exposers remain simple and focused on pod creation
+- All configuration flows through controllers consistently
+- No complex state synchronization between components
+- Configuration changes require component restart to take effect
+
+## Open Issues
+
+None.

design/Implemented/repo_maintenance_job_config.md

@@ -4,7 +4,7 @@
 Add this design to make the repository maintenance job can read configuration from a dedicate ConfigMap and make the Job's necessary parts configurable, e.g. `PodSpec.Affinity` and `PodSpec.Resources`.
 
 ## Background
-Repository maintenance is split from the Velero server to a k8s Job in v1.14 by design [repository maintenance job](Implemented/repository-maintenance.md).
+Repository maintenance is split from the Velero server to a k8s Job in v1.14 by design [repository maintenance job](repository-maintenance.md).
 The repository maintenance Job configuration was read from the Velero server CLI parameter, and it inherits the most of Velero server's Deployment's PodSpec to fill un-configured fields.
 
 This design introduces a new way to let the user to customize the repository maintenance behavior instead of inheriting from the Velero server Deployment or reading from `velero server` CLI parameters.
@@ -13,7 +13,7 @@ It's possible new configurations are introduced in future releases based on this
 
 For the node selection, the repository maintenance Job also inherits from the Velero server deployment before, but the Job may last for a while and cost noneligible resources, especially memory.
 The users have the need to choose which k8s node to run the maintenance Job.
-This design reuses the data structure introduced by design [node-agent affinity configuration](Implemented/node-agent-affinity.md) to make the repository maintenance job can choose which node running on.
+This design reuses the data structure introduced by design [Velero Generic Data Path affinity configuration](node-agent-affinity.md) to make the repository maintenance job can choose which node running on.
 
 ## Goals
 - Unify the repository maintenance Job configuration at one place.
@@ -118,7 +118,7 @@ For example, the following BackupRepository's key should be `test-default-kopia`
     volumeNamespace: test
 ```
 
-The `LoadAffinity` structure is reused from design [node-agent affinity configuration](Implemented/node-agent-affinity.md).
+The `LoadAffinity` structure is reused from design [Velero Generic Data Path affinity configuration](node-agent-affinity.md).
 It's possible that the users want to choose nodes that match condition A or condition B to run the job.
 For example, the user want to let the nodes is in a specified machine type or the nodes locate in the us-central1-x zones to run the job.
 This can be done by adding multiple entries in the `LoadAffinity` array.

design/Implemented/resource-status-restore.md

@@ -0,0 +1,113 @@
+# Allow Object-Level Resource Status Restore in Velero
+
+## Abstract
+This design proposes a way to enhance Velero’s restore functionality by enabling object-level resource status restoration through annotations. 
+Currently, Velero allows restoring resource statuses only at a resource type level, which lacks granularity of restoring the status of specific resources. 
+By introducing an annotation that controllers can set on individual resource objects, this design aims to improve flexibility and autonomy for users/resource-controllers, providing a more way
+to enable resource status restore.
+
+
+## Background
+Velero provides the `restoreStatus` field in the Restore API to specify resource types for status restoration. However, this feature is limited to resource types as a whole, lacking the granularity needed to restore specific objects of a resource type. Resource controllers, especially those managing custom resources with external dependencies, may need to restore status on a per-object basis based on internal logic and dependencies.
+
+This design adds an annotation-based approach to allow controllers to specify status restoration at the object level, enabling Velero to handle status restores more flexibly.
+
+## Goals
+- Provide a mechanism to specify the restoration of a resource’s status at an object level.
+- Maintain backwards compatibility with existing functionality, allowing gradual adoption of this feature.
+- Integrate the new annotation-based objects-level status restore with Velero’s existing resource-type-level `restoreStatus` configuration.
+
+## Non-Goals
+- Alter Velero’s existing resource type-level status restoration mechanism for resources without annotations.
+
+## Use-Cases/Scenarios
+
+1. Controller managing specific Resources
+  - A resource controller identifies that a specific object of a resource should have its status restored due to particular dependencies
+  - The controller automatically sets the `velero.io/restore-status: true` annotation on the resource.
+  - During restore, Velero restores the status of this object, while leaving other resources unaffected.
+  - The status for the annotated object will be restored regardless of its inclusion/exclusion in `restoreStatus.includedResources`
+
+2. A specific object must not have its status restored even if its included in `restoreStatus.includedResources`
+  - A user specifies a resource type in the `restoreStatus.includedResources` field within the Restore custom resource.
+  - A particular object of that resource type is annotated with `velero.io/restore-status: false` by the user.
+  - The status of the annotated object will not restored even though its included in `restoreStatus.includedResources` because annotation is `false` and it takes precedence.
+
+4. Default Behavior for objects Without the Annotation
+  - Objects without the `velero.io/restore-status` annotation behave as they currently do: Velero skips their status restoration unless the resource type is specified in the `restoreStatus.includedResources` field.
+
+## High-Level Design
+
+- Object-Level Status Restore Annotation: We are introducing the `velero.io/restore-status` annotation at the resource object level to mark specific objects for status restoration.
+  - `true`: Indicates that the status should be restored for this object
+  - `false`: Skip restoring status for this specific object
+  - Invalid or missing annotations defer to the meaning of existing resource type-level logic.
+
+- Restore logic precedence: 
+  - Annotations take precedence when they exist with valid values (`true` or `false`).
+  - Restore spec `restoreStatus.includedResources` is only used when annotations are invalid or missing.
+
+- Velero Restore Logic Update: During a restore operation, Velero will:
+  - Extend the existing restore logic to parse and prioritize annotations introduced in this design.
+  - Update resource objects accordingly based on their annotation values or fallback configuration.
+
+
+## Detailed Design
+
+- Annotation for object-Level Status Restore: The `velero.io/restore-status` annotation will be set on individual resource objects by users/controllers as needed:
+```yaml
+metadata:
+  annotations:
+    velero.io/restore-status: "true"
+```
+
+- Restore Logic Modifications: During the restore operation, the restore controller will follow these steps:
+   - Parse the `restoreStatus.includedResources` spec to determine resource types eligible for status restoration.
+   - For each resource object:
+     - Check for the `velero.io/restore-status` annotation.
+     - If the annotation value is:
+       - `true`: Restore the status of the object
+       - `false`: Skip restoring the status of the object
+     - If the annotation is invalid or missing:
+       - Default to the `restoreStatus.includedResources` configuration
+
+
+## Implementation
+
+We are targeting the implementation of this design for Velero 1.16 release.
+
+Current restoreStatus logic resides here: https://github.com/vmware-tanzu/velero/blob/32a8c62920ad96c70f1465252c0197b83d5fa6b6/pkg/restore/restore.go#L1652
+
+The modified logic would look somewhat like:
+
+```go
+// Determine whether to restore status from resource type configuration
+shouldRestoreStatus := ctx.resourceStatusIncludesExcludes != nil && ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String())
+
+// Check for object-level annotation
+annotations := obj.GetAnnotations()
+objectAnnotation := annotations["velero.io/restore-status"]
+annotationValid := objectAnnotation == "true" || objectAnnotation == "false"
+
+// Determine restore behavior based on annotation precedence
+shouldRestoreStatus = (annotationValid && objectAnnotation == "true") || (!annotationValid && shouldRestoreStatus)
+
+ctx.log.Debugf("status field for %s: exists: %v, should restore: %v (by annotation: %v)", newGR, statusFieldExists, shouldRestoreStatus, annotationValid)
+
+if shouldRestoreStatus && statusFieldExists {
+    if err := unstructured.SetNestedField(obj.Object, objStatus, "status"); err != nil {
+        ctx.log.Errorf("Could not set status field %s: %v", kube.NamespaceAndName(obj), err)
+        errs.Add(namespace, err)
+        return warnings, errs, itemExists
+    }
+    obj.SetResourceVersion(createdObj.GetResourceVersion())
+    updated, err := resourceClient.UpdateStatus(obj, metav1.UpdateOptions{})
+    if err != nil {
+        ctx.log.Infof("Status field update failed %s: %v", kube.NamespaceAndName(obj), err)
+        warnings.Add(namespace, err)
+	} else {
+        createdObj = updated
+    }
+}
+```
+

design/Implemented/schedule-skip-immediately-config_design.md

@@ -71,6 +71,20 @@ type ScheduleSpec struct {
 }
 ```
 
+**Note:** The Velero server automatically patches the `skipImmediately` field back to `false` after it's been used. This is because `skipImmediately` is designed to be a one-time operation rather than a persistent state. When the controller detects that `skipImmediately` is set to `true`, it:
+1. Sets the flag back to `false`
+2. Records the current time in `schedule.Status.LastSkipped`
+
+This "consume and reset" pattern ensures that after skipping one immediate backup, the schedule returns to normal behavior for subsequent runs. The `LastSkipped` timestamp is then used to determine when the next backup should run.
+
+```go
+// From pkg/controller/schedule_controller.go
+if schedule.Spec.SkipImmediately != nil && *schedule.Spec.SkipImmediately { 
+    *schedule.Spec.SkipImmediately = false 
+    schedule.Status.LastSkipped = &metav1.Time{Time: c.clock.Now()} 
+} 
+```
+
 `LastSkipped` will be added to `ScheduleStatus` struct to track the last time a schedule was skipped.
 ```diff
 // ScheduleStatus captures the current state of a Velero schedule
@@ -97,6 +111,8 @@ type ScheduleStatus struct {
 }
 ```
 
+The `LastSkipped` field is crucial for the schedule controller to determine the next run time. When a backup is skipped, this timestamp is used instead of `LastBackup` to calculate when the next backup should occur, ensuring the schedule maintains its intended cadence even after skipping a backup.
+
 When `schedule.spec.SkipImmediately` is `true`, `LastSkipped` will be set to the current time, and `schedule.spec.SkipImmediately` set to nil so it can be used again.
 
 The `getNextRunTime()` function below is updated so `LastSkipped` which is after `LastBackup` will be used to determine next run time.

design/Implemented/supporting-volumeattributes-resource-policy.md

@@ -0,0 +1,84 @@
+# Adding Support For VolumeAttributes in Resource Policy
+ 
+## Abstract
+Currently [Velero Resource policies](https://velero.io/docs/main/resource-filtering/#creating-resource-policies) are only supporting "Driver" to be filtered for [CSI volume conditions](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources_validator.go#L28)
+
+If user want to skip certain CSI volumes based on other volume attributes like protocol or SKU, etc, they can't do it with the current Velero resource policies. It would be convenient if Velero resource policies could be extended to filter on volume attributes along with existing driver filter in the resource policies `conditions` to handle the backup of volumes just by `some specific volumes attributes conditions`.
+ 
+## Background
+As of Today, Velero resource policy already provides us the way to filter volumes based on the `driver` name. But it's not enough to handle the volumes based on other volume attributes like protocol, SKU, etc.
+
+## Example:
+  - Provision Azure NFS: Define the Storage class with `protocol: nfs` under storage class parameters to provision [CSI NFS Azure File Shares](https://learn.microsoft.com/en-us/azure/aks/azure-files-csi#nfs-file-shares).
+  - User wants to back up AFS (Azure file shares) but only want to backup `SMB` type of file share volumes and not `NFS` file share volumes.
+
+## Goals
+- We are only bringing additional support in the resource policy to only handle volumes during backup.
+- Introducing support for `VolumeAttributes` filter along with `driver` filter in CSI volume conditions to handle volumes.
+ 
+## Non-Goals
+- Currently, only handles volumes, and does not support other resources.
+ 
+## Use-cases/Scenarios
+### Skip backup volumes by some volume attributes:
+Users want to skip PV with the requirements:
+- option to skip specified PV on volume attributes type (like Protocol as NFS, SMB, etc)
+
+### Sample Storage Class Used to create such Volumes
+```
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azurefile-csi-nfs
+provisioner: file.csi.azure.com
+allowVolumeExpansion: true
+parameters:
+  protocol: nfs 
+```
+
+## High-Level Design
+Modifying the existing Resource Policies code for [csiVolumeSource](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources_validator.go#L28C6-L28C22) to add the new `VolumeAttributes` filter for CSI volumes and adding validations in existing [csiCondition](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources.go#L150) to match with volume attributes in the conditions from Resource Policy config map and original persistent volume.
+
+## Detailed Design
+The volume resources policies should contain a list of policies which is the combination of conditions and related `action`, when target volumes meet the conditions, the related `action` will take effection.
+
+Below is the API Design for the user configuration:
+
+### API Design
+```go
+type csiVolumeSource struct {
+	Driver string `yaml:"driver,omitempty"`
+	// [NEW] CSI volume attributes
+	VolumeAttributes map[string]string `yaml:"volumeAttributes,omitempty"`
+}
+```
+
+The policies YAML config file would look like this:
+```yaml
+version: v1
+volumePolicies:
+  - conditions:
+      csi:
+        driver: disk.csi.azure.com
+    action:
+      type: skip
+  - conditions:
+      csi:
+        driver: file.csi.azure.com
+        volumeAttributes:
+          protocol: nfs
+    action:
+      type: skip`
+```
+ 
+### New Supported Conditions
+#### VolumeAttributes
+Existing CSI Volume Condition can now add `volumeAttributes` which will be key and value pairs.  
+
+ Specify details for the related volume source (currently only csi driver is supported filter)
+     ```yaml
+     csi: // match volume using `file.csi.azure.com` and with volumeAttributes protocol as nfs
+       driver: file.csi.azure.com 
+       volumeAttributes:
+          protocol: nfs
+     ```

design/Implemented/vgdp-affinity-enhancement.md

@@ -0,0 +1,257 @@
+# Velero Generic Data Path Load Affinity Enhancement Design
+
+## Glossary & Abbreviation
+
+**Velero Generic Data Path (VGDP)**: VGDP is the collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. 
+
+**Exposer**: Exposer is a module that is introduced in [Volume Snapshot Data Movement Design][1]. Velero uses this module to expose the volume snapshots to Velero node-agent pods or node-agent associated pods so as to complete the data movement from the snapshots.
+
+## Background
+
+The implemented [VGDP LoadAffinity design][3] already defined the a structure `LoadAffinity` in `--node-agent-configmap` parameter. The parameter is used to set the affinity of the backupPod of VGDP.
+
+There are still some limitations of this design:
+* The affinity setting is global. Say there are two StorageClasses and the underlying storage can only provision volumes to part of the cluster nodes. The supported nodes don't have intersection. Then the affinity will definitely not work in some cases.
+* The old design focuses on the backupPod affinity, but the restorePod also needs the affinity setting.
+
+As a result, create this design to address the limitations.
+
+## Goals
+
+- Enhance the node affinity of VGDP instances for volume snapshot data movement: add per StorageClass node affinity.
+- Enhance the node affinity of VGDP instances for volume snapshot data movement: support the or logic between affinity selectors.
+- Define the behaviors of node affinity of VGDP instances in node-agent for volume snapshot data movement restore, when the PVC restore doesn't require delay binding.
+
+## Non-Goals
+
+- It is also beneficial to support VGDP instances affinity for PodVolume backup/restore, this will be implemented after the PodVolume micro service completes.
+
+## Solution
+
+This design still uses the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` to host the node affinity configurations.
+
+Upon the implemented [VGDP LoadAffinity design][3] introduced `[]*LoadAffinity` structure, this design add a new field `StorageClass`. This field is optional.
+* If the `LoadAffinity` element's `StorageClass` doesn't have value, it means this element is applied to global, just as the old design.
+* If the `LoadAffinity` element's `StorageClass` has value, it means this element is applied to the VGDP instances' PVCs use the specified StorageClass.
+* The `LoadAffinity` element whose `StorageClass` has value has higher priority than the `LoadAffinity` element whose `StorageClass` doesn't have value.
+
+
+```go
+type Configs struct {
+    // LoadConcurrency is the config for load concurrency per node.
+    LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"`
+
+    // LoadAffinity is the config for data path load affinity.
+    LoadAffinity []*LoadAffinity `json:"loadAffinity,omitempty"`    
+}
+
+type LoadAffinity struct {
+    // NodeSelector specifies the label selector to match nodes
+    NodeSelector metav1.LabelSelector `json:"nodeSelector"`
+}
+```
+
+``` go
+type LoadAffinity struct {
+    // NodeSelector specifies the label selector to match nodes
+    NodeSelector metav1.LabelSelector `json:"nodeSelector"`
+
+    // StorageClass specifies the VGDPs the LoadAffinity applied to. If the StorageClass doesn't have value, it applies to all. If not, it applies to only the VGDPs that use this StorageClass.
+    StorageClass string `json:"storageClass"`
+}
+```
+
+### Decision Tree
+
+```mermaid
+flowchart TD
+    A[VGDP Pod Needs Scheduling] --> B{Is this a restore operation?}
+    
+    B -->|Yes| C{StorageClass has volumeBindingMode: WaitForFirstConsumer?}
+    B -->|No| D[Backup Operation]
+    
+    C -->|Yes| E{restorePVC.ignoreDelayBinding = true?}
+    C -->|No| F[StorageClass binding mode: Immediate]
+    
+    E -->|No| G[Wait for target Pod scheduling<br/>Use Pod's selected node<br/>⚠️ Affinity rules ignored]
+    E -->|Yes| H[Apply affinity rules<br/>despite WaitForFirstConsumer]
+    
+    F --> I{Check StorageClass in loadAffinity by StorageClass field}
+    H --> I
+    D --> J{Using backupPVC with different StorageClass?}
+    
+    J -->|Yes| K[Use final StorageClass<br/>for affinity lookup]
+    J -->|No| L[Use original PVC StorageClass<br/>for affinity lookup]
+    
+    K --> I
+    L --> I
+    
+    I -->|StorageClass found| N[Filter the LoadAffinity by <br/>the StorageClass<br/>🎯 and apply the LoadAffinity HIGHEST PRIORITY]
+    I -->|StorageClass not found| O{Check loadAffinity element without StorageClass field}
+
+    O -->|No loadAffinity configured| R[No affinity constraints<br/>Schedule on any available node<br/>🌐 DEFAULT]
+    
+    O --> V[Validate node-agent availability<br/>⚠️ Ensure node-agent pods exist on target nodes]
+    N --> V
+    
+    V --> W{Node-agent available on selected nodes?}
+    W -->|Yes| X[✅ VGDP Pod scheduled successfully]
+    W -->|No| Y[❌ Pod stays in Pending state<br/>Timeout after 30min<br/>Check node-agent DaemonSet coverage]
+    
+    R --> Z[Schedule on any node<br/>✅ Basic scheduling]
+    
+    %% Styling
+    classDef successNode fill:#d4edda,stroke:#155724,color:#155724
+    classDef warningNode fill:#fff3cd,stroke:#856404,color:#856404
+    classDef errorNode fill:#f8d7da,stroke:#721c24,color:#721c24
+    classDef priorityHigh fill:#e7f3ff,stroke:#0066cc,color:#0066cc
+    classDef priorityMedium fill:#f0f8ff,stroke:#4d94ff,color:#4d94ff
+    classDef priorityDefault fill:#f8f9fa,stroke:#6c757d,color:#6c757d
+    
+    class X,Z successNode
+    class G,V,Y warningNode
+    class Y errorNode
+    class N,T,U priorityHigh
+    class P,Q priorityMedium
+    class R priorityDefault
+```
+
+### Examples
+
+#### LoadAffinity interacts with LoadAffinityPerStorageClass
+
+``` json
+{
+    "loadAffinity": [
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "beta.kubernetes.io/instance-type": "Standard_B4ms"
+                }
+            }
+        },
+        {
+            "nodeSelector": {
+                "matchExpressions": [
+                    {
+                        "key": "kubernetes.io/os",
+                        "values": [
+                            "linux"
+                        ],
+                        "operator": "In"
+                    }
+                ]
+            },
+            "storageClass": "kibishii-storage-class"
+        },
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "beta.kubernetes.io/instance-type": "Standard_B8ms"
+                }
+            },
+            "storageClass": "kibishii-storage-class"
+        }
+    ]
+}
+```
+
+This sample demonstrates how the `loadAffinity` elements with `StorageClass` field and without `StorageClass` field setting work together.
+If the VGDP mounting volume is created from StorageClass `kibishii-storage-class`, its pod will run Linux nodes or instance type as `Standard_B8ms`.
+
+The other VGDP instances will run on nodes, which instance type is `Standard_B4ms`.
+
+#### LoadAffinity interacts with BackupPVC
+
+``` json
+{
+    "loadAffinity": [
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "beta.kubernetes.io/instance-type": "Standard_B4ms"
+                }
+            },
+            "storageClass": "kibishii-storage-class"
+        },
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "beta.kubernetes.io/instance-type": "Standard_B2ms"
+                }
+            },
+            "storageClass": "worker-storagepolicy"
+        }
+    ],
+    "backupPVC": {
+        "kibishii-storage-class": {
+            "storageClass": "worker-storagepolicy"
+        }
+    }
+}
+```
+
+Velero data mover supports to use different StorageClass to create backupPVC by [design](https://github.com/vmware-tanzu/velero/pull/7982).
+
+In this example, if the backup target PVC's StorageClass is `kibishii-storage-class`, its backupPVC should use StorageClass `worker-storagepolicy`. Because the final StorageClass is `worker-storagepolicy`, the backupPod uses the loadAffinity specified by `loadAffinity`'s elements with `StorageClass` field set to `worker-storagepolicy`. backupPod will be assigned to nodes, which instance type is `Standard_B2ms`.
+
+
+#### LoadAffinity interacts with RestorePVC
+
+``` json
+{
+    "loadAffinity": [
+        {
+            "nodeSelector": {
+                "matchLabels": {
+                    "beta.kubernetes.io/instance-type": "Standard_B4ms"
+                }
+            },
+            "storageClass": "kibishii-storage-class"
+        }
+    ],
+    "restorePVC": {
+        "ignoreDelayBinding": false
+    }
+}
+```
+
+##### StorageClass's bind mode is WaitForFirstConsumer
+
+``` yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: kibishii-storage-class
+parameters:
+  svStorageClass: worker-storagepolicy
+provisioner: csi.vsphere.vmware.com
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+```
+
+If restorePVC should be created from StorageClass `kibishii-storage-class`, and it's volumeBindingMode is `WaitForFirstConsumer`.
+Although `loadAffinityPerStorageClass` has a section matches the StorageClass, the `ignoreDelayBinding` is set `false`, the Velero exposer will wait until the target Pod scheduled to a node, and returns the node as SelectedNode for the restorePVC.
+As a result, the `loadAffinityPerStorageClass` will not take affect.
+
+##### StorageClass's bind mode is Immediate
+
+``` yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: kibishii-storage-class
+parameters:
+  svStorageClass: worker-storagepolicy
+provisioner: csi.vsphere.vmware.com
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+```
+
+Because the StorageClass volumeBindingMode is `Immediate`, although `ignoreDelayBinding` is set to `false`, restorePVC will not be created according to the target Pod.
+
+The restorePod will be assigned to nodes, which instance type is `Standard_B4ms`.
+
+[1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
+[2]: Implemented/volume-snapshot-data-movement/volume-snapshot-data-movement.md
+[3]: Implemented/node-agent-affinity.md

design/Implemented/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md

@@ -0,0 +1,662 @@
+# VGDP Micro Service For fs-backup
+
+## Glossary & Abbreviation
+
+**VGDP**: Velero Generic Data Path. The collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transmission for various purposes. It includes uploaders and the backup repository.  
+**fs-backup**: Also known as pod volume backup (PVB)/pod volume restore (PVR). It is one of the primary backup methods built-in with Velero. It has been refactored in [Unified Repository design][1].  
+**PVB**: Pod Volume Backup, the internal name for backup part of fs-backup.  
+**PVR**: Pod Volume Restore, the internal name for restore part of fs-backup.  
+**Exposer**: Exposer is introduced in [Volume Snapshot Data Movement design][2] and is used to expose the volume snapshots/volumes for VGDP to access locally.  
+**VGDP MS**: VGDP Micro Service, it is introduced in [VGDP Micro Service For Volume Snapshot Data Movement][3]. It hosts VGDP instances in dedicated backup/restore pods, instead of in node-agent pods.   
+
+## Background
+As described in [VGDP Micro Service For Volume Snapshot Data Movement][3], hosting VGDP instances in dedicated pods has solved many major problems and brought significant improvements in scalability. These improvements are also effective for fs-backup. And besides the benefits listed in [VGDP Micro Service For Volume Snapshot Data Movement][3], we can also see below ones specifically for fs-backup:
+- This enables fs-backup to support Windows workloads. Windows doesn't support propagate mount, so the current fs-backup solution doesn't work for Windows nodes and Windows workloads. However, if the final host-path for the source volume is mounted to the VGDP MS pods, it should work.   
+- This enables fs-backup to reuse the existing VGDP features seamlessly, i.e., concurrency control, node selector, etc.
+
+By moving all VGDP instances out of node-agent pods, we would further get prepared for below important features and improvements:
+- NFS support: NFS volumes are mounted to VGDP MS pods, so node-agent pods don't need to restart when a new BSL is added.
+- Performance improvement for Kopia uploader restore ([#7725][9]): dedicated cache volumes could be mounted to the VGDP MS pods, without affecting node-agent pods.
+- Controllable resource usage for node-agent: node-agent pods are long running and so not suitable for data path activities as the OS usually reclaim memory in a lazy reclaim behavior, so the unused memory may be shown as occupied by node-agent pods, which misleads Kubernetes or other related sub system. After this change, node-agent pods no longer require large resource (CPU/memory) usage, so no obvious memory retain will be observed.
+- Simplify node-agent configuration: host-path mounts, root user and privileged mode are no longer required by node-agent; and the configuration differences of node-agent for linux and Windows nodes could be eliminated. 
+
+## Goals
+- Create a solution to make VGDP instances as micro services for fs-backup
+- Modify the fs-backup workflow to offload the VGDP work from node-agent to the VGDP MS
+- Create the mechanism for fs-backup to control and monitor the VGDP MS in various scenarios
+
+## Non-Goals
+- The current solution covers the VGDP Micro Service for fs-backup itself, the potentional features/improvements that rely on this solution will be covered by further designs and implementations.   
+
+
+## Overview
+The solution is based on [VGDP Micro Service For Volume Snapshot Data Movement][3], the architecture is followed as is and existing components are not changed unless it is necessary.  
+Below diagram shows how these components work together:  
+![vgdp-ms-1.png](vgdp-ms-1.png)  
+
+Below lists the changed components, why and how:  
+**Pod-Volume Exposer**: A new exposer, pod-volume exposer is added. It retrieves the host path of the specific volume and then creates the backupPod/restorePod and mounts the host path to the pod. The command of the backupPod/restorePod is also changed to launch VGDP MS for PVB/PVR.  
+**PVB/PVR Controller**: The PVB/PVR controllers are refactored to work with podVolume exposer, VGDP-MS, etc. The controllers will also support Cancel and resume. So PVB/PVR CRD is also refactored to support these scenarios.    
+**PVB/PVR VGDP-MS**: New commands for PVB/PVR VGDP-MS are added. The VGDP instances are started in the backupPod/restorePod as result of the commands.  
+
+The VGDP Watcher and its mechanism are fully reused.
+
+The [Node-agent concurrency][4] is reused to control the concurrency of VGDP MS for fs-backup. When there are too many volumes in the backup/restore, which takes too much computing resources(CPU, memory, etc.) or Kubernetes resources(pods, PVCs, PVs, etc.), users could set the concurrency in each node so as to control the total number of concurrent VGDP instances in the cluster.  
+
+## Detailed Design
+### Exposer
+As the old behavior, the host path (e.g., `/var/lib/kubelet/pods`) for the Kubernetes pods are mounted to node-agent pods, then the VGDP instances running in the same pods access the data through subdir of the host path for a specific volume, e.g.,  `/var/lib/kubelet/pods/<pod UID>/volumes/kubernetes.io~csi/<PVC name>/mount`. Therefore, a node-agent pod could access all volumes attached to the same node.  
+For the new implementation, the exposer retrieves the host path for a specific volume directly, and then mount that host path to the backupPod/restorePod. This also means that the backupPod/restorePod could only access the volume to be backed up or restored.    
+
+The exposer creates backupPod/restorePod and sets ```velero pod-volume``` as the command run by backupPod/restorePod. And `velero` image is used for the backupPod/restorePod.   
+There are sub commands varying from backup and restore:  
+```velero pod-volume backup --volume-path xxx --pod-volume-backup xxx --resource-timeout xxx --log-format xxx --log-level xxx```  
+Or:  
+```velero pod-volume restore --volume-path xxx --pod-volume-restore xxx --resource-timeout xxx --log-format xxx --log-level xxx```  
+
+Below are the parameters of the commands:  
+**volume-path**: Deliver the full path inside the backupPod/restorePod for the volume to be backed up/restored.    
+**pod-volume-backup**: PVB CR for this backup.  
+**pod-volume-restore**: PVR CR for this restore.  
+**resource-timeout**: resource-timeout is used to control the timeout for operations related to resources. It has the same meaning with the resource-timeout for node-agent.  
+**log-format** and **log-level**: This is to control the behavior of log generation inside VGDP-MS.  
+
+Below pod configurations are inherited from node-agent and set to backupPod/restorePod's spec:
+- Volumes: Some configMaps will be mapped as volumes to node-agent, so we add the same volumes of node-agent to the backupPod/restorePod
+- Environment Variables
+- Security Contexts  
+
+Since the volume data is still accessed by host path, the backupPod/restorePod may still need to run in Privileged mode in some environments. Therefore, the Privileged mode setting which is a part of Security Contexts will be inherited from node-agent.    
+The root user is still required, especially by the restore (in order to restore the file system attributes, owners, etc.), so we will use root user for backupPod/restorePod.  
+
+As same as [VGDP Micro Service For Volume Snapshot Data Movement][3], the backupPod/restorePods's ```RestartPolicy``` is set to ```RestartPolicyNever```, so that once VGDP-MS terminates for any reason, backupPod/restorePod won't restart and the PVB/PVR is marked as one of the terminal phases (Completed/Failed/Cancelled) accordingly.  
+
+### VGDP Watcher
+The VGDP watcher is fully reused, specifically, we still use the dual mode event watcher to watch the status change from backupPod/restorePod or the VGDP instance.  
+The AsyncBR adapter and its interface is also fully reused.  
+
+### VGDP-MS
+The VGDP-MS that is represented by ```velero pod-volume``` keeps the same workflow as [VGDP Micro Service For Volume Snapshot Data Movement][3]:  
+![vgdp-ms-2.png](vgdp-ms-2.png)  
+
+**Start DUCR/DDCR Watcher**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3], except that it watches PVB/PVR CRs.  
+**Wait DUCR/DDCR InProgress**: The same as The same as [VGDP Micro Service For Volume Snapshot Data Movement][3], VGDP-MS won't start the VGDP instance until PVB/PVR CR turns to ```InProgress```.  
+**Record VGDP Starts**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3].  
+**VGDP Callbacks**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3].  
+**Record VGDP Ends**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3].    
+**Record VGDP Progress**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3].  
+**Set VGDP Output**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3].  
+
+The return message for VGDP completion is also reused, except that `VolMode` is always set to `PersistentVolumeFilesystem`:  
+```
+type BackupResult struct {
+    SnapshotID    string              `json:"snapshotID"`
+    EmptySnapshot bool                `json:"emptySnapshot"`
+    Source        exposer.AccessPoint `json:"source,omitempty"`
+}
+```
+```
+type RestoreResult struct {
+    Target exposer.AccessPoint `json:"target,omitempty"`
+}
+```  
+``` 
+type AccessPoint struct {
+    ByPath  string                        `json:"byPath"`
+    VolMode uploader.PersistentVolumeMode `json:"volumeMode"`
+}
+``` 
+
+And the mechanism and data struct for Progress update is also reused:
+``` 
+type Progress struct {
+    TotalBytes int64 `json:"totalBytes,omitempty"`
+    BytesDone  int64 `json:"doneBytes,omitempty"`
+}
+```    
+
+### Log Collection
+The log collection mechanism is the same as [VGDP Micro Service For Volume Snapshot Data Movement][3].       
+
+### Resource Control
+The resource control mechanism is the same as [VGDP Micro Service For Volume Snapshot Data Movement][3].  
+
+### Restic Restore
+As the current Restic path deprecation process, restore is still supported. On the other hand, we don't want to support Restic path for this new VGDP MS implementation.  
+Therefore, the legacy PVR controller and workflow is preserved for Restic path restore.  The controller watches legacy PVRs only, and then launches the legacy workflow. Meawhile, the new PVR controller should skip legacy PVRs.  
+After Restic path is full deprecated, the code for the legacy controller and workflow should be removed.  
+
+### Velero Server Restarts
+The backup/restore stays in InProgress phase during the running of PVB/PVR, no phase changes between completion of item iteration and completion of PVB/PVR.   As a result, on Velero server restarts, there is no way to resume a backup/restore.  
+Therefore, the backup/restore will be be marked as Failed, which is the same as the old behavior. And it is still not as good as CSI snapshot data movement for which the backup/restore could be resumed as long as it has iterated all items.  
+By the meanwhile, there is indeed some improvements. As the old behavior, once the backup/restore is set as Failed on Velero server restart, the running PVB/PVR will be left there, as a result, the VGDP instances may run for a long time and take lots of resource for nothing; for the new implementation, PVB/PVR will be set as Cancel immediately after the backup/restore is set as Failed.  
+
+### node-agent Restarts
+As the old behavior, once a node-agent pod restarts, all the PVBs/PVRs running in the same node will be set as Failed as there is no way to resume the VGDP instances for them.  
+For the new implementation, since the VGDP instances run in dedicated backupPods/restorePods without affected, the PVBs/PVRs will be resumed after node-agent restarts. This includes PVBs/PVRs in all phases.  
+
+The legacy PVRs handling Restic restore are processed by the old workflow, so they will still be set as Failed on node-agent restart.  
+
+### Windows Support
+Windows nodes and workloads will be supported by following the same changes for CSI snapshot data movement as listed in [Velero Windows Support][7]. There are some additional changes particularly for PVB/PVR.   
+
+#### Restore Helper 
+PVR requires an init-container, called `restore-wait`, to run in the workload pod. There are default configurations for the container and users could customize them by the `pod-volume-restore` RIA plugin configMap.
+The `pod-volume-restore` RIA is used to config the init-container, so it should support Windows pods for all the configurations.  
+Meanwhile, the customized options in the configMap should also support Windows pods. If an option is not suitable for Windows pods, it will be ignored by the RIA.  
+
+By default, the init-container uses `velero` image with a binary called `velero-restore-helper` inside, so that binary should be compiled and assembled to the `velero` image for Windows.  
+
+#### Privileged mode
+Privileged pods are implemented by [HostProcess Pods][8] on Windows and need to be specially configured. And there are many constrains for it.  
+As one of the constrains, HostProcess pods supports Windows service accounts only. As a result, restore will not be able to support it until [#8423][10] is fixed, otherwise, the restored files are not usable by workloads which run under genneral container users, e.g., `containerUser` or `containerAdministrator`.  
+Therefore, as the current implementation, fs-backup will not support Windows workloads in the environments where Privileged mode is required. A limitation should be documented.  
+
+## node-agent
+node-agent is required to host the PVB/PVR controller which reconciles PVB/PVR and operates PVB/PVR in other steps before the VGDP instance is started, i.e., Accept, Expose, etc.  
+node-agent still requires host path mount because of two deprecating features [in-tree storage provider support deprecation][5] and [emptyDir volume support deprecation][6]. As a result, Privileged mode and root user are still required in some environments. Therefore, we will keep the node-agent deamonset as is, until the two deprecations complete.   
+
+## CRD Changes
+In order to support the VGDP MS workflow, some elements in the PVB/PVR CRDs are added or extended:
+- New phases are added for PVB/PVR: `PodVolumeBackupPhaseAccepted`, `PodVolumeBackupPhasePrepared`, `PodVolumeBackupPhaseCanceling`, `PodVolumeBackupPhaseCanceled`; `PodVolumeRestorePhaseAccepted`, `PodVolumeRestorePhasePrepared`, `PodVolumeRestorePhaseCanceling`, `PodVolumeRestorePhaseCanceled`.  
+- New fields are added to PVB/PVR spec to support cancel: `Cancel bool`
+- New fields are added to PVB/PVR spec to support the accept phase and processing: `AcceptedTimestamp *metav1.Time`
+- A new field, which records the node the PVR is running, is added to PVR Status: `Node string`
+
+New changes happen to Backup/Restore CRDs.  
+
+Below is the new PVB CRD:
+```yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: podvolumebackups.velero.io
+spec:
+  group: velero.io
+  names:
+    kind: PodVolumeBackup
+    listKind: PodVolumeBackupList
+    plural: podvolumebackups
+    singular: podvolumebackup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: PodVolumeBackup status such as New/InProgress
+      jsonPath: .status.phase
+      name: Status
+      type: string
+    - description: Time duration since this PodVolumeBackup was started
+      jsonPath: .status.startTimestamp
+      name: Started
+      type: date
+    - description: Completed bytes
+      format: int64
+      jsonPath: .status.progress.bytesDone
+      name: Bytes Done
+      type: integer
+    - description: Total bytes
+      format: int64
+      jsonPath: .status.progress.totalBytes
+      name: Total Bytes
+      type: integer
+    - description: Name of the Backup Storage Location where this backup should be
+        stored
+      jsonPath: .spec.backupStorageLocation
+      name: Storage Location
+      type: string
+    - description: Time duration since this PodVolumeBackup was created
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Name of the node where the PodVolumeBackup is processed
+      jsonPath: .status.node
+      name: Node
+      type: string
+    - description: The type of the uploader to handle data transfer
+      jsonPath: .spec.uploaderType
+      name: Uploader
+      type: string
+    name: v1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: PodVolumeBackupSpec is the specification for a PodVolumeBackup.
+            properties:
+              backupStorageLocation:
+                description: |-
+                  BackupStorageLocation is the name of the backup storage location
+                  where the backup repository is stored.
+                type: string
+              cancel:
+                description: |-
+                  Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set
+                  when the PodVolumeBackup is in InProgress phase
+                type: boolean
+              node:
+                description: Node is the name of the node that the Pod is running
+                  on.
+                type: string
+              pod:
+                description: Pod is a reference to the pod containing the volume to
+                  be backed up.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: |-
+                      If referring to a piece of an object instead of an entire object, this string
+                      should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                      For example, if the object reference is to a container within a pod, this would take on a value like:
+                      "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                      the event) or if no container name is specified "spec.containers[2]" (container with
+                      index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                      referencing a part of an object.
+                    type: string
+                  kind:
+                    description: |-
+                      Kind of the referent.
+                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                    type: string
+                  name:
+                    description: |-
+                      Name of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                    type: string
+                  namespace:
+                    description: |-
+                      Namespace of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                    type: string
+                  resourceVersion:
+                    description: |-
+                      Specific resourceVersion to which this reference is made, if any.
+                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                    type: string
+                  uid:
+                    description: |-
+                      UID of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                    type: string
+                type: object
+                x-kubernetes-map-type: atomic
+              repoIdentifier:
+                description: RepoIdentifier is the backup repository identifier.
+                type: string
+              tags:
+                additionalProperties:
+                  type: string
+                description: |-
+                  Tags are a map of key-value pairs that should be applied to the
+                  volume backup as tags.
+                type: object
+              uploaderSettings:
+                additionalProperties:
+                  type: string
+                description: |-
+                  UploaderSettings are a map of key-value pairs that should be applied to the
+                  uploader configuration.
+                nullable: true
+                type: object
+              uploaderType:
+                description: UploaderType is the type of the uploader to handle the
+                  data transfer.
+                enum:
+                - kopia
+                - ""
+                type: string
+              volume:
+                description: |-
+                  Volume is the name of the volume within the Pod to be backed
+                  up.
+                type: string
+            required:
+            - backupStorageLocation
+            - node
+            - pod
+            - repoIdentifier
+            - volume
+            type: object
+          status:
+            description: PodVolumeBackupStatus is the current status of a PodVolumeBackup.
+            properties:
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the pod volume backup is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
+              completionTimestamp:
+                description: |-
+                  CompletionTimestamp records the time a backup was completed.
+                  Completion time is recorded even on failed backups.
+                  Completion time is recorded before uploading the backup object.
+                  The server's time is used for CompletionTimestamps
+                format: date-time
+                nullable: true
+                type: string
+              message:
+                description: Message is a message about the pod volume backup's status.
+                type: string
+              path:
+                description: Path is the full path within the controller pod being
+                  backed up.
+                type: string
+              phase:
+                description: Phase is the current state of the PodVolumeBackup.
+                enum:
+                - New
+                - Accepted
+                - Prepared
+                - InProgress
+                - Canceling
+                - Canceled
+                - Completed
+                - Failed
+                type: string
+              progress:
+                description: |-
+                  Progress holds the total number of bytes of the volume and the current
+                  number of backed up bytes. This can be used to display progress information
+                  about the backup operation.
+                properties:
+                  bytesDone:
+                    format: int64
+                    type: integer
+                  totalBytes:
+                    format: int64
+                    type: integer
+                type: object
+              snapshotID:
+                description: SnapshotID is the identifier for the snapshot of the
+                  pod volume.
+                type: string
+              startTimestamp:
+                description: |-
+                  StartTimestamp records the time a backup was started.
+                  Separate from CreationTimestamp, since that value changes
+                  on restores.
+                  The server's time is used for StartTimestamps
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+```
+
+Below is the new PVR CRD:
+```yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: podvolumerestores.velero.io
+spec:
+  group: velero.io
+  names:
+    kind: PodVolumeRestore
+    listKind: PodVolumeRestoreList
+    plural: podvolumerestores
+    singular: podvolumerestore
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: PodVolumeRestore status such as New/InProgress
+      jsonPath: .status.phase
+      name: Status
+      type: string
+    - description: Time duration since this PodVolumeRestore was started
+      jsonPath: .status.startTimestamp
+      name: Started
+      type: date
+    - description: Completed bytes
+      format: int64
+      jsonPath: .status.progress.bytesDone
+      name: Bytes Done
+      type: integer
+    - description: Total bytes
+      format: int64
+      jsonPath: .status.progress.totalBytes
+      name: Total Bytes
+      type: integer
+    - description: Name of the Backup Storage Location where the backup data is stored
+      jsonPath: .spec.backupStorageLocation
+      name: Storage Location
+      type: string
+    - description: Time duration since this PodVolumeRestore was created
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Name of the node where the PodVolumeRestore is processed
+      jsonPath: .status.node
+      name: Node
+      type: string
+    - description: The type of the uploader to handle data transfer
+      jsonPath: .spec.uploaderType
+      name: Uploader Type
+      type: string
+    name: v1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore.
+            properties:
+              backupStorageLocation:
+                description: |-
+                  BackupStorageLocation is the name of the backup storage location
+                  where the backup repository is stored.
+                type: string
+              cancel:
+                description: |-
+                  Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
+                  when the PodVolumeRestore is in InProgress phase
+                type: boolean
+              pod:
+                description: Pod is a reference to the pod containing the volume to
+                  be restored.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: |-
+                      If referring to a piece of an object instead of an entire object, this string
+                      should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                      For example, if the object reference is to a container within a pod, this would take on a value like:
+                      "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                      the event) or if no container name is specified "spec.containers[2]" (container with
+                      index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                      referencing a part of an object.
+                    type: string
+                  kind:
+                    description: |-
+                      Kind of the referent.
+                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                    type: string
+                  name:
+                    description: |-
+                      Name of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                    type: string
+                  namespace:
+                    description: |-
+                      Namespace of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                    type: string
+                  resourceVersion:
+                    description: |-
+                      Specific resourceVersion to which this reference is made, if any.
+                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                    type: string
+                  uid:
+                    description: |-
+                      UID of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                    type: string
+                type: object
+                x-kubernetes-map-type: atomic
+              repoIdentifier:
+                description: RepoIdentifier is the backup repository identifier.
+                type: string
+              snapshotID:
+                description: SnapshotID is the ID of the volume snapshot to be restored.
+                type: string
+              sourceNamespace:
+                description: SourceNamespace is the original namespace for namespace
+                  mapping.
+                type: string
+              uploaderSettings:
+                additionalProperties:
+                  type: string
+                description: |-
+                  UploaderSettings are a map of key-value pairs that should be applied to the
+                  uploader configuration.
+                nullable: true
+                type: object
+              uploaderType:
+                description: UploaderType is the type of the uploader to handle the
+                  data transfer.
+                enum:
+                - kopia
+                - ""
+                type: string
+              volume:
+                description: Volume is the name of the volume within the Pod to be
+                  restored.
+                type: string
+            required:
+            - backupStorageLocation
+            - pod
+            - repoIdentifier
+            - snapshotID
+            - sourceNamespace
+            - volume
+            type: object
+          status:
+            description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore.
+            properties:
+              acceptedTimestamp:
+                description: |-
+                  AcceptedTimestamp records the time the pod volume restore is to be prepared.
+                  The server's time is used for AcceptedTimestamp
+                format: date-time
+                nullable: true
+                type: string
+              completionTimestamp:
+                description: |-
+                  CompletionTimestamp records the time a restore was completed.
+                  Completion time is recorded even on failed restores.
+                  The server's time is used for CompletionTimestamps
+                format: date-time
+                nullable: true
+                type: string
+              message:
+                description: Message is a message about the pod volume restore's status.
+                type: string
+              node:
+                description: Node is name of the node where the pod volume restore
+                  is processed.
+                type: string
+              phase:
+                description: Phase is the current state of the PodVolumeRestore.
+                enum:
+                - New
+                - Accepted
+                - Prepared
+                - InProgress
+                - Canceling
+                - Canceled
+                - Completed
+                - Failed
+                type: string
+              progress:
+                description: |-
+                  Progress holds the total number of bytes of the snapshot and the current
+                  number of restored bytes. This can be used to display progress information
+                  about the restore operation.
+                properties:
+                  bytesDone:
+                    format: int64
+                    type: integer
+                  totalBytes:
+                    format: int64
+                    type: integer
+                type: object
+              startTimestamp:
+                description: |-
+                  StartTimestamp records the time a restore was started.
+                  The server's time is used for StartTimestamps
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+```
+
+## Installation Changes
+No changes to installation, the backupPod/restorePod's configurations are either inherited from node-agent or retrieved from node-agent-configmap.  
+
+## Upgrade
+Upgrade is not impacted.  
+
+## CLI
+CLI is not changed.
+
+
+
+[1]: ../unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
+[2]: ../volume-snapshot-data-movement/volume-snapshot-data-movement.md
+[3]: ../vgdp-micro-service/vgdp-micro-service.md
+[4]: ../node-agent-concurrency.md
+[5]: https://github.com/vmware-tanzu/velero/issues/8955
+[6]: https://github.com/vmware-tanzu/velero/issues/8956
+[7]: https://github.com/vmware-tanzu/velero/issues/8289
+[8]: https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/
+[9]: https://github.com/vmware-tanzu/velero/issues/7725
+[10]: https://github.com/vmware-tanzu/velero/issues/8423

design/Implemented/volume-group-snapshot.md

@@ -0,0 +1,611 @@
+# Add Support for VolumeGroupSnapshots
+
+This proposal outlines the design and implementation plan for incorporating VolumeGroupSnapshot support into Velero. The enhancement will allow Velero to perform consistent, atomic snapshots of groups of Volumes using the new Kubernetes [VolumeGroupSnapshot API](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/). This capability is especially critical for stateful applications that rely on multiple volumes to ensure data consistency, such as databases and analytics workloads.
+
+## Glossary & Abbreviation
+
+Terminology used in this document:
+- VGS: VolumeGroupSnapshot
+- VS: VolumeSnapshot
+- VGSC: VolumeGroupSnapshotContent
+- VSC: VolumeSnapshotContent
+- VGSClass: VolumeGroupSnapshotClass
+- VSClass: VolumeSnapshotClass
+
+## Background
+
+Velero currently enables snapshot-based backups on an individual Volume basis through CSI drivers. However, modern stateful applications often require multiple volumes for data, logs, and backups. This distributed data architecture increases the risk of inconsistencies when volumes are captured individually. Kubernetes has introduced the VolumeGroupSnapshot(VGS) API [(KEP-3476)](https://github.com/kubernetes/enhancements/pull/1551), which allows for the atomic snapshotting of multiple volumes in a coordinated manner. By integrating this feature, Velero can offer enhanced disaster recovery for multi-volume applications, ensuring consistency across all related data.
+
+## Goals
+- Ensure that multiple related volumes are snapshotted simultaneously, preserving consistency for stateful applications via VolumeGroupSnapshots(VGS) API.
+- Integrate VolumeGroupSnapshot functionality into Velero’s existing backup and restore workflows.
+- Allow users to opt in to volume group snapshots via specifying the group label.
+
+## Non-Goals
+- The proposal does not require a complete overhaul of Velero’s CSI integration, it will extend the current mechanism to support group snapshots.
+- No any changes pertaining to execution of Restore Hooks
+
+## High-Level Design
+
+### Backup workflow:
+#### Accept the label to be used for VGS from the user:
+  - Accept the label from the user, we will do this in 3 ways:
+    - Firstly, we will have a hard-coded default label key like `velero.io/volume-group-snapshot` that the users can directly use on their PVCs.
+    - Secondly, we will let the users override this default VGS label via a velero server arg, `--volume-group-nsaphot-label-key`, if needed.
+    - And Finally we will have the option to override the default label via Backup API spec, `backup.spec.volumeGroupSnapshotLabelKey`
+    - In all the instances, the VGS label key will be present on the backup spec, this makes the label key accessible to plugins during the execution of backup operation.
+  - This label will enable velero to filter the PVC to be included in the VGS spec.
+  - Users will have to label the PVCs before invoking the backup operation.
+  - This label would act as a group identifier for the PVCs to be grouped under a specific VGS.
+  - It will be used to collect the PVCs to be used for a particular instance of VGS object.  
+
+**Note:** 
+  - Modifying or adding VGS label on PVCs during an active backup operation may lead to unexpected or undesirable backup results. To avoid inconsistencies, ensure PVC labels remain unchanged throughout the backup execution.
+  - Label Key Precedence: When determining which label key to use for grouping PVCs into a VolumeGroupSnapshot, Velero applies overrides in the following order (highest to lowest):
+    - Backup API spec (`backup.spec.volumeGroupSnapshotLabelKey`)
+    - Server flag (`--volume-group-snapshot-label-key`)
+    - Built-in default (`velero.io/volume-group-snapshot`)
+
+    Whichever key wins this precedence is then injected into the Backup spec so that all Velero plugins can uniformly discover and use it during the backup execution.
+#### Changes to the Existing PVC ItemBlockAction plugin:
+  - Currently the PVC IBA plugin is applied to PVCs and adds the RelatedItems for the particular PVC into the ItemBlock.
+  - At first it checks whether the PVC is bound and VolumeName is non-empty.
+  - Then it adds the related PV under the list of relatedItems.
+  - Following on, the plugin adds the pods mounting the PVC as relatedItems.
+  - Now we need to extend this PVC IBA plugin to add the PVCs to be grouped for a particular VGS object, so that they are processed together under an ItemBlock by Velero.
+      - First we will check if the PVC that is being processed by the plugin has the user specified VGS label.
+      - If it is present then we will execute a List call in the namespace with the label as a matching criteria and see if this results in any PVCs (other than the current one).
+      - If there are PVCs matching the criteria then we add the PVCs to the relatedItems list.
+      - This helps in building the ItemBlock we need for VGS processing, i.e. we have the relevant pods and PVCs in the ItemBlock.
+
+**Note:** The ItemBlock to VGS relationship will not always be 1:1. There might be scenarios when the ItemBlock might have multiple VGS instances associated with it.
+Lets go over some ItemBlock/VGS scenarios that we might encounter and visualize them for clarity:
+1. Pod Mounts: Pod1 mounts both PVC1 and PVC2.  
+   Grouping: PVC1 and PVC2 share the same group label (group: A)  
+   ItemBlock: The item block includes Pod1, PVC1, and PVC2.  
+   VolumeGroupSnapshot (VGS): Because PVC1 and PVC2 are grouped together by their label, they trigger the creation of a single VGS (labeled with group: A).  
+
+```mermaid
+flowchart TD
+   subgraph ItemBlock
+   P1[Pod1]
+   PVC1[PVC1 group: A]
+   PVC2[PVC2 group: A]
+   end
+
+   P1 -->|mounts| PVC1
+   P1 -->|mounts| PVC2
+
+   PVC1 --- PVC2
+
+   PVC1 -- "group: A" --> VGS[VGS group: A]
+   PVC2 -- "group: A" --> VGS
+
+```   
+2. Pod Mounts: Pod1 mounts each of the four PVCs.  
+   Grouping:  
+     Group A: PVC1 and PVC2 share the same grouping label (group: A).  
+     Group B: PVC3 and PVC4 share the grouping label (group: B)   
+   ItemBlock: All objects (Pod1, PVC1, PVC2, PVC3, and PVC4) are collected into a single item block.   
+   VolumeGroupSnapshots:  
+      PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)).      
+      PVC3 and PVC4 (group B) point to a different VGS (VGS (group: B)).    
+```mermaid
+flowchart TD
+    subgraph ItemBlock
+        P1[Pod1]
+        PVC1[PVC1 group: A]
+        PVC2[PVC2 group: A]
+        PVC3[PVC3 group: B]
+        PVC4[PVC4 group: B]
+    end
+
+    %% Pod mounts all PVCs
+    P1 -->|mounts| PVC1
+    P1 -->|mounts| PVC2
+    P1 -->|mounts| PVC3
+    P1 -->|mounts| PVC4
+
+    %% Group A relationships: PVC1 and PVC2
+    PVC1 --- PVC2
+    PVC1 -- "group: A" --> VGS_A[VGS-A group: A]
+    PVC2 -- "group: A" --> VGS_A
+
+    %% Group B relationships: PVC3 and PVC4
+    PVC3 --- PVC4
+    PVC3 -- "group: B" --> VGS_B[VGS-B group: B]
+    PVC4 -- "group: B" --> VGS_B
+```
+
+3. Pod Mounts: Pod1 mounts both PVC1 and PVC2, Pod2 mounts PVC1 and PVC3.  
+   Grouping:   
+     Group A: PVC1 and PVC2  
+     Group B: PVC3  
+   ItemBlock: All objects-Pod1, Pod2, PVC1, PVC2, and PVC3, are collected into a single item block.  
+   VolumeGroupSnapshots:  
+     PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)).   
+     PVC3 (group B) point to a different VGS (VGS (group: B)).  
+```mermaid
+flowchart TD
+    subgraph ItemBlock
+        P1[Pod1]
+        P2[Pod2]
+        PVC1[PVC1 group: A]
+        PVC2[PVC2 group: A]
+        PVC3[PVC3 group: B]
+    end
+
+    %% Pod mount relationships
+    P1 -->|mounts| PVC1
+    P1 -->|mounts| PVC2
+    P2 -->|mounts| PVC1
+    P2 -->|mounts| PVC3
+
+    %% Grouping for Group A: PVC1 and PVC2 are grouped into VGS_A
+    PVC1 --- PVC2
+    PVC1 -- "Group A" --> VGS_A[VGS Group A]
+    PVC2 -- "Group A" --> VGS_A
+
+    %% Grouping for Group B: PVC3 grouped into VGS_B
+    PVC3 -- "Group B" --> VGS_B[VGS Group B]
+    
+```
+
+#### Updates to CSI PVC plugin:
+The CSI PVC plugin now supports obtaining a VolumeSnapshot (VS) reference for a PVC in three ways, and then applies common branching for datamover and non‑datamover workflows:
+
+- Scenario 1: PVC has a VGS label and no VS (created via the VGS workflow) exists for its volume group:
+    - Determine VGSClass: The plugin will pick `VolumeGroupSnapshotClass` by following the same tier based precedence as it does for individual `VolumeSnapshotClasses`:
+      - Default by Label: Use the one VGSClass labeled
+      ```yaml
+      metadata:
+        labels:
+        velero.io/csi-volumegroupsnapshot-class: "true"
+
+      ```
+      whose `spec.driver` matches the CSI driver used by the PVCs.
+      - Backup‑level Override: If the Backup CR has an annotation
+      ```yaml
+      metadata:
+        annotations:
+        velero.io/csi-volumegroupsnapshot-class_<driver>: <className>
+      ```
+      (with <driver> equal to the PVCs’ CSI driver), use that class.
+      - PVC‑level Override: Finally, if the PVC itself carries an annotation
+      ```yaml
+      metadata:
+        annotations:
+        velero.io/csi-volume-group-snapshot-class: <className>
+      ```
+      and that class exists, use it.
+      At each step, if the plugin finds zero or multiple matching classes, VGS creation is skipped and backup fails.
+    - Create VGS: The plugin creates a new VolumeGroupSnapshot (VGS) for the PVC’s volume group. This action automatically triggers creation of the corresponding VGSC, VS, and VSC objects.
+    - Wait for VS Status: The plugin waits until each VS (one per PVC in the group) has its `volumeGroupSnapshotName` populated. This confirms that the snapshot controller has completed its work. `CSISnapshotTimeout` will be used here.
+    - Update VS Objects: Once the VS objects are provisioned, the plugin updates them by removing VGS owner references and VGS-related finalizers, and by adding backup metadata labels (including BackupName, BackupUUID, and PVC name). These labels are later used to detect an existing VS when processing another PVC of the same group.
+    - Patch and Cleanup: The plugin patches the deletionPolicy of the VGSC to "Retain" (ensuring that deletion of the VGSC does not remove the underlying VSC objects or storage snapshots) and then deletes the temporary VGS and VGSC objects.
+        
+- Scenario 2: PVC has a VGS label and a VS created via an earlier VGS workflow already exists:
+    - The plugin lists VS objects in the PVC’s namespace using backup metadata labels (BackupUID, BackupName, and PVCName).
+    - It verifies that at least one VS has a non‑empty `volumeGroupSnapshotName` in its status.
+    - If such a VS exists, the plugin skips creating a new VGS (or VS) and proceeds with the legacy workflow using the existing VS.
+    - If a VS is found but its status does not indicate it was created by the VGS workflow (i.e. its `volumeGroupSnapshotName` is empty), the backup for that PVC is failed, resulting in a partially failed backup.
+- Scenario 3: PVC does not have a VGS label:
+    - The legacy workflow is followed, and an individual VolumeSnapshot (VS) is created for the PVC.
+- Common Branching for Datamover and Non‑datamover Workflows:
+    - Once a VS reference (`vsRef`) is determined—whether through the VGS workflow (Scenario 1 or 2) or the legacy workflow (Scenario 3)—the plugin then applies the common branching:
+        - Non‑datamover Case: The VS reference is directly added as an additional backup item.
+            
+        - Datamover Case: The plugin waits until the VS’s associated VSC snapshot handle is ready (using the configured CSISnapshotTimeout), then creates a DataUpload for the VS–PVC pair. The resulting DataUpload is then added as an additional backup item.
+
+
+```mermaid
+flowchart TD
+    %% Section 1: Accept VGS Label from User
+    subgraph Accept_Label
+      A1[User sets VGS label key using default velero.io/volume-group-snapshot or via server arg or Backup API spec]
+      A2[User labels PVCs before backup]
+      A1 --> A2
+    end
+
+    %% Section 2: PVC ItemBlockAction Plugin Extension
+    subgraph PVC_ItemBlockAction
+      B1[Check PVC is bound and has VolumeName]
+      B2[Add related PV to relatedItems]
+      B3[Add pods mounting PVC to relatedItems]
+      B4[Check if PVC has user-specified VGS label]
+      B5[List PVCs in namespace matching label criteria]
+      B6[Add matching PVCs to relatedItems]
+      B1 --> B2 --> B3 --> B4
+      B4 -- Yes --> B5
+      B5 --> B6
+    end
+
+    %% Section 3: CSI PVC Plugin Updates
+    subgraph CSI_PVC_Plugin
+       C1[For each PVC, check for VGS label]
+       C1 -- Has VGS label --> C2[Determine scenario]
+       C1 -- No VGS label --> C16[Scenario 3: Legacy workflow - create individual VS]
+
+       %% Scenario 1: No existing VS via VGS exists
+       subgraph Scenario1[Scenario 1: No existing VS via VGS]
+         S1[List grouped PVCs using VGS label]
+         S2[Determine CSI driver for grouped PVCs]
+         S3[If single CSI driver then select matching VGSClass; else fail backup]
+         S4[Create new VGS triggering VGSC, VS, and VSC creation]
+         S5[Wait for VS objects to have nonempty volumeGroupSnapshotName]
+         S6[Update VS objects; remove VGS owner refs and finalizers; add backup metadata labels]
+         S7[Patch VGSC deletionPolicy to Retain]
+         S8[Delete transient VGS and VGSC]
+         S1 --> S2 --> S3 --> S4 --> S5 --> S6 --> S7 --> S8
+		 
+       end
+
+       %% Scenario 2: Existing VS via VGS exists
+       subgraph Scenario2[Scenario 2: Existing VS via VGS exists]
+         S9[List VS objects using backup metadata - BackupUID, BackupName, PVCName]
+         S10[Check if any VS has nonempty volumeGroupSnapshotName]
+         S9 --> S10
+         S10 -- Yes --> S11[Use existing VS]
+         S10 -- No --> S12[Fail backup for PVC]
+       end
+
+       C2 -- Scenario1 applies --> S1
+       C2 -- Scenario2 applies --> S9
+
+       %% Common Branch: After obtaining a VS reference
+       subgraph Common_Branch[Common Branch]
+         CB1[Obtain VS reference as vsRef]
+         CB2[If non-datamover, add vsRef as additional backup item]
+         CB3[If datamover, wait for VSC handle and create DataUpload; add DataUpload as additional backup item]
+         CB1 --> CB2
+         CB1 --> CB3
+       end
+       
+       %% Connect Scenario outcomes and legacy branch to the common branch
+       S8 --> CB1
+       S11 --> CB1
+       C16 --> CB1
+    end
+
+    %% Overall Flow Connections
+    A2 --> B1
+    B6 --> C1
+
+```
+
+
+Restore workflow:
+
+- No changes required for the restore workflow.
+
+## Detailed Design
+
+Backup workflow:
+- Accept the label to be used for VGS from the user as a server argument:
+    - Set a default VGS label key to be used:
+    ```go
+    // default VolumeGroupSnapshot Label
+	defaultVGSLabelKey = "velero.io/volume-group-snapshot"
+    
+    ```
+    - Add this as a server flag and pass it to backup reconciler, so that we can use it during the backup request execution.
+    ```go
+    flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot")
+    ```
+
+    - Update the Backup CRD to accept the VGS Label Key as a spec value:
+    ```go
+    // VolumeGroupSnapshotLabelKey specifies the label key to be used for grouping the PVCs under
+	// an instance of VolumeGroupSnapshot, if left unspecified velero.io/volume-group-snapshot is used
+	// +optional
+	VolumeGroupSnapshotLabelKey string `json:"volumeGroupSnapshotLabelKey,omitempty"`
+    ```
+    - Modify the [`prepareBackupRequest` function](https://github.com/openshift/velero/blob/8c8a6cccd78b78bd797e40189b0b9bee46a97f9e/pkg/controller/backup_controller.go#L327) to set the default label key as a backup spec if the user does not specify any value:
+    ```go
+    if len(request.Spec.VolumeGroupSnapshotLabelKey) == 0 {
+		// set the default key value
+		request.Spec.VolumeGroupSnapshotLabelKey = b.defaultVGSLabelKey
+	}
+    ```
+
+- Changes to the Existing [PVC ItemBlockAction plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/itemblock/actions/pvc_action.go#L64) (Update the GetRelatedItems function):
+```go
+// Retrieve the VGS label key from the Backup spec.
+	vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey
+	if vgsLabelKey != "" {
+		// Check if the PVC has the specified VGS label.
+		if groupID, ok := pvc.Labels[vgsLabelKey]; ok {
+			// List all PVCs in the namespace with the same label key and value (i.e. same group).
+			pvcList := new(corev1api.PersistentVolumeClaimList)
+			if err := a.crClient.List(context.Background(), pvcList, crclient.InNamespace(pvc.Namespace), crclient.MatchingLabels{vgsLabelKey: groupID}); err != nil {
+				return nil, errors.Wrap(err, "failed to list PVCs for VGS grouping")
+			}
+			// Add each matching PVC (except the current one) to the relatedItems.
+			for _, groupPVC := range pvcList.Items {
+				if groupPVC.Name == pvc.Name {
+					continue
+				}
+				a.log.Infof("Adding grouped PVC %s to relatedItems for PVC %s", groupPVC.Name, pvc.Name)
+				relatedItems = append(relatedItems, velero.ResourceIdentifier{
+					GroupResource: kuberesource.PersistentVolumeClaims,
+					Namespace:     groupPVC.Namespace,
+					Name:          groupPVC.Name,
+				})
+			}
+		}
+	} else {
+		a.log.Info("No VolumeGroupSnapshotLabelKey provided in backup spec; skipping PVC grouping")
+	}
+```
+
+- Updates to [CSI PVC plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/backup/actions/csi/pvc_action.go#L200) (Update the Execute method):
+```go
+func (p *pvcBackupItemAction) Execute(
+    item runtime.Unstructured,
+    backup *velerov1api.Backup,
+) (
+    runtime.Unstructured,
+    []velero.ResourceIdentifier,
+    string,
+    []velero.ResourceIdentifier,
+    error,
+) {
+    p.log.Info("Starting PVCBackupItemAction")
+
+    // Validate backup policy and PVC/PV
+    if valid := p.validateBackup(*backup); !valid {
+        return item, nil, "", nil, nil
+    }
+
+    var pvc corev1api.PersistentVolumeClaim
+    if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil {
+        return nil, nil, "", nil, errors.WithStack(err)
+    }
+    if valid, item, err := p.validatePVCandPV(pvc, item); !valid {
+        if err != nil {
+            return nil, nil, "", nil, err
+        }
+        return item, nil, "", nil, nil
+    }
+
+    shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithBackup(
+        item,
+        kuberesource.PersistentVolumeClaims,
+        *backup,
+        p.crClient,
+        p.log,
+    )
+    if err != nil {
+        return nil, nil, "", nil, err
+    }
+    if !shouldSnapshot {
+        p.log.Debugf("CSI plugin skip snapshot for PVC %s according to VolumeHelper setting", pvc.Namespace+"/"+pvc.Name)
+        return nil, nil, "", nil, nil
+    }
+
+    var additionalItems []velero.ResourceIdentifier
+    var operationID string
+    var itemToUpdate []velero.ResourceIdentifier
+
+    // vsRef will be our common reference to the VolumeSnapshot (VS)
+    var vsRef *corev1api.ObjectReference
+
+    // Retrieve the VGS label key from the backup spec.
+    vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey
+
+    // Check if the PVC has the user-specified VGS label.
+    if group, ok := pvc.Labels[vgsLabelKey]; ok && group != "" {
+        p.log.Infof("PVC %s has VGS label with group %s", pvc.Name, group)
+        // --- VGS branch ---
+        // 1. Check if a VS created via a VGS workflow exists for this PVC.
+        existingVS, err := p.findExistingVSForBackup(backup.UID, backup.Name, pvc.Name, pvc.Namespace)
+        if err != nil {
+            return nil, nil, "", nil, err
+        }
+        if existingVS != nil && existingVS.Status.VolumeGroupSnapshotName != "" {
+            p.log.Infof("Existing VS %s found for PVC %s in group %s; skipping VGS creation", existingVS.Name, pvc.Name, group)
+            vsRef = &corev1api.ObjectReference{
+                Namespace: existingVS.Namespace,
+                Name:      existingVS.Name,
+            }
+        } else {
+            // 2. No existing VS via VGS; execute VGS creation workflow.
+            groupedPVCs, err := p.listGroupedPVCs(backup, pvc.Namespace, vgsLabelKey, group)
+            if err != nil {
+                return nil, nil, "", nil, err
+            }
+            pvcNames := extractPVCNames(groupedPVCs)
+            // Determine the CSI driver used by the grouped PVCs.
+            driver, err := p.determineCSIDriver(groupedPVCs)
+            if err != nil {
+                return nil, nil, "", nil, errors.Wrap(err, "failed to determine CSI driver for grouped PVCs")
+            }
+            if driver == "" {
+                return nil, nil, "", nil, errors.New("multiple CSI drivers found for grouped PVCs; failing backup")
+            }
+            // Retrieve the appropriate VGSClass for the CSI driver.
+            vgsClass := p.getVGSClassForDriver(driver)
+            p.log.Infof("Determined CSI driver %s with VGSClass %s for PVC group %s", driver, vgsClass, group)
+
+            newVGS, err := p.createVolumeGroupSnapshot(backup, pvc, pvcNames, vgsLabelKey, group, vgsClass)
+            if err != nil {
+                return nil, nil, "", nil, err
+            }
+            p.log.Infof("Created new VGS %s for PVC group %s", newVGS.Name, group)
+            
+            // Wait for the VS objects created via VGS to have volumeGroupSnapshotName in status.
+            if err := p.waitForVGSAssociatedVS(newVGS, pvc.Namespace, backup.Spec.CSISnapshotTimeout.Duration); err != nil {
+                return nil, nil, "", nil, err
+            }
+            // Update the VS objects: remove VGS owner references and finalizers; add backup metadata labels.
+            if err := p.updateVGSCreatedVS(newVGS, backup); err != nil {
+                return nil, nil, "", nil, err
+            }
+            // Patch the VGSC deletionPolicy to Retain.
+            if err := p.patchVGSCDeletionPolicy(newVGS, pvc.Namespace); err != nil {
+                return nil, nil, "", nil, err
+            }
+            // Delete the VGS and VGSC
+            if err := p.deleteVGSAndVGSC(newVGS, pvc.Namespace); err != nil {
+                return nil, nil, "", nil, err
+            }
+            // Fetch the VS that was created for this PVC via VGS.
+            vs, err := p.getVSForPVC(backup, pvc, vgsLabelKey, group)
+            if err != nil {
+                return nil, nil, "", nil, err
+            }
+            vsRef = &corev1api.ObjectReference{
+                Namespace: vs.Namespace,
+                Name:      vs.Name,
+            }
+        }
+    } else {
+        // Legacy workflow: PVC does not have a VGS label; create an individual VS.
+        vs, err := p.createVolumeSnapshot(pvc, backup)
+        if err != nil {
+            return nil, nil, "", nil, err
+        }
+        vsRef = &corev1api.ObjectReference{
+            Namespace: vs.Namespace,
+            Name:      vs.Name,
+        }
+    }
+
+    // --- Common Branch ---
+    // Now we have vsRef populated from one of the above cases.
+    // Branch further based on backup.Spec.SnapshotMoveData.
+    if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
+        // Datamover case:
+        operationID = label.GetValidName(
+            string(velerov1api.AsyncOperationIDPrefixDataUpload) + string(backup.UID) + "." + string(pvc.UID),
+        )
+        dataUploadLog := p.log.WithFields(logrus.Fields{
+            "Source PVC":     fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name),
+            "VolumeSnapshot": fmt.Sprintf("%s/%s", vsRef.Namespace, vsRef.Name),
+            "Operation ID":   operationID,
+            "Backup":         backup.Name,
+        })
+        // Retrieve the current VS using vsRef
+        vs := &snapshotv1api.VolumeSnapshot{}
+        if err := p.crClient.Get(context.TODO(), crclient.ObjectKey{Namespace: vsRef.Namespace, Name: vsRef.Name}, vs); err != nil {
+            return nil, nil, "", nil, errors.Wrapf(err, "failed to get VolumeSnapshot %s", vsRef.Name)
+        }
+        // Wait until the VS-associated VSC snapshot handle is ready.
+        _, err := csi.WaitUntilVSCHandleIsReady(
+            vs,
+            p.crClient,
+            p.log,
+            true,
+            backup.Spec.CSISnapshotTimeout.Duration,
+        )
+        if err != nil {
+            dataUploadLog.Errorf("Failed to wait for VolumeSnapshot to become ReadyToUse: %s", err.Error())
+            csi.CleanupVolumeSnapshot(vs, p.crClient, p.log)
+            return nil, nil, "", nil, errors.WithStack(err)
+        }
+        dataUploadLog.Info("Starting data upload of backup")
+        dataUpload, err := createDataUpload(
+            context.Background(),
+            backup,
+            p.crClient,
+            vs,
+            &pvc,
+            operationID,
+        )
+        if err != nil {
+            dataUploadLog.WithError(err).Error("Failed to submit DataUpload")
+            if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil && !apierrors.IsNotFound(deleteErr) {
+                dataUploadLog.WithError(deleteErr).Error("Failed to delete VolumeSnapshot")
+            }
+            return item, nil, "", nil, nil
+        }
+        dataUploadLog.Info("DataUpload submitted successfully")
+        itemToUpdate = []velero.ResourceIdentifier{
+            {
+                GroupResource: schema.GroupResource{
+                    Group:    "velero.io",
+                    Resource: "datauploads",
+                },
+                Namespace: dataUpload.Namespace,
+                Name:      dataUpload.Name,
+            },
+        }
+        annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name
+        // For the datamover case, add the dataUpload as an additional item directly.
+        vsRef = &corev1api.ObjectReference{
+            Namespace: dataUpload.Namespace,
+            Name:      dataUpload.Name,
+        }
+        additionalItems = append(additionalItems, velero.ResourceIdentifier{
+            GroupResource: schema.GroupResource{
+                Group:    "velero.io",
+                Resource: "datauploads",
+            },
+            Namespace: dataUpload.Namespace,
+            Name:      dataUpload.Name,
+        })
+    } else {
+        // Non-datamover case:
+        // Use vsRef for snapshot purposes.
+        additionalItems = append(additionalItems, convertVSToResourceIdentifiersFromRef(vsRef)...)
+        p.log.Infof("VolumeSnapshot additional item added for VS %s", vsRef.Name)
+    }
+
+    // Update PVC metadata with common labels and annotations.
+    labels := map[string]string{
+        velerov1api.VolumeSnapshotLabel: vsRef.Name,
+        velerov1api.BackupNameLabel:     backup.Name,
+    }
+    annotations := map[string]string{
+        velerov1api.VolumeSnapshotLabel:                 vsRef.Name,
+        velerov1api.MustIncludeAdditionalItemAnnotation: "true",
+    }
+    kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations)
+    kubeutil.AddLabels(&pvc.ObjectMeta, labels)
+
+    p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", len(additionalItems))
+    for _, ai := range additionalItems {
+        p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name)
+    }
+
+    pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc)
+    if err != nil {
+        return nil, nil, "", nil, errors.WithStack(err)
+    }
+
+    return &unstructured.Unstructured{Object: pvcMap},
+        additionalItems, operationID, itemToUpdate, nil
+}
+
+
+```
+
+## Implementation
+
+This design proposal is targeted for velero 1.16.
+
+The implementation of this proposed design is targeted for velero 1.17.
+
+**Note:**
+- VGS support isn't a requirement on restore. The design does not have any VGS related elements/considerations in the restore workflow.
+
+## Requirements and Assumptions
+- Kubernetes Version:
+  - Minimum: v1.32.0 or later, since the VolumeGroupSnapshot API goes beta in 1.32.
+  - Assumption: CRDs for `VolumeGroupSnapshot`, `VolumeGroupSnapshotClass`, and `VolumeGroupSnapshotContent` are already installed.
+
+- VolumeGroupSnapshot API Availability:
+  - If the VGS API group (`groupsnapshot.storage.k8s.io/v1beta1`) is not present, Velero backup will fail.
+
+- CSI Driver Compatibility
+  - Only CSI drivers that implement the VolumeGroupSnapshot admission and controller support this feature.
+  - Upon VGS creation, we assume the driver will atomically snapshot all matching PVCs; if it does not, the plugin may time out.
+
+## Performance Considerations
+- Use VGS if you have many similar volumes that must be snapped together and you want to minimize API/server load.
+- Use individual VS if you have only a few volumes, or want one‐volume failures to be isolated.
+
+## Testing Strategy
+
+- Unit tests: We will add targeted unit tests to cover all new code paths—including existing-VS detection, VGS creation, legacy VS fallback, and error scenarios.
+- E2E tests: For E2E we would need, a Kind cluster with a CSI driver that supports group snapshots, deploy an application with multiple PVCs, execute a Velero backup and restore, and verify that VGS is created, all underlying VS objects reach ReadyToUse, and every PVC is restored successfully.

design/Implemented/volume-policy-label-selector-criteria.md

@@ -0,0 +1,202 @@
+# Add Label Selector as a criteria for Volume Policy
+
+## Abstract
+Velero’s volume policies currently support several criteria (such as capacity, storage class, and volume source type) to select volumes for backup. This update extends the design by allowing users to specify required labels on the associated PersistentVolumeClaim (PVC) via a simple key/value map. At runtime, Velero looks up the PVC (when a PV has a ClaimRef), extracts its labels, and compares them with the user-specified map. If all key/value pairs match, the volume qualifies for backup.
+
+## Background 
+PersistentVolumes (PVs) in Kubernetes are typically bound to PersistentVolumeClaims (PVCs) that include labels (for example, indicating environment, application, or region). Basing backup policies on these PVC labels enables more precise control over which volumes are processed.
+
+## Goals
+- Allow users to specify a simple key/value mapping in the volume policy YAML so that only volumes whose associated PVCs contain those labels are selected.
+- Support policies that target volumes based on criteria such as environment=production or region=us-west.
+
+## Non-Goals
+- No changes will be made to the actions (skip, snapshot, fs-backup) of the volume policy engine. This update focuses solely on how volumes are selected.
+- The design does not support other label selector operations (e.g., NotIn, Exists, DoesNotExist) and only allows for exact key/value matching.
+
+## Use-cases/scenarios
+1. Environment-Specific Backup:
+- A user wishes to back up only those volumes whose associated PVCs have labels such as `environment=production` and `app=database`.
+- The volume policy specifies a pvcLabels map with those key/value pairs; only volumes whose PVCs match are processed.
+```yaml
+volumePolicies:
+  - conditions:
+      pvcLabels:
+        environment: production
+        app: database
+    action:
+      type: snapshot
+```
+2. Region-Specific Backup:
+- A user operating in multiple regions wants to back up only volumes in the `us-west` region.
+- The policy includes `pvcLabels: { region: us-west }`, so only PVs bound to PVCs with that label are selected.
+```yaml
+volumePolicies:
+  - conditions:
+      pvcLabels:
+        region: us-west
+    action:
+      type: snapshot
+```
+3. Automated Label-Based Backups:
+- An external system automatically labels new PVCs (for example, `backup: true`).
+- A volume policy with `pvcLabels: { backup: true }` ensures that any new volume whose PVC contains that label is included in backup operations.
+```yaml
+version: v1
+volumePolicies:
+  - conditions:
+      pvcLabels:
+        backup: true
+    action:
+      type: snapshot
+```
+## High-Level Design
+
+1. Extend Volume Policy Schema:
+- The YAML schema for volume conditions is extended to include an optional field pvcLabels of type `map[string]string`.
+2. Implement New Condition Type:
+- A new condition, `pvcLabelsCondition`, is created. It implements the `volumeCondition` interface and simply compares the user-specified key/value pairs with the actual PVC labels (populated at runtime).
+3. Update Structured Volume:
+- The internal representation of a volume (`structuredVolume`) is extended with a new field `pvcLabels map[string]string` to store the labels from the associated PVC.
+- A new helper function (or an updated parsing function) is used to perform a PVC lookup when a PV has a ClaimRef, populating the pvcLabels field.
+4. Integrate with Policy Engine:
+- The policy builder is updated to create and add a `pvcLabelsCondition` if the policy YAML contains a `pvcLabels` entry.
+- The matching entry point uses the updated `structuredVolume` (populated with PVC labels) to evaluate all conditions, including the new PVC labels condition.
+## Detailed Design
+
+1. Update Volume Conditions Schema: Define the conditions struct with a simple map for PVC labels:
+```go
+// volumeConditions defines the current format of conditions we parse.
+type volumeConditions struct {
+    Capacity     string            `yaml:"capacity,omitempty"`
+    StorageClass []string          `yaml:"storageClass,omitempty"`
+    NFS          *nFSVolumeSource  `yaml:"nfs,omitempty"`
+    CSI          *csiVolumeSource  `yaml:"csi,omitempty"`
+    VolumeTypes  []SupportedVolume `yaml:"volumeTypes,omitempty"`
+    // New field: pvcLabels for simple exact-match filtering.
+    PVCLabels map[string]string `yaml:"pvcLabels,omitempty"`
+}
+```
+2. New Condition: `pvcLabelsCondition`: Implement a condition that compares expected labels with those on the PVC:
+```go
+// pvcLabelsCondition defines a condition that matches if the PVC's labels contain all the specified key/value pairs.
+type pvcLabelsCondition struct {
+    labels map[string]string
+}
+
+func (c *pvcLabelsCondition) match(v *structuredVolume) bool {
+    if len(c.labels) == 0 {
+        return true // No label condition specified; always match.
+    }
+    if v.pvcLabels == nil {
+        return false // No PVC labels found.
+    }
+    for key, expectedVal := range c.labels {
+        if actualVal, exists := v.pvcLabels[key]; !exists || actualVal != expectedVal {
+            return false
+        }
+    }
+    return true
+}
+
+func (c *pvcLabelsCondition) validate() error {
+    // No extra validation needed for a simple map.
+    return nil
+}
+```
+3. Update `structuredVolume`: Extend the internal volume representation with a field for PVC labels:
+```go
+// structuredVolume represents a volume with parsed fields.
+type structuredVolume struct {
+    capacity     resource.Quantity
+    storageClass string
+    // New field: pvcLabels stores labels from the associated PVC.
+    pvcLabels    map[string]string
+    nfs          *nFSVolumeSource
+    csi          *csiVolumeSource
+    volumeType   SupportedVolume
+}
+```
+4. Update PVC Lookup – `parsePVWithPVC`: Modify the PV parsing function to perform a PVC lookup:
+```go
+func (s *structuredVolume) parsePVWithPVC(pv *corev1.PersistentVolume, client crclient.Client) error {
+    s.capacity = *pv.Spec.Capacity.Storage()
+    s.storageClass = pv.Spec.StorageClassName
+
+    if pv.Spec.NFS != nil {
+        s.nfs = &nFSVolumeSource{
+            Server: pv.Spec.NFS.Server,
+            Path:   pv.Spec.NFS.Path,
+        }
+    }
+    if pv.Spec.CSI != nil {
+        s.csi = &csiVolumeSource{
+            Driver:           pv.Spec.CSI.Driver,
+            VolumeAttributes: pv.Spec.CSI.VolumeAttributes,
+        }
+    }
+    s.volumeType = getVolumeTypeFromPV(pv)
+
+    // If the PV is bound to a PVC, look it up and store its labels.
+    if pv.Spec.ClaimRef != nil {
+        pvc := &corev1.PersistentVolumeClaim{}
+        err := client.Get(context.Background(), crclient.ObjectKey{
+            Namespace: pv.Spec.ClaimRef.Namespace,
+            Name:      pv.Spec.ClaimRef.Name,
+        }, pvc)
+        if err != nil {
+            return errors.Wrap(err, "failed to get PVC for PV")
+        }
+        s.pvcLabels = pvc.Labels
+    }
+    return nil
+}
+```
+5. Update the Policy Builder: Add the new condition to the policy if pvcLabels is provided:
+```go
+func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error {
+    for _, vp := range resPolicies.VolumePolicies {
+        con, err := unmarshalVolConditions(vp.Conditions)
+        if err != nil {
+            return errors.WithStack(err)
+        }
+        volCap, err := parseCapacity(con.Capacity)
+        if err != nil {
+            return errors.WithStack(err)
+        }
+        var volP volPolicy
+        volP.action = vp.Action
+        volP.conditions = append(volP.conditions, &capacityCondition{capacity: *volCap})
+        volP.conditions = append(volP.conditions, &storageClassCondition{storageClass: con.StorageClass})
+        volP.conditions = append(volP.conditions, &nfsCondition{nfs: con.NFS})
+        volP.conditions = append(volP.conditions, &csiCondition{csi: con.CSI})
+        volP.conditions = append(volP.conditions, &volumeTypeCondition{volumeTypes: con.VolumeTypes})
+        // If a pvcLabels map is provided, add the pvcLabelsCondition.
+        if con.PVCLabels != nil && len(con.PVCLabels) > 0 {
+            volP.conditions = append(volP.conditions, &pvcLabelsCondition{labels: con.PVCLabels})
+        }
+        p.volumePolicies = append(p.volumePolicies, volP)
+    }
+    p.version = resPolicies.Version
+    return nil
+}
+```
+6. Update the Matching Entry Point: Use the updated PV parsing that performs a PVC lookup:
+```go
+func (p *Policies) GetMatchAction(res interface{}, client crclient.Client) (*Action, error) {
+    volume := &structuredVolume{}
+    switch obj := res.(type) {
+    case *corev1.PersistentVolume:
+        if err := volume.parsePVWithPVC(obj, client); err != nil {
+            return nil, errors.Wrap(err, "failed to parse PV with PVC lookup")
+        }
+    case *corev1.Volume:
+        volume.parsePodVolume(obj)
+    default:
+        return nil, errors.New("failed to convert object")
+    }
+    return p.match(volume), nil
+}
+```
+
+Note: The matching loop (p.match(volume)) iterates over all conditions (including our new pvcLabelsCondition) and returns the corresponding action if all conditions match.

design/concurrent-backup-processing.md

@@ -0,0 +1,257 @@
+# Concurrent Backup Processing
+
+This enhancement will enable Velero to process multiple backups at the same time. This is largely a usability enhancement rather than a performance enhancement, since the overall backup throughput may not be significantly improved over the current implementation, since we are already processing individual backup items in parallel. It is a significant usability improvement, though, as with the current design, a user who submits a small backup may have to wait significantly longer than expected if the backup is submitted immediately after a large backup.
+
+## Background
+
+With the current implementation, only one backup may be `InProgress` at a time. A second backup created will not start processing until the first backup moves on to `WaitingForPluginOperations` or `Finalizing`. This is a usability concern, especially in clusters when multiple users are initiating backups. With this enhancement, we intend to allow multiple backups to be processed concurrently. This will allow backups to start processing immediately, even if a large backup was just submitted by another user. This enhancement will build on top of the prior parallel item processing feature by creating a dedicatede ItemBlock worker pool for each running backup. The pool will be created at the beginning of the backup reconcile, and the input channel will be passed to the Kubernetes backupper just like it is in the current release.
+
+The primary challenge is to make sure that the same workload in multiple backups is not backed up concurrently. If that were to happen, we would risk data corruption, especially around the processing of pod hooks and volume backup. For this first release we will take a conservative, high-level approach to overlap detection. Two backups will not run concurrently if there is any overlap in included namespaces. For example, if a backup that includes `ns1` and `ns2` is running, then a second backup for `ns2` and `ns3` will not be started. If a backup which does not filter namespaces is running (either a whole cluster backup or a non-namespace-limited backup with a label selector) then no other backups will be started, since a backup across all namespaces overlaps with any other backup. Calculating item-level overlap for queued backups is problematic since we don't know which items are included in a backup until backup processing has begun. A future release may add ItemBlock overlap detection, where at the item block worker level, the same item will not be processed by two different workers at the same time. This works together with workload conflict detection to further detect conflicts in a more granular level for shared resources between backups. Eventually, with a more complete understanding of individual workloads (either via ItemBlocks or some higher level model), the namespace level overlap detection may be relaxed in future versions.
+
+## Goals
+- Process multiple backups concurrently
+- Detect namespace overlap to avoid conflicts
+- For queued backups (not yet runnable due to concurrency limits or overlap), indicate the queue position in status
+
+## Non Goals
+- Handling NFS PVs when more than one PV point to the same underlying NFS share
+- Handling VGDP cancellation for failed backups on restart
+- Mounting a PVC for scenarios in which /tmp is too small for the number of concurrent backups
+- Providing a mechanism to identify high priority backups which get preferential treatment in terms of ItemBlock worker availability
+- Item-level overlap detection (future feature)
+- Providing the ability to disable namespace-level overlap detection once Item-level overlap detection is in place (although this may be supported in a future version).
+
+## High-Level Design
+
+### Backup CRD changes
+
+Two new backup phases will be added: `Queued` and `ReadyToStart`. In the Backup workflow, new backups will be moved to the Queued phase when they are added to the backup queue. When a backup is removed from the queue because it is now able to run, it will be moved to the `ReadyToStart` phase, which will allow the backup controller to start processing it.
+
+In addition, a new Status field, `QueuePosition`, will be added to track the backup's current position in the queue.
+
+### New Controller: `backupQueueReconciler`
+
+A new reconciler will be added, `backupQueueReconciler` which will use the current `backupReconciler` logic for reconciling `New` backups but instead of running the backup, it will move the Backup to the `Queued` phase and set `QueuePosition`.
+
+In addition, this reconciler will periodically reconcile all queued backups (on some configurable time interval) and if there is a runnable backup, remove it from the queue, update `QueuePosition` for any queued backups behind it, and update its phase to `ReadyToStart`.
+
+Queued backups will be reconciled in order based on `QueuePosition`, so the first runnable backup found will be processed. A backup is runnable if both of the following conditions are true:
+1) The total number of backups either `InProgress` or `ReadyToStart` is less than the configured number of concurrent backups.
+2) The backup has no overlap with any backups currently `InProgress` or `ReadyToStart` or with any `Queued` backups with a higher (i.e. closer to 1) queue position than this backup.
+
+### Updates to Backup controller
+
+The current `backupReconciler` will change its reconciling rules. Instead of watching and reconciling New backups, it will reconcile `ReadyToStart` backups. In addition, it will be configured to run in parallel by setting `MaxConcurrentReconciles` based on the `concurrent-backups` server arg.
+
+The startup (and shutdown) of the ItemBlock worker pool will be moved from reconciler startup to the backup reconcile, which will give each running backup its own dedicated worker pool. The per-backup worker pool will will use the existing `--item-block-worker-count` installer/server arg. This means that the maximum number of ItemBlock workers for the entire Velero pod will be the ItemBlock worker count multiplied by concurrentBackups. For example, if concurrentBackups is 5, and itemBlockWorkerCount is 6, then there will be, at most, 30 worker threads active, 5 dedicated to each InProgress backup, but this maximum will only be achieved when the maximum number of backups are InProgress. This also means that each InProgress backup will have a dedicated ItemBlock input channel with the same fixed buffer size.
+
+## Detailed Design
+
+### New Install/Server configuration args
+
+A new install/server arg, `concurrent-backups` will be added. This will be an int-valued field specifying the number of backups which may be processed concurrently (with phase `InProgress`). If not specified, the default value of 1 will be used.
+
+### Consideration of backup overlap and concurrent backup processing
+
+The primary consideration for running additional backups concurrently is the configured `concurrent-backups` parameter. If the total number of `InProgress` and `ReadyToStart` backups is equal to `concurrent-backups` then any `Queued` backups will remain in the queue.
+
+The second consideration is backup overlap. In order to prevent interaction between running backups (particularly around volume backup and pod hooks), we cannot allow two overlapping backups to run at the same time. For now, we will define overlap broadly -- requiring that two concurrent backups don't include any of the same namespaces. A backup for `ns1` can run concurrently with a backup for `ns2`, but a backup for `[ns1,ns2]` cannot run concurrently with a backup for `ns1`. One consequence of this approach is that a backup which includes all namespaces (even if further filtered by resource or label) cannot run concurrently with *any other backup*.
+
+When determining which queued backup to run next, velero will look for the next queued backup which has no overlap with any InProgress backup or any Queued backup ahead of it. The reason we need to consider queued as well as running backups for overlap detection is as follows.
+
+Consider the following scenario. These are the current not-completed backups (ordered from oldest to newest)
+1. backup1, includedNamespaces: [ns1, ns2], phase: InProgress
+2. backup2, includedNamespaces: [ns2, ns3, ns5], phase: Queued, QueuePosition: 1
+3. backup3, includedNamespaces: [ns4, ns3], phase: Queued, QueuePosition: 2
+4. backup4, includedNamespaces: [ns5, ns6], phase: Queued, QueuePosition: 2
+5. backup5, includedNamespaces: [ns8, ns9], phase: Queued, QueuePosition: 3
+
+Assuming `concurrent-backups` is 2, on the next reconcile, Velero will be able to start a second backup if there is one with no overlap. `backup2` cannot run, since `ns2` overlaps between it and the running `backup1`. If we only considered running overlap (and not queued overlap), then `backup3` could run now. It conflicts with the queued `backup2` on `ns3` but it does not conflict with the running backup. However, if it runs now, then when `backup1` completes, then `backup2` still can't run (since it now overlaps with running `backup3`on `ns3`), so `backup4` starts instead. Now when `backup3` completes, `backup2` still can't run (since it now conflicts with `backup4` on `ns5`). This means that even though it was the second backup created, it's the fourth to run -- providing worse time to completion than without parallel backups. If a queued backup has a large number of namespaces (a full-cluster backup for example), it would never run as long as new single-namespace backups keep being added to the queue.
+
+To resolve this problem we consider both running backups as well as backups ahead in the queue when resolving overlap conflicts. In the above scenario, `backup2` can't run yet since it overlaps with the running backup on `ns2`. In addition, `backup3` and `backup4` also can't run yet since they overlap with queued `backup2`. Therefore, `backup5` will run now. Once `backup1` completes, `backup2` will be free to run.
+
+### Backup CRD changes
+
+New Backup phases:
+```go
+const (
+	// BackupPhaseQueued means the backup has been added to the
+	// queue by the BackupQueueReconciler.
+	BackupPhaseQueued BackupPhase = "Queued"
+
+	// BackupPhaseReadyToStart means the backup has been removed from the
+	// queue by the BackupQueueReconciler and is ready to start.
+	BackupPhaseReadyToStart BackupPhase = "ReadyToStart"
+)
+```
+
+In addition, a new Status field, `queuePosition`, will be added to track the backup's current position in the queue.
+```go
+	// QueuePosition is the position held by the backup in the queue.
+	// QueuePosition=1 means this backup is the next to be considered.
+	// Only relevant when Phase is "Queued"
+	// +optional
+	QueuePosition int `json:"queuePosition,omitempty"`
+```
+
+### New Controller: `backupQueueReconciler`
+
+A new reconciler will be added, `backupQueueReconciler` which will reconcile backups under these conditions:
+1) Watching Create/Update for backups in `New` (or empty) phase
+2) Watching for Backup phase transition from `InProgress` to something else to reconcile all `Queued` backups
+2) Watching for Backup phase transition from `New` (or empty) to `Queued` to reconcile all `Queued` backups
+2) Periodic reconcile of `Queued` backups to handle backups queued at server startup as well as to make sure we never have a situation where backups are queued indefinitely because of a race condition or was otherwise missed in the reconcile on prior backup completion.
+
+The reconciler will be set up as follows -- note that New backups are reconciled on Create/Update, while Queued backups are reconciled when an InProgress backup moves on to another state or when a new backup moves to the Queued state. We also reconcile Queued backups periodically to handle the case of a Velero pod restart with Queued backups, as well as to handle possible edge cases where a queued backup doesn't get moved out of the queue at the point of backup completion or an error occurs during a prior Queued backup reconcile.
+
+```go
+func (c *backupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error {
+        // only consider Queued backups, order by QueuePosition
+	gp := kube.NewGenericEventPredicate(func(object client.Object) bool {
+		backup := object.(*velerov1api.Backup)
+		return (backup.Status.Phase == velerov1api.BackupPhaseQueued)
+	})
+	s := kube.NewPeriodicalEnqueueSource(c.logger.WithField("controller", constant.ControllerBackupOperations), mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{
+		Predicates: []predicate.Predicate{gp},
+		OrderFunc: queuePositionOrderFunc,
+	})
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{
+				UpdateFunc: func(ue event.UpdateEvent) bool {
+					backup := ue.ObjectNew.(*velerov1api.Backup)
+					return backup.Status.Phase == "" || backup.status.Phase == velerov1api.BackupPhaseNew
+				},
+				CreateFunc: func(event.CreateEvent) bool {
+					return backup.Status.Phase == "" || backup.status.Phase == velerov1api.BackupPhaseNew
+				},
+				DeleteFunc: func(de event.DeleteEvent) bool {
+					return false
+				},
+				GenericFunc: func(ge event.GenericEvent) bool {
+					return false
+				},
+		})).
+		Watch(
+			&source.Kind{Type: &velerov1api.Backup{}},
+			&handler.EnqueueRequestsFromMapFunc{
+				ToRequests: handler.ToRequestsFunc(func(a handler.MapObject) []reconcile.Request {
+					backupList := velerov1api.BackupList{}
+					if err := p.List(ctx, backupList); err != nil {
+						p.logger.WithError(err).Error("error listing backups")
+						return
+					}
+					requests = []reconcile.request{}
+					// filter backup list by Phase=queued
+					// sort backup list by queuePosition
+					return requests
+				}),
+		},
+		builder.WithPredicates(predicate.Funcs{
+				UpdateFunc: func(ue event.UpdateEvent) bool {
+					oldBackup := ue.ObjectOld.(*velerov1api.Backup)
+					newBackup := ue.ObjectNew.(*velerov1api.Backup)
+					return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress &&
+						newBackup.Status.Phase != velerov1api.BackupPhaseInProgress ||
+						oldBackup.Status.Phase != velerov1api.BackupPhaseQueued &&
+						newBackup.Status.Phase == velerov1api.BackupPhaseQueued
+				},
+				CreateFunc: func(event.CreateEvent) bool {
+					return false
+				},
+				DeleteFunc: func(de event.DeleteEvent) bool {
+					return false
+				},
+				GenericFunc: func(ge event.GenericEvent) bool {
+					return false
+				},
+		}).
+		WatchesRawSource(s).
+		Named(constant.ControllerBackupQueue).
+		Complete(c)
+}
+```
+
+New backups will be queued: Phase will be set to `Queued`, and `QueuePosition` will be set to a int value incremented from the highest current `QueuePosition` value among Queued backups.
+
+Queued backups will be removed from the queue if runnable:
+1) If the total number of backups either InProgress or ReadyToStart is greater than or equal to the concurrency limit, then exit without removing from the queue.
+2) If the current backup overlaps with any InProgress, ReadyToStart, or Queued backup with `QueuePosition < currentBackup.QueuePosition` then exit without removing from the queue.
+3) If we get here, the backup is runnable. To resolve a potential race condition where an InProgress backup completes between reconciling the backup with QueuePosition `n-1` and reconciling the current backup with QueuePosition `n`, we also check to see whether there are any runnable backups in the queue ahead of this one. The only time this will happen is if a backup completes immediately before reconcile starts which either frees up a concurrency slot or removes a namespace conflict. In this case, we don't want to run the current backup since the one ahead of this one in the queue (which was recently passed over before the InProgress backup completed) must run first. In this case, exit without removing from the queue.
+4) If we get here, remove the backup from the queue by setting Phase to `ReadyToStart` and `QueuePosition` to zero. Decrement the `QueuePosition` of any other Queued backups with a `QueuePosition` higher than the current backup's queue position prior to dequeuing. At this point, the backup reconciler will start the backup.
+
+`if len(inProgressBackups)+len(pendingStartBackups) >= concurrentBackups`
+
+```
+	switch original.Status.Phase {
+	case "", velerov1api.BackupPhaseNew:
+		// enqueue backup -- set phase=Queued, set queuePosition=maxCurrentQueuePosition+1
+	}
+	// We should only ever get these events when added in order by the periodical enqueue source
+	// so as long as the current backup has not conflicts ahead of it or running, we should be good to
+	// dequeue
+	case "", velerov1api.BackupPhaseQueued:
+		// list backups, filter on Queued, ReadyToStart, and InProgress
+		// if number of InProgress backups + number of ReadyToStart backups >= concurrency limit, exit
+		// generate list of all namespaces included in InProgress, ReadyToStart, and Queued backups with
+		// queuePosition < backup.Status.QueuePosition
+		// if overlap found, exit
+		// check backups ahead of this one in the queue for runnability. If any are runnable, exit
+		// dequeue backup: set Phase to ReadyToStart, QueuePosition to 0, and decrement QueuePosition
+		// for all QueuedBackups behind this one in the queue
+	}
+
+```
+
+The queue controller will run as a single reconciler thread, so we will not need to deal with concurrency issues when moving backups from New to Queued or from Queued to ReadyToStart, and all of the updates to QueuePosition will be from a single thread.
+
+### Updates to Backup controller
+
+The Reconcile logic will be updated to respond to ReadyToStart backups instead of New backups:
+
+```
+@@ -234,8 +234,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
+        // InProgress, we still need this check so we can return nil to indicate we've finished processing
+        // this key (even though it was a no-op).
+        switch original.Status.Phase {
+-       case "", velerov1api.BackupPhaseNew:
+-               // only process new backups
++       case velerov1api.BackupPhaseReadyToStart:
++               // only process ReadyToStart backups
+        default:
+                b.logger.WithFields(logrus.Fields{
+                        "backup": kubeutil.NamespaceAndName(original),
+```
+
+In addition, it will be configured to run in parallel by setting `MaxConcurrentReconciles` based on the `concurrent-backups` server arg.
+
+```
+@@ -149,6 +149,9 @@ func NewBackupReconciler(
+ func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error {
+        return ctrl.NewControllerManagedBy(mgr).
+                For(&velerov1api.Backup{}).
++                WithOptions(controller.Options{
++                       MaxConcurrentReconciles: concurrentBackups,
++               }).
+                Named(constant.ControllerBackup).
+                Complete(b)
+ }
+```
+
+The controller-runtime core reconciler logic already prevents the same resource from being reconciled by two different reconciler threads, so we don't need to worry about concurrency issues at the controller level.
+
+The workerPool reference will be moved from the backupReconciler to the backupRequest, since this will now be backup-specific, and the initialization code for the worker pool will be moved from the reconciler init into the backup reconcile. This worker pool will be shut down upon exiting the Reconcile method.
+
+### Resilience to restart of velero pod
+
+The new backup phases (`Queued` and `ReadyToStart`) will be resilient to velero pod restarts. If the velero pod crashes or is restarted, only backups in the `InProgress` phase will be failed, so there is no change to current behavior. Queued backups will retain their queue position on restart, and ReadyToStart backups will move to InProgress when reconciled.
+
+### Observability
+
+#### Logging
+
+When a backup is dequeued, an info log message will also include the wait time, calculated as `now - creationTimestamp`. When a backup is passed over due to overlap, an info log message will indicate which namespaces were in conflict.
+
+#### Velero CLI
+
+The `velero backup describe` output will include the current queue position for queued backups.

examples/minio/00-minio-deployment.yaml

@@ -107,7 +107,7 @@ spec:
         command:
         - /bin/sh
         - -c
-        - "mc --config-dir=/config config host add velero http://minio:9000 minio minio123 && mc --config-dir=/config mb -p velero/velero"
+        - "mc --config-dir=/config alias set velero http://minio:9000 minio minio123 && mc --config-dir=/config mb -p velero/velero"
         volumeMounts:
         - name: config
           mountPath: "/config"

go.mod

@@ -1,14 +1,14 @@
 module github.com/vmware-tanzu/velero
 
-go 1.22.0
+go 1.24.0
 
 require (
-	cloud.google.com/go/storage v1.40.0
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
+	cloud.google.com/go/storage v1.55.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
 	github.com/aws/aws-sdk-go-v2 v1.24.1
 	github.com/aws/aws-sdk-go-v2/config v1.26.3
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.14
@@ -17,58 +17,66 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7
 	github.com/bombsimon/logrusr/v3 v3.0.0
-	github.com/evanphx/json-patch/v5 v5.8.0
-	github.com/fatih/color v1.16.0
+	github.com/evanphx/json-patch/v5 v5.9.11
+	github.com/fatih/color v1.18.0
 	github.com/gobwas/glob v0.2.3
-	github.com/google/go-cmp v0.6.0
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-plugin v1.6.0
 	github.com/joho/godotenv v1.3.0
 	github.com/kopia/kopia v0.16.0
-	github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0
-	github.com/onsi/ginkgo/v2 v2.19.0
-	github.com/onsi/gomega v1.33.1
+	github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0
+	github.com/onsi/ginkgo/v2 v2.22.0
+	github.com/onsi/gomega v1.36.1
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/client_golang v1.22.0
+	github.com/prometheus/client_model v0.6.2
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/afero v1.6.0
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/afero v1.10.0
+	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/vmware-tanzu/crash-diagnostics v0.3.7
 	go.uber.org/zap v1.27.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/mod v0.17.0
-	golang.org/x/net v0.26.0
-	golang.org/x/oauth2 v0.19.0
-	golang.org/x/text v0.16.0
-	google.golang.org/api v0.172.0
-	google.golang.org/grpc v1.63.2
-	google.golang.org/protobuf v1.33.0
+	golang.org/x/mod v0.26.0
+	golang.org/x/net v0.42.0
+	golang.org/x/oauth2 v0.30.0
+	golang.org/x/text v0.27.0
+	google.golang.org/api v0.241.0
+	google.golang.org/grpc v1.73.0
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.29.0
-	k8s.io/apiextensions-apiserver v0.29.0
-	k8s.io/apimachinery v0.29.0
-	k8s.io/cli-runtime v0.24.0
-	k8s.io/client-go v0.29.0
-	k8s.io/klog/v2 v2.110.1
-	k8s.io/kube-aggregator v0.19.12
-	k8s.io/metrics v0.25.6
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
-	sigs.k8s.io/controller-runtime v0.17.2
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd
+	k8s.io/api v0.33.3
+	k8s.io/apiextensions-apiserver v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/cli-runtime v0.33.3
+	k8s.io/client-go v0.33.3
+	k8s.io/klog/v2 v2.130.1
+	k8s.io/kube-aggregator v0.33.3
+	k8s.io/metrics v0.33.3
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
+	sigs.k8s.io/controller-runtime v0.21.0
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
-	cloud.google.com/go v0.112.1 // indirect
-	cloud.google.com/go/compute v1.24.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.7 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	cel.dev/expr v0.23.0 // indirect
+	cloud.google.com/go v0.121.1 // indirect
+	cloud.google.com/go/auth v0.16.2 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.7.0 // indirect
+	cloud.google.com/go/iam v1.5.2 // indirect
+	cloud.google.com/go/monitoring v1.24.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
@@ -83,56 +91,61 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
 	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/edsrzf/mmap-go v1.2.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
-	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
-	github.com/klauspost/reedsolomon v1.12.1 // indirect
+	github.com/klauspost/reedsolomon v1.12.4 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/crc64nvme v1.0.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.69 // indirect
-	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/minio/minio-go/v7 v7.0.94 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -141,41 +154,48 @@ require (
 	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/oklog/run v1.0.0 // indirect
-	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
+	github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.52.3 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
-	github.com/rs/xid v1.5.0 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/tinylib/msgp v1.3.0 // indirect
 	github.com/vladimirvivien/gexe v0.1.1 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.25.0 // indirect
-	go.opentelemetry.io/otel/metric v1.25.0 // indirect
-	go.opentelemetry.io/otel/trace v1.25.0 // indirect
-	go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/zeebo/blake3 v0.2.4 // indirect
+	github.com/zeebo/errs v1.4.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/term v0.21.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	golang.org/x/crypto v0.40.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/term v0.33.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	golang.org/x/tools v0.34.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
+	google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/component-base v0.29.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
 )
 
-replace github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20241016073907-939dae5f9001
+replace github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20250722052735-3ea24d208777

hack/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$TARGETPLATFORM golang:1.22-bookworm
+FROM --platform=$TARGETPLATFORM golang:1.24-bookworm
 
 ARG GOPROXY
 
@@ -30,11 +30,11 @@ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/downloa
     chmod +x /usr/local/kubebuilder/bin/kubebuilder
 
 # get controller-tools
-RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0
+RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5
 
 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit
 # is not important)
-RUN go install golang.org/x/tools/cmd/goimports@11e9d9cc0042e6bd10337d4d2c3e5d9295508e7d
+RUN go install golang.org/x/tools/cmd/goimports@v0.33.0
 
 # get protoc compiler and golang plugin
 WORKDIR /root
@@ -94,7 +94,7 @@ RUN ARCH=$(go env GOARCH) && \
     chmod +x /usr/bin/goreleaser
 
 # get golangci-lint
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.57.2
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.1
 
 # install kubectl
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl
@@ -102,4 +102,4 @@ RUN chmod +x ./kubectl
 RUN mv ./kubectl /usr/local/bin
 
 # Fix the "dubious ownership" issue from git when running goreleaser.sh
-RUN echo "[safe] \n\t directory = *" > /.gitconfig
+RUN echo "[safe] \n\t directory = *" > /.gitconfig

hack/ci/build_util.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -x
-
-set -e
-
-function uploader {
-    gsutil cp $1 gs://$2/$1
-    gsutil -D setacl public-read gs://$2/$1 &> /dev/null
-}

hack/docker-push.sh

@@ -63,7 +63,7 @@ fi
 if [[ -z $BRANCH && -z $TAG ]]; then
     echo "Test Velero container build without pushing, when Dockerfile is changed by PR."
     BRANCH="${GITHUB_BASE_REF}-container"
-    OUTPUT_TYPE="local,dest=."
+    OUTPUT_TYPE="tar"
 else
     OUTPUT_TYPE="registry"
 fi
@@ -88,8 +88,12 @@ else
     fi
 fi
 
-if [[ -z "$BUILDX_PLATFORMS" ]]; then
-    BUILDX_PLATFORMS="linux/amd64,linux/arm64"
+if [[ -z "$BUILD_OS" ]]; then
+    BUILD_OS="linux,windows"
+fi
+
+if [[ -z "$BUILD_ARCH" ]]; then
+    BUILD_ARCH="amd64,arm64"
 fi
 
 # Debugging info
@@ -98,13 +102,15 @@ echo "BRANCH: $BRANCH"
 echo "TAG: $TAG"
 echo "TAG_LATEST: $TAG_LATEST"
 echo "VERSION: $VERSION"
-echo "BUILDX_PLATFORMS: $BUILDX_PLATFORMS"
+echo "BUILD_OS: $BUILD_OS"
+echo "BUILD_ARCH: $BUILD_ARCH"
 
 echo "Building and pushing container images."
 
 
 VERSION="$VERSION" \
 TAG_LATEST="$TAG_LATEST" \
-BUILDX_PLATFORMS="$BUILDX_PLATFORMS" \
-BUILDX_OUTPUT_TYPE=$OUTPUT_TYPE \
-make all-containers
+BUILD_OS="$BUILD_OS" \
+BUILD_ARCH="$BUILD_ARCH" \
+BUILD_OUTPUT_TYPE=$OUTPUT_TYPE \
+make all-containers

hack/fix_restic_cve.txt

@@ -1,35 +1,36 @@
 diff --git a/go.mod b/go.mod
-index 5f939c481..1caa51275 100644
+index 5f939c481..6ae17f4a1 100644
 --- a/go.mod
 +++ b/go.mod
-@@ -24,32 +24,32 @@ require (
+@@ -24,32 +24,31 @@ require (
  	github.com/restic/chunker v0.4.0
  	github.com/spf13/cobra v1.6.1
  	github.com/spf13/pflag v1.0.5
 -	golang.org/x/crypto v0.5.0
 -	golang.org/x/net v0.5.0
 -	golang.org/x/oauth2 v0.4.0
-+	golang.org/x/crypto v0.21.0
-+	golang.org/x/net v0.23.0
-+	golang.org/x/oauth2 v0.7.0
- 	golang.org/x/sync v0.1.0
+-	golang.org/x/sync v0.1.0
 -	golang.org/x/sys v0.4.0
 -	golang.org/x/term v0.4.0
 -	golang.org/x/text v0.6.0
 -	google.golang.org/api v0.106.0
-+	golang.org/x/sys v0.18.0
-+	golang.org/x/term v0.18.0
-+	golang.org/x/text v0.14.0
++	golang.org/x/crypto v0.36.0
++	golang.org/x/net v0.38.0
++	golang.org/x/oauth2 v0.28.0
++	golang.org/x/sync v0.12.0
++	golang.org/x/sys v0.31.0
++	golang.org/x/term v0.30.0
++	golang.org/x/text v0.23.0
 +	google.golang.org/api v0.114.0
  )
  
  require (
 -	cloud.google.com/go v0.108.0 // indirect
 -	cloud.google.com/go/compute v1.15.1 // indirect
-+	cloud.google.com/go v0.110.0 // indirect
-+	cloud.google.com/go/compute v1.19.1 // indirect
- 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 -	cloud.google.com/go/iam v0.10.0 // indirect
++	cloud.google.com/go v0.110.0 // indirect
++	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 +	cloud.google.com/go/iam v0.13.0 // indirect
  	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
  	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@@ -48,7 +49,7 @@ index 5f939c481..1caa51275 100644
  	github.com/inconshreveable/mousetrap v1.1.0 // indirect
  	github.com/json-iterator/go v1.1.12 // indirect
  	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
-@@ -63,9 +63,9 @@ require (
+@@ -63,11 +62,13 @@ require (
  	go.opencensus.io v0.24.0 // indirect
  	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
  	google.golang.org/appengine v1.6.7 // indirect
@@ -61,32 +62,58 @@ index 5f939c481..1caa51275 100644
  	gopkg.in/ini.v1 v1.67.0 // indirect
  	gopkg.in/yaml.v3 v3.0.1 // indirect
  )
+ 
+-go 1.18
++go 1.23.0
++
++toolchain go1.23.7
 diff --git a/go.sum b/go.sum
-index 026e1d2fa..27d4207f4 100644
+index 026e1d2fa..805792055 100644
 --- a/go.sum
 +++ b/go.sum
-@@ -1,13 +1,13 @@
+@@ -1,23 +1,24 @@
  cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 -cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk=
 -cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q=
 -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE=
 -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
-+cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
-+cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
-+cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
-+cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
- cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
- cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 -cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI=
 -cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
 -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
++cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
++cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
++cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
++cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
 +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
 +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
++cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
  cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI=
  cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
  github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI=
-@@ -70,8 +70,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0=
+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
++github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
+ github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
++github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
+ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+ github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
+ github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do=
+@@ -54,6 +55,7 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
+ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+ github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
++github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+@@ -70,8 +72,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
  github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
  github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
  github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -97,12 +124,13 @@ index 026e1d2fa..27d4207f4 100644
  github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
  github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
  github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-@@ -82,17 +82,17 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
+@@ -82,17 +84,18 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
  github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
  github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
  github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
 +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
++github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
  github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
  github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc=
  github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
@@ -120,58 +148,82 @@ index 026e1d2fa..27d4207f4 100644
  github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
  github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
  github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-@@ -172,8 +172,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
+@@ -114,6 +117,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+ github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs=
+ github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
++github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+ github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+ github.com/minio/minio-go/v7 v7.0.46 h1:Vo3tNmNXuj7ME5qrvN4iadO7b4mzu/RSFdUkUhaPldk=
+@@ -129,6 +133,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P
+ github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
+ github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
++github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
+ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+ github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+@@ -172,8 +177,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
  golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
  golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
  golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
 -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
-+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
++golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
++golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
  golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
  golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
  golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-@@ -189,11 +189,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
+@@ -189,17 +194,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
  golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
  golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
  golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
 -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
++golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
++golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
  golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
 -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
-+golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
-+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
++golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc=
++golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
  golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
  golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
  golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-@@ -214,17 +214,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
+ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
++golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
++golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+@@ -214,17 +219,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
  golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
  golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
  golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
++golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
++golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
  golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
 -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
-+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
++golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
++golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
  golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
  golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
 -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
++golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
++golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
  golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
  golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
  golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-@@ -237,8 +237,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
+@@ -237,8 +242,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
  golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
  golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
  golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
@@ -182,7 +234,7 @@ index 026e1d2fa..27d4207f4 100644
  google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
  google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
  google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-@@ -246,15 +246,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
+@@ -246,15 +251,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
  google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
  google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
  google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
@@ -202,7 +254,7 @@ index 026e1d2fa..27d4207f4 100644
  google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
  google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
  google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-@@ -266,8 +266,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
+@@ -266,14 +271,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
  google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
  google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
  google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
@@ -213,3 +265,10 @@ index 026e1d2fa..27d4207f4 100644
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
  gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
++gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/credentials/file_store.go

@@ -71,7 +71,8 @@ func (n *namespacedFileStore) Path(selector *corev1api.SecretKeySelector) (strin
 
 	keyFilePath := filepath.Join(n.fsRoot, fmt.Sprintf("%s-%s", selector.Name, selector.Key))
 
-	file, err := n.fs.OpenFile(keyFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	// owner RW perms, group R perms, no public perms
+	file, err := n.fs.OpenFile(keyFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0640)
 	if err != nil {
 		return "", errors.Wrap(err, "unable to open credentials file for writing")
 	}

internal/credentials/file_store_test.go

@@ -17,11 +17,10 @@ limitations under the License.
 package credentials
 
 import (
-	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	corev1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
@@ -32,8 +31,8 @@ func TestNamespacedFileStore(t *testing.T) {
 		name             string
 		namespace        string
 		fsRoot           string
-		secrets          []*corev1.Secret
-		secretSelector   *corev1.SecretKeySelector
+		secrets          []*corev1api.Secret
+		secretSelector   *corev1api.SecretKeySelector
 		wantErr          string
 		expectedPath     string
 		expectedContents string
@@ -48,7 +47,7 @@ func TestNamespacedFileStore(t *testing.T) {
 			namespace:      "ns1",
 			fsRoot:         "/tmp/credentials",
 			secretSelector: builder.ForSecretKeySelector("credential", "key2").Result(),
-			secrets: []*corev1.Secret{
+			secrets: []*corev1api.Secret{
 				builder.ForSecret("ns1", "credential").Data(map[string][]byte{
 					"key1": []byte("ns1-secretdata1"),
 					"key2": []byte("ns1-secretdata2"),
@@ -68,7 +67,7 @@ func TestNamespacedFileStore(t *testing.T) {
 			client := velerotest.NewFakeControllerRuntimeClient(t)
 
 			for _, secret := range tc.secrets {
-				require.NoError(t, client.Create(context.Background(), secret))
+				require.NoError(t, client.Create(t.Context(), secret))
 			}
 
 			fs := velerotest.NewFakeFileSystem()

internal/credentials/local.go

@@ -0,0 +1,7 @@
+package credentials
+
+import "os"
+
+func DefaultStoreDirectory() string {
+	return os.TempDir() + "/credentials"
+}

internal/credentials/mocks/FileStore.go

@@ -4,7 +4,7 @@ package mocks
 
 import (
 	mock "github.com/stretchr/testify/mock"
-	v1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 )
 
 // FileStore is an autogenerated mock type for the FileStore type
@@ -13,18 +13,18 @@ type FileStore struct {
 }
 
 // Path provides a mock function with given fields: selector
-func (_m *FileStore) Path(selector *v1.SecretKeySelector) (string, error) {
+func (_m *FileStore) Path(selector *corev1api.SecretKeySelector) (string, error) {
 	ret := _m.Called(selector)
 
 	var r0 string
-	if rf, ok := ret.Get(0).(func(*v1.SecretKeySelector) string); ok {
+	if rf, ok := ret.Get(0).(func(*corev1api.SecretKeySelector) string); ok {
 		r0 = rf(selector)
 	} else {
 		r0 = ret.Get(0).(string)
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func(*v1.SecretKeySelector) error); ok {
+	if rf, ok := ret.Get(1).(func(*corev1api.SecretKeySelector) error); ok {
 		r1 = rf(selector)
 	} else {
 		r1 = ret.Error(1)

internal/credentials/mocks/SecretStore.go

@@ -4,7 +4,7 @@ package mocks
 
 import (
 	mock "github.com/stretchr/testify/mock"
-	v1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 )
 
 // SecretStore is an autogenerated mock type for the SecretStore type
@@ -13,18 +13,18 @@ type SecretStore struct {
 }
 
 // Get provides a mock function with given fields: selector
-func (_m *SecretStore) Get(selector *v1.SecretKeySelector) (string, error) {
+func (_m *SecretStore) Get(selector *corev1api.SecretKeySelector) (string, error) {
 	ret := _m.Called(selector)
 
 	var r0 string
-	if rf, ok := ret.Get(0).(func(*v1.SecretKeySelector) string); ok {
+	if rf, ok := ret.Get(0).(func(*corev1api.SecretKeySelector) string); ok {
 		r0 = rf(selector)
 	} else {
 		r0 = ret.Get(0).(string)
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func(*v1.SecretKeySelector) error); ok {
+	if rf, ok := ret.Get(1).(func(*corev1api.SecretKeySelector) error); ok {
 		r1 = rf(selector)
 	} else {
 		r1 = ret.Error(1)

internal/delete/actions/csi/volumesnapshot_action.go

@@ -1,120 +0,0 @@
-/*
-Copyright the Velero contributors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package csi
-
-import (
-	"context"
-	"fmt"
-
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/runtime"
-	crclient "sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/vmware-tanzu/velero/pkg/client"
-	plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
-	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	"github.com/vmware-tanzu/velero/pkg/util/csi"
-	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
-)
-
-// volumeSnapshotDeleteItemAction is a backup item action plugin for Velero.
-type volumeSnapshotDeleteItemAction struct {
-	log      logrus.FieldLogger
-	crClient crclient.Client
-}
-
-// AppliesTo returns information indicating that the
-// VolumeSnapshotBackupItemAction should be invoked to backup
-// VolumeSnapshots.
-func (p *volumeSnapshotDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) {
-	p.log.Debug("VolumeSnapshotBackupItemAction AppliesTo")
-
-	return velero.ResourceSelector{
-		IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"},
-	}, nil
-}
-
-func (p *volumeSnapshotDeleteItemAction) Execute(
-	input *velero.DeleteItemActionExecuteInput,
-) error {
-	p.log.Info("Starting VolumeSnapshotDeleteItemAction for volumeSnapshot")
-
-	var vs snapshotv1api.VolumeSnapshot
-
-	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(
-		input.Item.UnstructuredContent(),
-		&vs,
-	); err != nil {
-		return errors.Wrapf(err, "failed to convert input.Item from unstructured")
-	}
-
-	// We don't want this DeleteItemAction plugin to delete VolumeSnapshot
-	// taken outside of Velero.  So skip deleting VolumeSnapshot objects
-	// that were not created in the process of creating the Velero
-	// backup being deleted.
-	if !kubeutil.HasBackupLabel(&vs.ObjectMeta, input.Backup.Name) {
-		p.log.Info(
-			"VolumeSnapshot %s/%s was not taken by backup %s, skipping deletion",
-			vs.Namespace, vs.Name, input.Backup.Name,
-		)
-		return nil
-	}
-
-	p.log.Infof("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name)
-	if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil {
-		// we patch the DeletionPolicy of the VolumeSnapshotContent
-		// to set it to Delete.  This ensures that the volume snapshot
-		// in the storage provider is also deleted.
-		err := csi.SetVolumeSnapshotContentDeletionPolicy(
-			*vs.Status.BoundVolumeSnapshotContentName,
-			p.crClient,
-		)
-		if err != nil && !apierrors.IsNotFound(err) {
-			return errors.Wrapf(
-				err,
-				fmt.Sprintf("failed to patch DeletionPolicy of volume snapshot %s/%s",
-					vs.Namespace, vs.Name),
-			)
-		}
-
-		if apierrors.IsNotFound(err) {
-			return nil
-		}
-	}
-	err := p.crClient.Delete(context.TODO(), &vs)
-	if err != nil && !apierrors.IsNotFound(err) {
-		return err
-	}
-	return nil
-}
-
-func NewVolumeSnapshotDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer {
-	return func(logger logrus.FieldLogger) (interface{}, error) {
-		crClient, err := f.KubebuilderClient()
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-
-		return &volumeSnapshotDeleteItemAction{
-			log:      logger,
-			crClient: crClient,
-		}, nil
-	}
-}

internal/delete/actions/csi/volumesnapshot_action_test.go

@@ -1,151 +0,0 @@
-/*
-Copyright the Velero contributors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package csi
-
-import (
-	"context"
-	"fmt"
-	"testing"
-
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/require"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-
-	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	"github.com/vmware-tanzu/velero/pkg/builder"
-	factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks"
-	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	velerotest "github.com/vmware-tanzu/velero/pkg/test"
-)
-
-func TestVSExecute(t *testing.T) {
-	tests := []struct {
-		name      string
-		item      runtime.Unstructured
-		vs        *snapshotv1api.VolumeSnapshot
-		backup    *velerov1api.Backup
-		createVS  bool
-		expectErr bool
-	}{
-		{
-			name: "VolumeSnapshot doesn't have backup label",
-			item: velerotest.UnstructuredOrDie(
-				`
-				{
-					"apiVersion": "snapshot.storage.k8s.io/v1",
-					"kind": "VolumeSnapshot",
-					"metadata": {
-						"namespace": "ns",
-						"name": "foo"
-					}
-				}
-				`,
-			),
-			backup:    builder.ForBackup("velero", "backup").Result(),
-			expectErr: false,
-		},
-		{
-			name: "VolumeSnapshot doesn't exist in the cluster",
-			vs: builder.ForVolumeSnapshot("foo", "bar").
-				ObjectMeta(builder.WithLabelsMap(
-					map[string]string{velerov1api.BackupNameLabel: "backup"},
-				)).Status().
-				BoundVolumeSnapshotContentName("vsc").
-				Result(),
-			backup:    builder.ForBackup("velero", "backup").Result(),
-			expectErr: true,
-		},
-		{
-			name: "Normal case, VolumeSnapshot should be deleted",
-			vs: builder.ForVolumeSnapshot("foo", "bar").
-				ObjectMeta(builder.WithLabelsMap(
-					map[string]string{velerov1api.BackupNameLabel: "backup"},
-				)).Status().
-				BoundVolumeSnapshotContentName("vsc").
-				Result(),
-			backup:    builder.ForBackup("velero", "backup").Result(),
-			expectErr: false,
-			createVS:  true,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			crClient := velerotest.NewFakeControllerRuntimeClient(t)
-			logger := logrus.StandardLogger()
-
-			p := volumeSnapshotDeleteItemAction{log: logger, crClient: crClient}
-
-			if test.vs != nil {
-				vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vs)
-				require.NoError(t, err)
-				test.item = &unstructured.Unstructured{Object: vsMap}
-			}
-
-			if test.createVS {
-				require.NoError(t, crClient.Create(context.TODO(), test.vs))
-			}
-
-			err := p.Execute(
-				&velero.DeleteItemActionExecuteInput{
-					Item:   test.item,
-					Backup: test.backup,
-				},
-			)
-
-			if test.expectErr == false {
-				require.NoError(t, err)
-			}
-		})
-	}
-}
-
-func TestVSAppliesTo(t *testing.T) {
-	p := volumeSnapshotDeleteItemAction{
-		log: logrus.StandardLogger(),
-	}
-	selector, err := p.AppliesTo()
-
-	require.NoError(t, err)
-
-	require.Equal(
-		t,
-		velero.ResourceSelector{
-			IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"},
-		},
-		selector,
-	)
-}
-
-func TestNewVolumeSnapshotDeleteItemAction(t *testing.T) {
-	logger := logrus.StandardLogger()
-	crClient := velerotest.NewFakeControllerRuntimeClient(t)
-
-	f := &factorymocks.Factory{}
-	f.On("KubebuilderClient").Return(nil, fmt.Errorf(""))
-	plugin := NewVolumeSnapshotDeleteItemAction(f)
-	_, err := plugin(logger)
-	require.Error(t, err)
-
-	f1 := &factorymocks.Factory{}
-	f1.On("KubebuilderClient").Return(crClient, nil)
-	plugin1 := NewVolumeSnapshotDeleteItemAction(f1)
-	_, err1 := plugin1(logger)
-	require.NoError(t, err1)
-}

internal/delete/actions/csi/volumesnapshotcontent_action.go

@@ -18,19 +18,23 @@ package csi
 
 import (
 	"context"
-	"fmt"
+	"time"
 
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
+	"github.com/google/uuid"
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	corev1api "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	"github.com/vmware-tanzu/velero/pkg/util/csi"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
@@ -77,25 +81,55 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute(
 
 	p.log.Infof("Deleting VolumeSnapshotContent %s", snapCont.Name)
 
-	if err := csi.SetVolumeSnapshotContentDeletionPolicy(
-		snapCont.Name,
-		p.crClient,
+	uuid, err := uuid.NewRandom()
+	if err != nil {
+		p.log.WithError(err).Errorf("Fail to generate the UUID to create VSC %s", snapCont.Name)
+		return errors.Wrapf(err, "Fail to generate the UUID to create VSC %s", snapCont.Name)
+	}
+	snapCont.Name = "vsc-" + uuid.String()
+
+	snapCont.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete
+
+	snapCont.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{
+		SnapshotHandle: snapCont.Status.SnapshotHandle,
+	}
+
+	snapCont.Spec.VolumeSnapshotRef = corev1api.ObjectReference{
+		APIVersion: snapshotv1api.SchemeGroupVersion.String(),
+		Kind:       "VolumeSnapshot",
+		Namespace:  "ns-" + string(snapCont.UID),
+		Name:       "name-" + string(snapCont.UID),
+	}
+
+	snapCont.ResourceVersion = ""
+
+	if err := p.crClient.Create(context.TODO(), &snapCont); err != nil {
+		return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", snapCont.Name)
+	}
+
+	// Read resource timeout from backup annotation, if not set, use default value.
+	timeout, err := time.ParseDuration(
+		input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation])
+	if err != nil {
+		p.log.Warnf("fail to parse resource timeout annotation %s: %s",
+			input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error())
+		timeout = 10 * time.Minute
+	}
+	p.log.Debugf("resource timeout is set to %s", timeout.String())
+
+	interval := 5 * time.Second
+
+	// Wait until VSC created and ReadyToUse is true.
+	if err := wait.PollUntilContextTimeout(
+		context.Background(),
+		interval,
+		timeout,
+		true,
+		func(ctx context.Context) (bool, error) {
+			return checkVSCReadiness(ctx, &snapCont, p.crClient)
+		},
 	); err != nil {
-		// #4764: Leave a warning when VolumeSnapshotContent cannot be found for deletion.
-		// Manual deleting VolumeSnapshotContent can cause this.
-		// It's tricky for Velero to handle this inconsistency.
-		// Even if Velero restores the VolumeSnapshotContent, CSI snapshot controller
-		// may not delete it correctly due to the snapshot represented by VolumeSnapshotContent
-		// already deleted on cloud provider.
-		if apierrors.IsNotFound(err) {
-			p.log.Warnf(
-				"VolumeSnapshotContent %s of backup %s cannot be found. May leave orphan snapshot %s on cloud provider.",
-				snapCont.Name, input.Backup.Name, *snapCont.Status.SnapshotHandle)
-			return nil
-		}
-		return errors.Wrapf(err, fmt.Sprintf(
-			"failed to set DeletionPolicy on volumesnapshotcontent %s. Skipping deletion",
-			snapCont.Name))
+		return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name)
 	}
 
 	if err := p.crClient.Delete(
@@ -109,10 +143,29 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute(
 	return nil
 }
 
+var checkVSCReadiness = func(
+	ctx context.Context,
+	vsc *snapshotv1api.VolumeSnapshotContent,
+	client crclient.Client,
+) (bool, error) {
+	tmpVSC := new(snapshotv1api.VolumeSnapshotContent)
+	if err := client.Get(ctx, crclient.ObjectKeyFromObject(vsc), tmpVSC); err != nil {
+		return false, errors.Wrapf(
+			err, "failed to get VolumeSnapshotContent %s", vsc.Name,
+		)
+	}
+
+	if tmpVSC.Status != nil && boolptr.IsSetToTrue(tmpVSC.Status.ReadyToUse) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func NewVolumeSnapshotContentDeleteItemAction(
 	f client.Factory,
 ) plugincommon.HandlerInitializer {
-	return func(logger logrus.FieldLogger) (interface{}, error) {
+	return func(logger logrus.FieldLogger) (any, error) {
 		crClient, err := f.KubebuilderClient()
 		if err != nil {
 			return nil, err

internal/delete/actions/csi/volumesnapshotcontent_action_test.go

@@ -21,11 +21,14 @@ import (
 	"fmt"
 	"testing"
 
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
@@ -37,11 +40,15 @@ import (
 func TestVSCExecute(t *testing.T) {
 	snapshotHandleStr := "test"
 	tests := []struct {
-		name      string
-		item      runtime.Unstructured
-		vsc       *snapshotv1api.VolumeSnapshotContent
-		backup    *velerov1api.Backup
-		createVSC bool
+		name     string
+		item     runtime.Unstructured
+		vsc      *snapshotv1api.VolumeSnapshotContent
+		backup   *velerov1api.Backup
+		function func(
+			ctx context.Context,
+			vsc *snapshotv1api.VolumeSnapshotContent,
+			client crclient.Client,
+		) (bool, error)
 		expectErr bool
 	}{
 		{
@@ -62,17 +69,30 @@ func TestVSCExecute(t *testing.T) {
 			expectErr: false,
 		},
 		{
-			name:      "VolumeSnapshotContent doesn't exist in the cluster, no error",
+			name:      "Normal case, VolumeSnapshot should be deleted",
 			vsc:       builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(),
-			backup:    builder.ForBackup("velero", "backup").Result(),
+			backup:    builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(),
 			expectErr: false,
+			function: func(
+				ctx context.Context,
+				vsc *snapshotv1api.VolumeSnapshotContent,
+				client crclient.Client,
+			) (bool, error) {
+				return true, nil
+			},
 		},
 		{
 			name:      "Normal case, VolumeSnapshot should be deleted",
 			vsc:       builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(),
-			backup:    builder.ForBackup("velero", "backup").Result(),
-			expectErr: false,
-			createVSC: true,
+			backup:    builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(),
+			expectErr: true,
+			function: func(
+				ctx context.Context,
+				vsc *snapshotv1api.VolumeSnapshotContent,
+				client crclient.Client,
+			) (bool, error) {
+				return false, errors.Errorf("test error case")
+			},
 		},
 	}
 
@@ -80,6 +100,7 @@ func TestVSCExecute(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			crClient := velerotest.NewFakeControllerRuntimeClient(t)
 			logger := logrus.StandardLogger()
+			checkVSCReadiness = test.function
 
 			p := volumeSnapshotContentDeleteItemAction{log: logger, crClient: crClient}
 
@@ -89,10 +110,6 @@ func TestVSCExecute(t *testing.T) {
 				test.item = &unstructured.Unstructured{Object: vscMap}
 			}
 
-			if test.createVSC {
-				require.NoError(t, crClient.Create(context.TODO(), test.vsc))
-			}
-
 			err := p.Execute(
 				&velero.DeleteItemActionExecuteInput{
 					Item:   test.item,
@@ -140,3 +157,53 @@ func TestNewVolumeSnapshotContentDeleteItemAction(t *testing.T) {
 	_, err1 := plugin1(logger)
 	require.NoError(t, err1)
 }
+
+func TestCheckVSCReadiness(t *testing.T) {
+	tests := []struct {
+		name      string
+		vsc       *snapshotv1api.VolumeSnapshotContent
+		createVSC bool
+		expectErr bool
+		ready     bool
+	}{
+		{
+			name: "VSC not exist",
+			vsc: &snapshotv1api.VolumeSnapshotContent{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "vsc-1",
+					Namespace: "velero",
+				},
+			},
+			createVSC: false,
+			expectErr: true,
+			ready:     false,
+		},
+		{
+			name: "VSC not ready",
+			vsc: &snapshotv1api.VolumeSnapshotContent{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "vsc-1",
+					Namespace: "velero",
+				},
+			},
+			createVSC: true,
+			expectErr: false,
+			ready:     false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			crClient := velerotest.NewFakeControllerRuntimeClient(t)
+			if test.createVSC {
+				require.NoError(t, crClient.Create(t.Context(), test.vsc))
+			}
+
+			ready, err := checkVSCReadiness(t.Context(), test.vsc, crClient)
+			require.Equal(t, test.ready, ready)
+			if test.expectErr {
+				require.Error(t, err)
+			}
+		})
+	}
+}

internal/delete/delete_item_action_handler_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package delete
 
 import (
-	"context"
 	"io"
 	"sort"
 	"testing"
@@ -221,16 +220,16 @@ func (h *harness) addResource(t *testing.T, resource *test.APIResource) {
 
 		unstructuredObj := &unstructured.Unstructured{Object: obj}
 		if resource.Namespaced {
-			_, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{})
+			_, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{})
 		} else {
-			_, err = h.DynamicClient.Resource(resource.GVR()).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{})
+			_, err = h.DynamicClient.Resource(resource.GVR()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{})
 		}
 		require.NoError(t, err)
 	}
 }
 
 // recordResourcesAction is a delete item action that can be configured to run
-// for specific resources/namespaces and simply record the items that is is
+// for specific resources/namespaces and simply record the items that is
 // executed for.
 type recordResourcesAction struct {
 	selector velero.ResourceSelector

internal/hook/hook_tracker.go

@@ -46,6 +46,9 @@ type hookKey struct {
 	// Container indicates the container hooks use.
 	// For hooks specified in the backup/restore spec, the container might be the same under different hookName.
 	container string
+	// hookIndex contains the slice index for the specific hook, in order to track multiple hooks
+	// for the same container
+	hookIndex int
 }
 
 // hookStatus records the execution status of a specific hook.
@@ -83,7 +86,7 @@ func NewHookTracker() *HookTracker {
 // Add adds a hook to the hook tracker
 // Add must precede the Record for each individual hook.
 // In other words, a hook must be added to the tracker before its execution result is recorded.
-func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName string, hookPhase HookPhase) {
+func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int) {
 	ht.lock.Lock()
 	defer ht.lock.Unlock()
 
@@ -94,6 +97,7 @@ func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName st
 		container:    container,
 		hookPhase:    hookPhase,
 		hookName:     hookName,
+		hookIndex:    hookIndex,
 	}
 
 	if _, ok := ht.tracker[key]; !ok {
@@ -108,7 +112,7 @@ func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName st
 // Record records the hook's execution status
 // Add must precede the Record for each individual hook.
 // In other words, a hook must be added to the tracker before its execution result is recorded.
-func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookFailed bool, hookErr error) error {
+func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int, hookFailed bool, hookErr error) error {
 	ht.lock.Lock()
 	defer ht.lock.Unlock()
 
@@ -119,6 +123,7 @@ func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName
 		container:    container,
 		hookPhase:    hookPhase,
 		hookName:     hookName,
+		hookIndex:    hookIndex,
 	}
 
 	if _, ok := ht.tracker[key]; !ok {
@@ -179,24 +184,24 @@ func NewMultiHookTracker() *MultiHookTracker {
 }
 
 // Add adds a backup/restore hook to the tracker
-func (mht *MultiHookTracker) Add(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase) {
+func (mht *MultiHookTracker) Add(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int) {
 	mht.lock.Lock()
 	defer mht.lock.Unlock()
 
 	if _, ok := mht.trackers[name]; !ok {
 		mht.trackers[name] = NewHookTracker()
 	}
-	mht.trackers[name].Add(podNamespace, podName, container, source, hookName, hookPhase)
+	mht.trackers[name].Add(podNamespace, podName, container, source, hookName, hookPhase, hookIndex)
 }
 
 // Record records a backup/restore hook execution status
-func (mht *MultiHookTracker) Record(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookFailed bool, hookErr error) error {
+func (mht *MultiHookTracker) Record(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int, hookFailed bool, hookErr error) error {
 	mht.lock.RLock()
 	defer mht.lock.RUnlock()
 
 	var err error
 	if _, ok := mht.trackers[name]; ok {
-		err = mht.trackers[name].Record(podNamespace, podName, container, source, hookName, hookPhase, hookFailed, hookErr)
+		err = mht.trackers[name].Record(podNamespace, podName, container, source, hookName, hookPhase, hookIndex, hookFailed, hookErr)
 	} else {
 		err = fmt.Errorf("the backup/restore not exist in hook tracker, backup/restore name: %s", name)
 	}

internal/hook/hook_tracker_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewHookTracker(t *testing.T) {
@@ -33,7 +34,7 @@ func TestNewHookTracker(t *testing.T) {
 func TestHookTracker_Add(t *testing.T) {
 	tracker := NewHookTracker()
 
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
 
 	key := hookKey{
 		podNamespace: "ns1",
@@ -50,8 +51,8 @@ func TestHookTracker_Add(t *testing.T) {
 
 func TestHookTracker_Record(t *testing.T) {
 	tracker := NewHookTracker()
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	err := tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	err := tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 
 	key := hookKey{
 		podNamespace: "ns1",
@@ -65,42 +66,43 @@ func TestHookTracker_Record(t *testing.T) {
 	info := tracker.tracker[key]
 	assert.True(t, info.hookFailed)
 	assert.True(t, info.hookExecuted)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
-	err = tracker.Record("ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
-	assert.Error(t, err)
+	err = tracker.Record("ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
+	require.Error(t, err)
 
-	err = tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", false, nil)
-	assert.NoError(t, err)
+	err = tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, false, nil)
+	require.NoError(t, err)
 	assert.True(t, info.hookFailed)
 }
 
 func TestHookTracker_Stat(t *testing.T) {
 	tracker := NewHookTracker()
 
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", "")
-	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0)
+	tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1)
+	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 
 	attempted, failed := tracker.Stat()
-	assert.Equal(t, 2, attempted)
+	assert.Equal(t, 3, attempted)
 	assert.Equal(t, 1, failed)
 }
 
 func TestHookTracker_IsComplete(t *testing.T) {
 	tracker := NewHookTracker()
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
-	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, true, fmt.Errorf("err"))
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0)
+	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0, true, fmt.Errorf("err"))
 	assert.True(t, tracker.IsComplete())
 
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
 	assert.False(t, tracker.IsComplete())
 }
 
 func TestHookTracker_HookErrs(t *testing.T) {
 	tracker := NewHookTracker()
-	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 
 	hookErrs := tracker.HookErrs()
 	assert.Len(t, hookErrs, 1)
@@ -109,7 +111,7 @@ func TestHookTracker_HookErrs(t *testing.T) {
 func TestMultiHookTracker_Add(t *testing.T) {
 	mht := NewMultiHookTracker()
 
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
 
 	key := hookKey{
 		podNamespace: "ns1",
@@ -118,6 +120,7 @@ func TestMultiHookTracker_Add(t *testing.T) {
 		hookPhase:    "",
 		hookSource:   HookSourceAnnotation,
 		hookName:     "h1",
+		hookIndex:    0,
 	}
 
 	_, ok := mht.trackers["restore1"].tracker[key]
@@ -126,8 +129,8 @@ func TestMultiHookTracker_Add(t *testing.T) {
 
 func TestMultiHookTracker_Record(t *testing.T) {
 	mht := NewMultiHookTracker()
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	err := mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	err := mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 
 	key := hookKey{
 		podNamespace: "ns1",
@@ -136,36 +139,39 @@ func TestMultiHookTracker_Record(t *testing.T) {
 		hookPhase:    "",
 		hookSource:   HookSourceAnnotation,
 		hookName:     "h1",
+		hookIndex:    0,
 	}
 
 	info := mht.trackers["restore1"].tracker[key]
 	assert.True(t, info.hookFailed)
 	assert.True(t, info.hookExecuted)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
-	err = mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
-	assert.Error(t, err)
+	err = mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
+	require.Error(t, err)
 
-	err = mht.Record("restore2", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	err = mht.Record("restore2", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 	assert.Error(t, err)
 }
 
 func TestMultiHookTracker_Stat(t *testing.T) {
 	mht := NewMultiHookTracker()
 
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	mht.Add("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "")
-	mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
-	mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", false, nil)
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	mht.Add("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0)
+	mht.Add("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1)
+	mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
+	mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0, false, nil)
+	mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1, false, nil)
 
 	attempted, failed := mht.Stat("restore1")
-	assert.Equal(t, 2, attempted)
+	assert.Equal(t, 3, attempted)
 	assert.Equal(t, 1, failed)
 }
 
 func TestMultiHookTracker_Delete(t *testing.T) {
 	mht := NewMultiHookTracker()
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
 	mht.Delete("restore1")
 
 	_, ok := mht.trackers["restore1"]
@@ -174,11 +180,11 @@ func TestMultiHookTracker_Delete(t *testing.T) {
 
 func TestMultiHookTracker_IsComplete(t *testing.T) {
 	mht := NewMultiHookTracker()
-	mht.Add("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
-	mht.Record("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, true, fmt.Errorf("err"))
+	mht.Add("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0)
+	mht.Record("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0, true, fmt.Errorf("err"))
 	assert.True(t, mht.IsComplete("backup1"))
 
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
 	assert.False(t, mht.IsComplete("restore1"))
 
 	assert.True(t, mht.IsComplete("restore2"))
@@ -186,8 +192,8 @@ func TestMultiHookTracker_IsComplete(t *testing.T) {
 
 func TestMultiHookTracker_HookErrs(t *testing.T) {
 	mht := NewMultiHookTracker()
-	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "")
-	mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", true, fmt.Errorf("err"))
+	mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0)
+	mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err"))
 
 	hookErrs := mht.HookErrs("restore1")
 	assert.Len(t, hookErrs, 1)

internal/hook/item_hook_handler.go

@@ -223,7 +223,7 @@ func (h *DefaultItemHookHandler) HandleHooks(
 		hookFromAnnotations = getPodExecHookFromAnnotations(metadata.GetAnnotations(), "", log)
 	}
 	if hookFromAnnotations != nil {
-		hookTracker.Add(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase)
+		hookTracker.Add(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, 0)
 
 		hookLog := log.WithFields(
 			logrus.Fields{
@@ -239,7 +239,7 @@ func (h *DefaultItemHookHandler) HandleHooks(
 			hookLog.WithError(errExec).Error("Error executing hook")
 			hookFailed = true
 		}
-		errTracker := hookTracker.Record(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, hookFailed, errExec)
+		errTracker := hookTracker.Record(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, 0, hookFailed, errExec)
 		if errTracker != nil {
 			hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker")
 		}
@@ -267,10 +267,10 @@ func (h *DefaultItemHookHandler) HandleHooks(
 			hooks = resourceHook.Post
 		}
 
-		for _, hook := range hooks {
+		for i, hook := range hooks {
 			if groupResource == kuberesource.Pods {
 				if hook.Exec != nil {
-					hookTracker.Add(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase)
+					hookTracker.Add(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, i)
 					// The remaining hooks will only be executed if modeFailError is nil.
 					// Otherwise, execution will stop and only hook collection will occur.
 					if modeFailError == nil {
@@ -291,7 +291,7 @@ func (h *DefaultItemHookHandler) HandleHooks(
 								modeFailError = err
 							}
 						}
-						errTracker := hookTracker.Record(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, hookFailed, err)
+						errTracker := hookTracker.Record(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, i, hookFailed, err)
 						if errTracker != nil {
 							hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker")
 						}
@@ -534,6 +534,11 @@ type PodExecRestoreHook struct {
 	HookSource string
 	Hook       velerov1api.ExecRestoreHook
 	executed   bool
+	// hookIndex contains the slice index for the specific hook from the restore spec
+	// in order to track multiple hooks. Stored here because restore hook results are recorded
+	// outside of the original slice iteration
+	// for the same container
+	hookIndex int
 }
 
 // GroupRestoreExecHooks returns a list of hooks to be executed in a pod grouped by
@@ -561,12 +566,13 @@ func GroupRestoreExecHooks(
 		if hookFromAnnotation.Container == "" {
 			hookFromAnnotation.Container = pod.Spec.Containers[0].Name
 		}
-		hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), hookFromAnnotation.Container, HookSourceAnnotation, "<from-annotation>", HookPhase(""))
+		hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), hookFromAnnotation.Container, HookSourceAnnotation, "<from-annotation>", HookPhase(""), 0)
 		byContainer[hookFromAnnotation.Container] = []PodExecRestoreHook{
 			{
 				HookName:   "<from-annotation>",
 				HookSource: HookSourceAnnotation,
 				Hook:       *hookFromAnnotation,
+				hookIndex:  0,
 			},
 		}
 		return byContainer, nil
@@ -579,7 +585,7 @@ func GroupRestoreExecHooks(
 		if !rrh.Selector.applicableTo(kuberesource.Pods, namespace, labels) {
 			continue
 		}
-		for _, rh := range rrh.RestoreHooks {
+		for i, rh := range rrh.RestoreHooks {
 			if rh.Exec == nil {
 				continue
 			}
@@ -587,6 +593,7 @@ func GroupRestoreExecHooks(
 				HookName:   rrh.Name,
 				Hook:       *rh.Exec,
 				HookSource: HookSourceSpec,
+				hookIndex:  i,
 			}
 			// default to false if attr WaitForReady not set
 			if named.Hook.WaitForReady == nil {
@@ -596,7 +603,7 @@ func GroupRestoreExecHooks(
 			if named.Hook.Container == "" {
 				named.Hook.Container = pod.Spec.Containers[0].Name
 			}
-			hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), named.Hook.Container, HookSourceSpec, rrh.Name, HookPhase(""))
+			hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), named.Hook.Container, HookSourceSpec, rrh.Name, HookPhase(""), i)
 			byContainer[named.Hook.Container] = append(byContainer[named.Hook.Container], named)
 		}
 	}

internal/hook/item_hook_handler_test.go

@@ -120,7 +120,7 @@ func TestHandleHooksSkips(t *testing.T) {
 
 			groupResource := schema.ParseGroupResource(test.groupResource)
 			err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, PhasePre, hookTracker)
-			assert.NoError(t, err)
+			require.NoError(t, err)
 		})
 	}
 }
@@ -1151,6 +1151,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 							WaitTimeout:  metav1.Duration{Duration: time.Minute},
 							WaitForReady: boolptr.False(),
 						},
+						hookIndex: 0,
 					},
 					{
 						HookName:   "hook1",
@@ -1163,6 +1164,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 							WaitTimeout:  metav1.Duration{Duration: time.Minute * 2},
 							WaitForReady: boolptr.False(),
 						},
+						hookIndex: 2,
 					},
 					{
 						HookName:   "hook2",
@@ -1175,6 +1177,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 							WaitTimeout:  metav1.Duration{Duration: time.Minute * 4},
 							WaitForReady: boolptr.True(),
 						},
+						hookIndex: 0,
 					},
 				},
 				"container2": {
@@ -1189,6 +1192,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 							WaitTimeout:  metav1.Duration{Duration: time.Second * 3},
 							WaitForReady: boolptr.False(),
 						},
+						hookIndex: 1,
 					},
 				},
 			},
@@ -1199,7 +1203,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			actual, err := GroupRestoreExecHooks("restore1", tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger(), hookTracker)
-			assert.NoError(t, err)
+			require.NoError(t, err)
 			assert.Equal(t, tc.expected, actual)
 		})
 	}
@@ -1955,13 +1959,13 @@ func TestHandleRestoreHooks(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			handler := InitContainerRestoreHookHandler{}
 			podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.podInput)
-			assert.NoError(t, err)
+			require.NoError(t, err)
 			actual, err := handler.HandleRestoreHooks(velerotest.NewLogger(), kuberesource.Pods, &unstructured.Unstructured{Object: podMap}, tc.restoreHooks, tc.namespaceMapping)
 			assert.Equal(t, tc.expectedError, err)
 			if actual != nil {
 				actualPod := new(corev1api.Pod)
 				err = runtime.DefaultUnstructuredConverter.FromUnstructured(actual.UnstructuredContent(), actualPod)
-				assert.NoError(t, err)
+				require.NoError(t, err)
 				assert.Equal(t, tc.expectedPod, actualPod)
 			}
 		})
@@ -1976,7 +1980,7 @@ func TestValidateContainer(t *testing.T) {
 	expectedError := fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field")
 
 	// valid string should return nil as result.
-	assert.NoError(t, ValidateContainer([]byte(valid)))
+	require.NoError(t, ValidateContainer([]byte(valid)))
 
 	// noName string should return expected error as result.
 	assert.Equal(t, expectedError, ValidateContainer([]byte(noName)))

internal/hook/wait_exec_hook_handler.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
-	v1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/cache"
@@ -37,7 +37,7 @@ type WaitExecHookHandler interface {
 	HandleHooks(
 		ctx context.Context,
 		log logrus.FieldLogger,
-		pod *v1.Pod,
+		pod *corev1api.Pod,
 		byContainer map[string][]PodExecRestoreHook,
 		multiHookTracker *MultiHookTracker,
 		restoreName string,
@@ -73,7 +73,7 @@ var _ WaitExecHookHandler = &DefaultWaitExecHookHandler{}
 func (e *DefaultWaitExecHookHandler) HandleHooks(
 	ctx context.Context,
 	log logrus.FieldLogger,
-	pod *v1.Pod,
+	pod *corev1api.Pod,
 	byContainer map[string][]PodExecRestoreHook,
 	multiHookTracker *MultiHookTracker,
 	restoreName string,
@@ -116,8 +116,8 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 	// not yet been observed to be running. It relies on the Informer not to be called concurrently.
 	// When a container is observed running and its hooks are executed, the container is deleted
 	// from the byContainer map. When the map is empty the watch is ended.
-	handler := func(newObj interface{}) {
-		newPod, ok := newObj.(*v1.Pod)
+	handler := func(newObj any) {
+		newPod, ok := newObj.(*corev1api.Pod)
 		if !ok {
 			return
 		}
@@ -128,7 +128,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 			},
 		)
 
-		if newPod.Status.Phase == v1.PodSucceeded || newPod.Status.Phase == v1.PodFailed {
+		if newPod.Status.Phase == corev1api.PodSucceeded || newPod.Status.Phase == corev1api.PodFailed {
 			err := fmt.Errorf("pod entered phase %s before some post-restore exec hooks ran", newPod.Status.Phase)
 			podLog.Warning(err)
 			cancel()
@@ -169,7 +169,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 					hookLog.Error(err)
 					errors = append(errors, err)
 
-					errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), true, err)
+					errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), i, true, err)
 					if errTracker != nil {
 						hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker")
 					}
@@ -195,7 +195,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 					hookFailed = true
 				}
 
-				errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), hookFailed, hookErr)
+				errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), i, hookFailed, hookErr)
 				if errTracker != nil {
 					hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker")
 				}
@@ -214,18 +214,23 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 
 	selector := fields.OneTermEqualSelector("metadata.name", pod.Name)
 	lw := e.ListWatchFactory.NewListWatch(pod.Namespace, selector)
-
-	_, podWatcher := cache.NewInformer(lw, pod, 0, cache.ResourceEventHandlerFuncs{
-		AddFunc: handler,
-		UpdateFunc: func(_, newObj interface{}) {
-			handler(newObj)
+	_, podWatcher := cache.NewInformerWithOptions(cache.InformerOptions{
+		ListerWatcher: lw,
+		ObjectType:    pod,
+		ResyncPeriod:  0,
+		Handler: cache.ResourceEventHandlerFuncs{
+			AddFunc: handler,
+			UpdateFunc: func(_, newObj any) {
+				handler(newObj)
+			},
+			DeleteFunc: func(obj any) {
+				err := fmt.Errorf("pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod))
+				log.Error(err)
+				cancel()
+			},
 		},
-		DeleteFunc: func(obj interface{}) {
-			err := fmt.Errorf("pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod))
-			log.Error(err)
-			cancel()
-		},
-	})
+	},
+	)
 
 	podWatcher.Run(ctx.Done())
 
@@ -234,7 +239,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 	// containers to become ready.
 	// Each unexecuted hook is logged as an error and this error will be returned from this function.
 	for _, hooks := range byContainer {
-		for _, hook := range hooks {
+		for i, hook := range hooks {
 			if hook.executed {
 				continue
 			}
@@ -247,7 +252,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 				},
 			)
 
-			errTracker := multiHookTracker.Record(restoreName, pod.Namespace, pod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), true, err)
+			errTracker := multiHookTracker.Record(restoreName, pod.Namespace, pod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), i, true, err)
 			if errTracker != nil {
 				hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker")
 			}
@@ -260,7 +265,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 	return errors
 }
 
-func podHasContainer(pod *v1.Pod, containerName string) bool {
+func podHasContainer(pod *corev1api.Pod, containerName string) bool {
 	if pod == nil {
 		return false
 	}
@@ -273,7 +278,7 @@ func podHasContainer(pod *v1.Pod, containerName string) bool {
 	return false
 }
 
-func isContainerUp(pod *v1.Pod, containerName string, hooks []PodExecRestoreHook) bool {
+func isContainerUp(pod *corev1api.Pod, containerName string, hooks []PodExecRestoreHook) bool {
 	if pod == nil {
 		return false
 	}

internal/hook/wait_exec_hook_handler_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	v1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -52,18 +52,18 @@ func TestWaitExecHandleHooks(t *testing.T) {
 	type change struct {
 		// delta to wait since last change applied or pod added
 		wait    time.Duration
-		updated *v1.Pod
+		updated *corev1api.Pod
 	}
 	type expectedExecution struct {
 		hook  *velerov1api.ExecHook
 		name  string
 		error error
-		pod   *v1.Pod
+		pod   *corev1api.Pod
 	}
 	tests := []struct {
 		name string
 		// Used as argument to HandleHooks and first state added to ListerWatcher
-		initialPod         *v1.Pod
+		initialPod         *corev1api.Pod
 		groupResource      string
 		byContainer        map[string][]PodExecRestoreHook
 		expectedExecutions []expectedExecution
@@ -83,13 +83,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					podRestoreHookTimeoutAnnotationKey, "1s",
 					podRestoreHookWaitTimeoutAnnotationKey, "1m",
 				)).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -128,13 +128,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -152,13 +152,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					podRestoreHookTimeoutAnnotationKey, "1s",
 					podRestoreHookWaitTimeoutAnnotationKey, "1m",
 				)).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -197,13 +197,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -221,13 +221,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					podRestoreHookTimeoutAnnotationKey, "1s",
 					podRestoreHookWaitTimeoutAnnotationKey, "1m",
 				)).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -266,13 +266,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -290,13 +290,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					podRestoreHookTimeoutAnnotationKey, "1s",
 					podRestoreHookWaitTimeoutAnnotationKey, "1m",
 				)).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -335,13 +335,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -359,13 +359,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -376,13 +376,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			name:          "should return no error when hook from spec executes successfully",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -408,13 +408,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					},
 					pod: builder.ForPod("default", "my-pod").
 						ObjectMeta(builder.WithResourceVersion("1")).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -425,13 +425,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			name:          "should return error when spec hook with wait timeout expires with OnError mode Continue",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -456,13 +456,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			name:          "should return an error when spec hook with wait timeout expires with OnError mode Fail",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -487,13 +487,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			name:          "should return an error when shared hooks context is canceled before spec hook with OnError mode Fail executes",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -519,13 +519,13 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
 			groupResource:  "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -549,23 +549,23 @@ func TestWaitExecHandleHooks(t *testing.T) {
 			name:          "should return no error with 2 spec hooks in 2 different containers, 1st container starts running after 10ms, 2nd container after 20ms, both succeed",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container2",
 				}).
 				// initially both are waiting
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container2",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -602,23 +602,23 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					error: nil,
 					pod: builder.ForPod("default", "my-pod").
 						ObjectMeta(builder.WithResourceVersion("2")).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container2",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						// container 2 is still waiting when the first hook executes in container1
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container2",
-							State: v1.ContainerState{
-								Waiting: &v1.ContainerStateWaiting{},
+							State: corev1api.ContainerState{
+								Waiting: &corev1api.ContainerStateWaiting{},
 							},
 						}).
 						Result(),
@@ -632,22 +632,22 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					error: nil,
 					pod: builder.ForPod("default", "my-pod").
 						ObjectMeta(builder.WithResourceVersion("3")).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container2",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container2",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -659,22 +659,22 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					wait: 10 * time.Millisecond,
 					updated: builder.ForPod("default", "my-pod").
 						ObjectMeta(builder.WithResourceVersion("2")).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container2",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container2",
-							State: v1.ContainerState{
-								Waiting: &v1.ContainerStateWaiting{},
+							State: corev1api.ContainerState{
+								Waiting: &corev1api.ContainerStateWaiting{},
 							},
 						}).
 						Result(),
@@ -684,22 +684,22 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					wait: 10 * time.Millisecond,
 					updated: builder.ForPod("default", "my-pod").
 						ObjectMeta(builder.WithResourceVersion("3")).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container2",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container2",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -732,11 +732,11 @@ func TestWaitExecHandleHooks(t *testing.T) {
 
 			for _, e := range test.expectedExecutions {
 				obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod)
-				assert.NoError(t, err)
+				require.NoError(t, err)
 				podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error)
 			}
 
-			ctx := context.Background()
+			ctx := t.Context()
 			if test.sharedHooksContextTimeout > 0 {
 				var ctxCancel context.CancelFunc
 				ctx, ctxCancel = context.WithTimeout(ctx, test.sharedHooksContextTimeout)
@@ -758,7 +758,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 func TestPodHasContainer(t *testing.T) {
 	tests := []struct {
 		name      string
-		pod       *v1.Pod
+		pod       *corev1api.Pod
 		container string
 		expect    bool
 	}{
@@ -767,7 +767,7 @@ func TestPodHasContainer(t *testing.T) {
 			expect:    true,
 			container: "container1",
 			pod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
 				Result(),
@@ -777,7 +777,7 @@ func TestPodHasContainer(t *testing.T) {
 			expect:    false,
 			container: "container1",
 			pod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container2",
 				}).
 				Result(),
@@ -794,7 +794,7 @@ func TestPodHasContainer(t *testing.T) {
 func TestIsContainerUp(t *testing.T) {
 	tests := []struct {
 		name      string
-		pod       *v1.Pod
+		pod       *corev1api.Pod
 		container string
 		expect    bool
 		hooks     []PodExecRestoreHook
@@ -804,10 +804,10 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    true,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -818,10 +818,10 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    false,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 					Ready: false,
 				}).
@@ -839,10 +839,10 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    true,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 					Ready: true,
 				}).
@@ -860,9 +860,9 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    false,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name:  "container1",
-					State: v1.ContainerState{},
+					State: corev1api.ContainerState{},
 				}).
 				Result(),
 			hooks: []PodExecRestoreHook{},
@@ -872,10 +872,10 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    false,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -886,16 +886,16 @@ func TestIsContainerUp(t *testing.T) {
 			container: "container1",
 			expect:    true,
 			pod: builder.ForPod("default", "my-pod").
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container0",
-					State: v1.ContainerState{
-						Terminated: &v1.ContainerStateTerminated{},
+					State: corev1api.ContainerState{
+						Terminated: &corev1api.ContainerStateTerminated{},
 					},
 				},
-					&v1.ContainerStatus{
+					&corev1api.ContainerStatus{
 						Name: "container1",
-						State: v1.ContainerState{
-							Running: &v1.ContainerStateRunning{},
+						State: corev1api.ContainerState{
+							Running: &corev1api.ContainerStateRunning{},
 						},
 					}).
 				Result(),
@@ -999,34 +999,29 @@ func TestMaxHookWait(t *testing.T) {
 }
 
 func TestRestoreHookTrackerUpdate(t *testing.T) {
-	type change struct {
-		// delta to wait since last change applied or pod added
-		wait    time.Duration
-		updated *v1.Pod
-	}
 	type expectedExecution struct {
 		hook  *velerov1api.ExecHook
 		name  string
 		error error
-		pod   *v1.Pod
+		pod   *corev1api.Pod
 	}
 
 	hookTracker1 := NewMultiHookTracker()
-	hookTracker1.Add("restore1", "default", "my-pod", "container1", HookSourceAnnotation, "<from-annotation>", HookPhase(""))
+	hookTracker1.Add("restore1", "default", "my-pod", "container1", HookSourceAnnotation, "<from-annotation>", HookPhase(""), 0)
 
 	hookTracker2 := NewMultiHookTracker()
-	hookTracker2.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""))
+	hookTracker2.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0)
 
 	hookTracker3 := NewMultiHookTracker()
-	hookTracker3.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""))
-	hookTracker3.Add("restore1", "default", "my-pod", "container2", HookSourceSpec, "my-hook-2", HookPhase(""))
+	hookTracker3.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0)
+	hookTracker3.Add("restore1", "default", "my-pod", "container2", HookSourceSpec, "my-hook-2", HookPhase(""), 0)
 
 	hookTracker4 := NewMultiHookTracker()
-	hookTracker4.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""))
+	hookTracker4.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0)
 
 	tests1 := []struct {
 		name               string
-		initialPod         *v1.Pod
+		initialPod         *corev1api.Pod
 		groupResource      string
 		byContainer        map[string][]PodExecRestoreHook
 		expectedExecutions []expectedExecution
@@ -1043,13 +1038,13 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 					podRestoreHookTimeoutAnnotationKey, "1s",
 					podRestoreHookWaitTimeoutAnnotationKey, "1m",
 				)).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Running: &v1.ContainerStateRunning{},
+					State: corev1api.ContainerState{
+						Running: &corev1api.ContainerStateRunning{},
 					},
 				}).
 				Result(),
@@ -1088,13 +1083,13 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 							podRestoreHookTimeoutAnnotationKey, "1s",
 							podRestoreHookWaitTimeoutAnnotationKey, "1m",
 						)).
-						Containers(&v1.Container{
+						Containers(&corev1api.Container{
 							Name: "container1",
 						}).
-						ContainerStatuses(&v1.ContainerStatus{
+						ContainerStatuses(&corev1api.ContainerStatus{
 							Name: "container1",
-							State: v1.ContainerState{
-								Running: &v1.ContainerStateRunning{},
+							State: corev1api.ContainerState{
+								Running: &corev1api.ContainerStateRunning{},
 							},
 						}).
 						Result(),
@@ -1107,13 +1102,13 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 			name:          "a hook with OnError mode Fail failed to execute",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -1138,13 +1133,13 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 			name:          "a hook with OnError mode Continue failed to execute",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -1169,23 +1164,23 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 			name:          "two hooks with OnError mode Continue failed to execute",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container2",
 				}).
 				// initially both are waiting
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container2",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -1222,13 +1217,13 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 			name:          "a hook was recorded before added to tracker",
 			groupResource: "pods",
 			initialPod: builder.ForPod("default", "my-pod").
-				Containers(&v1.Container{
+				Containers(&corev1api.Container{
 					Name: "container1",
 				}).
-				ContainerStatuses(&v1.ContainerStatus{
+				ContainerStatuses(&corev1api.ContainerStatus{
 					Name: "container1",
-					State: v1.ContainerState{
-						Waiting: &v1.ContainerStateWaiting{},
+					State: corev1api.ContainerState{
+						Waiting: &corev1api.ContainerStateWaiting{},
 					},
 				}).
 				Result(),
@@ -1269,11 +1264,11 @@ func TestRestoreHookTrackerUpdate(t *testing.T) {
 
 			for _, e := range test.expectedExecutions {
 				obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod)
-				assert.NoError(t, err)
+				require.NoError(t, err)
 				podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error)
 			}
 
-			ctx := context.Background()
+			ctx := t.Context()
 			_ = h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer, test.hookTracker, "restore1")
 			_, actualFailed := test.hookTracker.Stat("restore1")
 			assert.Equal(t, test.expectedFailed, actualFailed)

internal/resourcemodifiers/json_merge_patch_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -28,7 +29,7 @@ func TestJsonMergePatchFailure(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			scheme := runtime.NewScheme()
 			err := clientgoscheme.AddToScheme(scheme)
-			assert.NoError(t, err)
+			require.NoError(t, err)
 			pt := &JSONMergePatcher{
 				patches: []JSONMergePatch{{PatchData: tt.data}},
 			}

internal/resourcemodifiers/resource_modifiers.go

@@ -23,7 +23,7 @@ import (
 	"github.com/gobwas/glob"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	v1 "k8s.io/api/core/v1"
+	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
@@ -63,7 +63,7 @@ type ResourceModifiers struct {
 	ResourceModifierRules []ResourceModifierRule `json:"resourceModifierRules"`
 }
 
-func GetResourceModifiersFromConfig(cm *v1.ConfigMap) (*ResourceModifiers, error) {
+func GetResourceModifiersFromConfig(cm *corev1api.ConfigMap) (*ResourceModifiers, error) {
 	if cm == nil {
 		return nil, fmt.Errorf("could not parse config from nil configmap")
 	}

internal/resourcemodifiers/resource_modifiers_test.go

@@ -21,7 +21,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
-	v1 "k8s.io/api/core/v1"
+	"github.com/stretchr/testify/require"
+	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -31,7 +32,7 @@ import (
 )
 
 func TestGetResourceModifiersFromConfig(t *testing.T) {
-	cm1 := &v1.ConfigMap{
+	cm1 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -64,7 +65,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 			},
 		},
 	}
-	cm2 := &v1.ConfigMap{
+	cm2 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -99,7 +100,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm3 := &v1.ConfigMap{
+	cm3 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -109,7 +110,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm4 := &v1.ConfigMap{
+	cm4 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -135,7 +136,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm5 := &v1.ConfigMap{
+	cm5 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -170,7 +171,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm6 := &v1.ConfigMap{
+	cm6 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -199,7 +200,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm7 := &v1.ConfigMap{
+	cm7 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -228,7 +229,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 		},
 	}
 
-	cm8 := &v1.ConfigMap{
+	cm8 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -256,7 +257,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 			},
 		},
 	}
-	cm9 := &v1.ConfigMap{
+	cm9 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -285,7 +286,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 			},
 		},
 	}
-	cm10 := &v1.ConfigMap{
+	cm10 := &corev1api.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-configmap",
 			Namespace: "test-namespace",
@@ -316,7 +317,7 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 	}
 
 	type args struct {
-		cm *v1.ConfigMap
+		cm *corev1api.ConfigMap
 	}
 	tests := []struct {
 		name    string
@@ -429,69 +430,69 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 
 func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 	pvcStandardSc := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "v1",
 			"kind":       "PersistentVolumeClaim",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-pvc",
 				"namespace": "foo",
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"storageClassName": "standard",
 			},
 		},
 	}
 
 	pvcPremiumSc := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "v1",
 			"kind":       "PersistentVolumeClaim",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-pvc",
 				"namespace": "foo",
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"storageClassName": "premium",
 			},
 		},
 	}
 
 	pvcGoldSc := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "v1",
 			"kind":       "PersistentVolumeClaim",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-pvc",
 				"namespace": "foo",
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"storageClassName": "gold",
 			},
 		},
 	}
 
 	deployNginxOneReplica := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "apps/v1",
 			"kind":       "Deployment",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-deployment",
 				"namespace": "foo",
-				"labels": map[string]interface{}{
+				"labels": map[string]any{
 					"app": "nginx",
 				},
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"replicas": int64(1),
-				"template": map[string]interface{}{
-					"metadata": map[string]interface{}{
-						"labels": map[string]interface{}{
+				"template": map[string]any{
+					"metadata": map[string]any{
+						"labels": map[string]any{
 							"app": "nginx",
 						},
 					},
-					"spec": map[string]interface{}{
-						"containers": []interface{}{
-							map[string]interface{}{
+					"spec": map[string]any{
+						"containers": []any{
+							map[string]any{
 								"name":  "nginx",
 								"image": "nginx:latest",
 							},
@@ -502,27 +503,27 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 		},
 	}
 	deployNginxTwoReplica := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "apps/v1",
 			"kind":       "Deployment",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-deployment",
 				"namespace": "foo",
-				"labels": map[string]interface{}{
+				"labels": map[string]any{
 					"app": "nginx",
 				},
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"replicas": int64(2),
-				"template": map[string]interface{}{
-					"metadata": map[string]interface{}{
-						"labels": map[string]interface{}{
+				"template": map[string]any{
+					"metadata": map[string]any{
+						"labels": map[string]any{
 							"app": "nginx",
 						},
 					},
-					"spec": map[string]interface{}{
-						"containers": []interface{}{
-							map[string]interface{}{
+					"spec": map[string]any{
+						"containers": []any{
+							map[string]any{
 								"name":  "nginx",
 								"image": "nginx:latest",
 							},
@@ -533,31 +534,31 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 		},
 	}
 	deployNginxMysql := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "apps/v1",
 			"kind":       "Deployment",
-			"metadata": map[string]interface{}{
+			"metadata": map[string]any{
 				"name":      "test-deployment",
 				"namespace": "foo",
-				"labels": map[string]interface{}{
+				"labels": map[string]any{
 					"app": "nginx",
 				},
 			},
-			"spec": map[string]interface{}{
+			"spec": map[string]any{
 				"replicas": int64(1),
-				"template": map[string]interface{}{
-					"metadata": map[string]interface{}{
-						"labels": map[string]interface{}{
+				"template": map[string]any{
+					"metadata": map[string]any{
+						"labels": map[string]any{
 							"app": "nginx",
 						},
 					},
-					"spec": map[string]interface{}{
-						"containers": []interface{}{
-							map[string]interface{}{
+					"spec": map[string]any{
+						"containers": []any{
+							map[string]any{
 								"name":  "nginx",
 								"image": "nginx:latest",
 							},
-							map[string]interface{}{
+							map[string]any{
 								"name":  "mysql",
 								"image": "mysql:latest",
 							},
@@ -568,19 +569,19 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 		},
 	}
 	cmTrue := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "v1",
 			"kind":       "ConfigMap",
-			"data": map[string]interface{}{
+			"data": map[string]any{
 				"test": "true",
 			},
 		},
 	}
 	cmFalse := &unstructured.Unstructured{
-		Object: map[string]interface{}{
+		Object: map[string]any{
 			"apiVersion": "v1",
 			"kind":       "ConfigMap",
-			"data": map[string]interface{}{
+			"data": map[string]any{
 				"test": "false",
 			},
 		},
@@ -1303,27 +1304,27 @@ func TestResourceModifiers_ApplyResourceModifierRules_StrategicMergePatch(t *tes
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
 	o1, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNFSVolume), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	podWithNFSVolume := o1.(*unstructured.Unstructured)
 
 	o2, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithPVCVolume), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	podWithPVCVolume := o2.(*unstructured.Unstructured)
 
 	o3, _, err := unstructuredSerializer.Decode([]byte(svcYAMLWithPort8000), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	svcWithPort8000 := o3.(*unstructured.Unstructured)
 
 	o4, _, err := unstructuredSerializer.Decode([]byte(svcYAMLWithPort9000), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	svcWithPort9000 := o4.(*unstructured.Unstructured)
 
 	o5, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNginxImage), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	podWithNginxImage := o5.(*unstructured.Unstructured)
 
 	o6, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNginx1Image), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	podWithNginx1Image := o6.(*unstructured.Unstructured)
 
 	tests := []struct {
@@ -1467,15 +1468,15 @@ func TestResourceModifiers_ApplyResourceModifierRules_StrategicMergePatch(t *tes
 func TestResourceModifiers_ApplyResourceModifierRules_JSONMergePatch(t *testing.T) {
 	unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
 	o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToB := o1.(*unstructured.Unstructured)
 
 	o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToC := o2.(*unstructured.Unstructured)
 
 	o3, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithoutLabelA), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithoutLabelA := o3.(*unstructured.Unstructured)
 
 	tests := []struct {
@@ -1618,11 +1619,11 @@ func TestResourceModifiers_ApplyResourceModifierRules_JSONMergePatch(t *testing.
 func TestResourceModifiers_wildcard_in_GroupResource(t *testing.T) {
 	unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
 	o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToB := o1.(*unstructured.Unstructured)
 
 	o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToC := o2.(*unstructured.Unstructured)
 
 	tests := []struct {
@@ -1694,11 +1695,11 @@ func TestResourceModifiers_wildcard_in_GroupResource(t *testing.T) {
 func TestResourceModifiers_conditional_patches(t *testing.T) {
 	unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
 	o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToB := o1.(*unstructured.Unstructured)
 
 	o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	cmWithLabelAToC := o2.(*unstructured.Unstructured)
 
 	tests := []struct {