Add v1.11.1 changelog. (#6522)

Signed-off-by: Xun Jiang <jxun@vmware.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -5,7 +5,7 @@ about: Tell us about a problem you are experiencing
 ---
 
 **What steps did you take and what happened:**
-[A clear and concise description of what the bug is, and what commands you ran.)
+<!--A clear and concise description of what the bug is, and what commands you ran.-->
 
 
 **What did you expect to happen:**
@@ -25,7 +25,7 @@ Please provide the output of the following commands (Pasting long output into a
 
 
 **Anything else you would like to add:**
-[Miscellaneous information that will assist in solving the issue.]
+<!--Miscellaneous information that will assist in solving the issue.-->
 
 
 **Environment:**

.github/ISSUE_TEMPLATE/feature-enhancement-request.md

@@ -5,15 +5,15 @@ about: Suggest an idea for this project
 ---
 
 **Describe the problem/challenge you have**
-[A description of the current limitation/problem/challenge that you are experiencing.]
+<!--A description of the current limitation/problem/challenge that you are experiencing.-->
 
 
 **Describe the solution you'd like**
-[A clear and concise description of what you want to happen.]
+<!--A clear and concise description of what you want to happen.-->
 
 
 **Anything else you would like to add:**
-[Miscellaneous information that will assist in solving the issue.]
+<!--Miscellaneous information that will assist in solving the issue.-->
 
 
 **Environment:**

.github/auto-assignees.yml

@@ -9,16 +9,19 @@ reviewers:
 
   groups:
     maintainers:
-      - dsu-igeek
       - sseago
       - reasonerjt
       - ywk253100
       - blackpiglet
       - qiuming-best
       - shubham-pampattiwar
+      - Lyndon-Li
 
     tech-writer:
-      - a-mccarthy
+      - sseago
+      - reasonerjt
+      - ywk253100
+      - Lyndon-Li
 
 files:
   'site/**':

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  # Dependencies listed in go.mod
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    labels:
+      - "kind/changelog-not-required"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]

.github/workflows/crds-verify-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.8
+          go-version: '1.20.6'
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
@@ -61,6 +61,9 @@ jobs:
           - 1.20.2
           - 1.21.1
           - 1.22.0
+          - 1.23.6
+          - 1.24.2
+          - 1.25.3
     # All steps run in parallel unless otherwise specified.
     # See https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#creating-dependent-jobs
     steps:
@@ -78,7 +81,7 @@ jobs:
             velero-${{ github.event.pull_request.number }}-
       - uses: engineerd/setup-kind@v0.5.0
         with:
-          version: "v0.11.1"
+          version: "v0.17.0"
           image: "kindest/node:v${{ matrix.k8s }}"
       - name: Install CRDs
         run: |

.github/workflows/e2e-test-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.8
+          go-version: '1.20.6'
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
@@ -66,12 +66,13 @@ jobs:
           - 1.22.9
           - 1.23.6
           - 1.24.0
+          - 1.25.3
       fail-fast: false
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.8
+          go-version: '1.20.6'
         id: go
       - name: Check out the code
         uses: actions/checkout@v2
@@ -80,7 +81,7 @@ jobs:
           docker run -d --rm -p 9000:9000 -e "MINIO_ACCESS_KEY=minio" -e "MINIO_SECRET_KEY=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:2021.6.17-debian-10-r7
       - uses: engineerd/setup-kind@v0.5.0
         with:
-          version: "v0.14.0"
+          version: "v0.17.0"
           image: "kindest/node:v${{ matrix.k8s }}"
       - name: Fetch built CLI
         id: cli-cache
@@ -112,6 +113,11 @@ jobs:
           aws_access_key_id=minio
           aws_secret_access_key=minio123
           EOF
+
+          # Match kubectl version to k8s server version
+          curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
           GOPATH=~/go CLOUD_PROVIDER=kind \
               OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
               CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
@@ -125,4 +131,4 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: DebugBundle
-          path: /home/runner/work/velero/velero/test/e2e/debug-bundle*
+          path: /home/runner/work/velero/velero/test/e2e/debug-bundle*

.github/workflows/nightly-trivy-scan.yml

@@ -0,0 +1,36 @@
+name: Trivy Nightly Scan
+on:
+  schedule:
+    - cron: '0 2 * * *' # run at 2 AM UTC
+
+jobs:
+  nightly-scan:
+    name: Trivy nightly scan
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # maintain the versions of Velero those need security scan
+        versions: [main]
+        # list of images that need scan
+        images: [velero, velero-restore-helper]
+    permissions:
+      security-events: write  # for github/codeql-action/upload-sarif to upload SARIF results
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'docker.io/velero/${{ matrix.images }}:${{ matrix.versions }}'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+          format: 'template'
+          template: '@/contrib/sarif.tpl'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'

.github/workflows/opened-issues-triage.yml

@@ -1,15 +0,0 @@
-name: Move new issues into Triage
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  automate-project-columns:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: alex-page/github-project-automation-plus@v0.3.0
-        with:
-          project: "Velero Support Board"
-          column: "New"
-          repo-token: ${{ secrets.GH_TOKEN }}

.github/workflows/pr-changelog-check.yml

@@ -1,5 +1,9 @@
 name: Pull Request Changelog Check
-on: [pull_request]
+# by setting `on: [pull_request]`, that means action will be trigger when PR is opened, synchronize, reopened.
+# Add labeled and unlabeled events too.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
 jobs:
 
   build:

.github/workflows/pr-ci-check.yml

@@ -4,11 +4,13 @@ jobs:
   build:
     name: Run CI
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.8
+          go-version: '1.20.6'
         id: go
       - name: Check out the code
         uses: actions/checkout@v2
@@ -27,3 +29,8 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.out
           verbose: true
+      - name: Run staticcheck
+        uses: dominikh/staticcheck-action@v1.3.0
+        with:
+          version: "2023.1.3"
+          install-go: false

.github/workflows/pr-codespell.yml

@@ -14,7 +14,7 @@ jobs:
       uses: codespell-project/actions-codespell@master
       with:
         # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere.
-        skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./go.sum
-        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot
+        skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./go.sum,./LICENSE
+        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast
         check_filenames: true
         check_hidden: true

.github/workflows/push.yml

@@ -18,11 +18,22 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.19.8
+        go-version: '1.20.6'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+
+    # Fix issue of setup-gcloud
+    - run: |
+        sudo apt-get install python2.7
+        export CLOUDSDK_PYTHON="/usr/bin/python2"
+
+    - uses: google-github-actions/setup-gcloud@v0
+      with:
+        version: '285.0.0'
+        service_account_key: ${{ secrets.GCS_SA_KEY }}
+        export_default_credentials: true
+    - run: gcloud info
 
     - name: Set up QEMU
       id: qemu
@@ -49,22 +60,41 @@ jobs:
         files: coverage.out
         verbose: true
 
-    # Only try to publish the container image from the root repo; forks don't have permission to do so and will always get failures.
-    - name: Publish container image
-      if: github.repository == 'vmware-tanzu/velero'
-      run: |
-        docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }}
-        ./hack/docker-push.sh
-
     # Use the JSON key in secret to login gcr.io
-    - uses: 'docker/login-action@v1'
+    - uses: 'docker/login-action@v2'
       with:
         registry: 'gcr.io' # or REGION.docker.pkg.dev
         username: '_json_key'
         password: '${{ secrets.GCR_SA_KEY }}'
 
-    # Push image to GCR to facilitate some environments that have rate limitation to docker hub, e.g. vSphere.
-    - name: Publish container image to GCR
+    # Only try to publish the container image from the root repo; forks don't have permission to do so and will always get failures.
+    - name: Publish container image
       if: github.repository == 'vmware-tanzu/velero'
       run: |
-        REGISTRY=gcr.io/velero-gcp ./hack/docker-push.sh
+        sudo swapoff -a
+        sudo rm -f /mnt/swapfile
+        docker image prune -a --force
+
+        # Build and push Velero image to docker registry
+        docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }}
+        VERSION=$(./hack/docker-push.sh | grep 'VERSION:' | awk -F: '{print $2}' | xargs)
+
+        # Upload Velero image package to GCS
+        source hack/ci/build_util.sh
+        BIN=velero
+        RESTORE_HELPER_BIN=velero-restore-helper
+        GCS_BUCKET=velero-builds
+        VELERO_IMAGE=${BIN}-${VERSION}
+        VELERO_RESTORE_HELPER_IMAGE=${RESTORE_HELPER_BIN}-${VERSION}
+        VELERO_IMAGE_FILE=${VELERO_IMAGE}.tar.gz
+        VELERO_RESTORE_HELPER_IMAGE_FILE=${VELERO_RESTORE_HELPER_IMAGE}.tar.gz
+        VELERO_IMAGE_BACKUP_FILE=${VELERO_IMAGE}-'build.'${GITHUB_RUN_NUMBER}.tar.gz
+        VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE=${VELERO_RESTORE_HELPER_IMAGE}-'build.'${GITHUB_RUN_NUMBER}.tar.gz
+
+        cp ${VELERO_IMAGE_FILE} ${VELERO_IMAGE_BACKUP_FILE}
+        cp ${VELERO_RESTORE_HELPER_IMAGE_FILE} ${VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE}
+
+        uploader ${VELERO_IMAGE_FILE} ${GCS_BUCKET}
+        uploader ${VELERO_RESTORE_HELPER_IMAGE_FILE} ${GCS_BUCKET}
+        uploader ${VELERO_IMAGE_BACKUP_FILE} ${GCS_BUCKET}
+        uploader ${VELERO_RESTORE_HELPER_IMAGE_BACKUP_FILE} ${GCS_BUCKET}

.gitignore

@@ -46,5 +46,9 @@ _tiltbuild
 tilt-resources/tilt-settings.json
 tilt-resources/velero_v1_backupstoragelocation.yaml
 tilt-resources/deployment.yaml
-tilt-resources/restic.yaml
+tilt-resources/node-agent.yaml
 tilt-resources/cloud
+
+# test generated files
+test/e2e/report.xml
+coverage.out

.goreleaser.yml

@@ -54,3 +54,10 @@ release:
     name: velero
   draft: true
   prerelease: auto
+
+git:
+  # What should be used to sort tags when gathering the current and previous
+  # tags if there are more than one tag in the same commit.
+  #
+  # Default: `-version:refname`
+  tag_sort: -version:creatordate

ADOPTERS.md

@@ -40,7 +40,9 @@ We have integrated our [solution with Velero][11] to provide our customers with
 Kyma [integrates with Velero][41] to effortlessly back up and restore Kyma clusters with all its resources. Velero capabilities allow Kyma users to define and run manual and scheduled backups in order to successfully handle a disaster-recovery scenario.
 
 **[Red Hat][50]**  
-Red Hat has developed the [Cluster Application Migration Tool][51] which uses [Velero and Restic][52] to drive the migration of applications between OpenShift clusters.
+Red Hat has developed 2 operators for the OpenShift platform:
+- [Migration Toolkit for Containers][51] (Crane): This operator uses [Velero and Restic][52] to drive the migration of applications between OpenShift clusters.
+- [OADP (OpenShift API for Data Protection) Operator][53]: This operator sets up and installs Velero on the OpenShift platform, allowing users to backup and restore applications.
 
 **[Dell EMC][70]**  
 For Kubernetes environments, [PowerProtect Data Manager][71] leverages the Container Storage Interface (CSI) framework to take snapshots to back up the persistent data or the data that the application creates e.g. databases. [Dell EMC leverages Velero][72] to backup the namespace configuration files (also known as Namespace meta data) for enterprise grade data protection.
@@ -89,6 +91,7 @@ If you would like to add your logo to a future `Adopters of Velero` section on [
 [50]: https://redhat.com
 [51]: https://github.com/fusor/mig-operator
 [52]: https://github.com/fusor/mig-operator/blob/master/docs/usage/2.md
+[53]: https://github.com/openshift/oadp-operator
 
 [60]: https://banzaicloud.com
 [61]: https://banzaicloud.com/products/pipeline/

Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Velero binary build section
-FROM --platform=$BUILDPLATFORM golang:1.19.8 as velero-builder
+FROM --platform=$BUILDPLATFORM golang:1.20.6-bullseye as velero-builder
 
 ARG GOPROXY
 ARG BIN
@@ -44,7 +44,7 @@ RUN mkdir -p /output/usr/bin && \
     -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN}
 
 # Restic binary build section
-FROM --platform=$BUILDPLATFORM golang:1.19.8-bullseye as restic-builder
+FROM --platform=$BUILDPLATFORM golang:1.20.6-bullseye as restic-builder
 
 ARG BIN
 ARG TARGETOS
@@ -61,11 +61,8 @@ env CGO_ENABLED=0 \
 
 COPY . /go/src/github.com/vmware-tanzu/velero
 
-# Not sure why v1.10 and main branch works without adding executable permission.
-# Only v1.9 has the problem.
 RUN mkdir -p /output/usr/bin && \
     export GOARM=$(echo "${GOARM}" | cut -c2-) && \
-    chmod +x /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh && \
     /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh
 
 # Velero image packing section
@@ -78,3 +75,4 @@ COPY --from=velero-builder /output /
 COPY --from=restic-builder /output /
 
 USER nonroot:nonroot
+

MAINTAINERS.md

@@ -4,15 +4,16 @@
 
 ## Maintainers
 
-| Maintainer | GitHub ID | Affiliation |
-| --------------- | --------- | ----------- |
-| Dave Smith-Uchida | [dsu-igeek](https://github.com/dsu-igeek) | [Kasten](https://github.com/kastenhq/) |
-| Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift)
-| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/)
-| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) |
-| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | [VMware](https://www.github.com/vmware/) |
-| Ming Qiu | [qiuming-best](https://github.com/qiuming-best) | [VMware](https://www.github.com/vmware/) |
-| Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift)
+| Maintainer          | GitHub ID                                                     | Affiliation                               |
+|---------------------|---------------------------------------------------------------|-------------------------------------------|
+| Dave Smith-Uchida   | [dsu-igeek](https://github.com/dsu-igeek)                     | [Kasten](https://github.com/kastenhq/)    |
+| Scott Seago         | [sseago](https://github.com/sseago)                           | [OpenShift](https://github.com/openshift) |
+| Daniel Jiang        | [reasonerjt](https://github.com/reasonerjt)                   | [VMware](https://www.github.com/vmware/)  |
+| Wenkai Yin          | [ywk253100](https://github.com/ywk253100)                     | [VMware](https://www.github.com/vmware/)  |
+| Xun Jiang           | [blackpiglet](https://github.com/blackpiglet)                 | [VMware](https://www.github.com/vmware/)  |
+| Ming Qiu            | [qiuming-best](https://github.com/qiuming-best)               | [VMware](https://www.github.com/vmware/)  |
+| Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift) |
+| Yonghui Li          | [Lyndon-Li](https://github.com/Lyndon-Li)                     | [VMware](https://www.github.com/vmware/)  |
 
 ## Emeritus Maintainers
 * Adnan Abdulhussein ([prydonius](https://github.com/prydonius))
@@ -27,11 +28,12 @@
 
 ## Velero Contributors & Stakeholders
 
-| Feature Area | Lead |
-| ----------------------------- | :---------------------: |
-| Architect | Dave Smith-Uchida (dsu-igeek) |
-| Technical Lead | Daniel Jiang (reasonerjt) |
-| Kubernetes CSI Liaison |  |
-| Deployment |  |
-| Community Management | Orlin Vasilev (OrlinVasilev) |
-| Product Management | Eleanor Millman (eleanor-millman) |
+| Feature Area           |                                         Lead                                         |
+|------------------------|:------------------------------------------------------------------------------------:|
+| Architect              |             Dave Smith-Uchida [dsu-igeek](https://github.com/dsu-igeek)              |
+| Technical Lead         |               Daniel Jiang [reasonerjt](https://github.com/reasonerjt)               |
+| Kubernetes CSI Liaison |                                                                                      |
+| Deployment             |                                                                                      |
+| Community Management   |            Orlin Vasilev [OrlinVasilev](https://github.com/OrlinVasilev)             |
+| Product Management     | Pradeep Kumar Chaturvedi [pradeepkchaturvedi](https://github.com/pradeepkchaturvedi) |
+

Makefile

@@ -22,9 +22,11 @@ PKG := github.com/vmware-tanzu/velero
 
 # Where to push the docker image.
 REGISTRY ?= velero
+GCR_REGISTRY ?= gcr.io/velero-gcp
 
 # Image name
 IMAGE ?= $(REGISTRY)/$(BIN)
+GCR_IMAGE ?= $(GCR_REGISTRY)/$(BIN)
 
 # We allow the Dockerfile to be configurable to enable the use of custom Dockerfiles
 # that pull base images from different registries.
@@ -66,8 +68,10 @@ TAG_LATEST ?= false
 
 ifeq ($(TAG_LATEST), true)
 	IMAGE_TAGS ?= $(IMAGE):$(VERSION) $(IMAGE):latest
+	GCR_IMAGE_TAGS ?= $(GCR_IMAGE):$(VERSION) $(GCR_IMAGE):latest
 else
 	IMAGE_TAGS ?= $(IMAGE):$(VERSION)
+	GCR_IMAGE_TAGS ?= $(GCR_IMAGE):$(VERSION)
 endif
 
 ifeq ($(shell docker buildx inspect 2>/dev/null | awk '/Status/ { print $$2 }'), running)
@@ -82,7 +86,7 @@ see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-v
 endef
 
 # The version of restic binary to be downloaded
-RESTIC_VERSION ?= 0.14.0
+RESTIC_VERSION ?= 0.15.0
 
 CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le
 BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH))
@@ -96,9 +100,6 @@ else
 	GIT_TREE_STATE ?= clean
 endif
 
-# The default linters used by lint and local-lint
-LINTERS ?= "gosec,goconst,gofmt,goimports,unparam"
-
 ###
 ### These variables should not need tweaking.
 ###
@@ -112,17 +113,17 @@ GOPROXY ?= https://proxy.golang.org
 # If you want to build all containers, see the 'all-containers' rule.
 all:
 	@$(MAKE) build
-	@$(MAKE) build BIN=velero-restic-restore-helper
+	@$(MAKE) build BIN=velero-restore-helper
 
 build-%:
 	@$(MAKE) --no-print-directory ARCH=$* build
-	@$(MAKE) --no-print-directory ARCH=$* build BIN=velero-restic-restore-helper
+	@$(MAKE) --no-print-directory ARCH=$* build BIN=velero-restore-helper
 
 all-build: $(addprefix build-, $(CLI_PLATFORMS))
 
 all-containers:
 	@$(MAKE) --no-print-directory container
-	@$(MAKE) --no-print-directory container BIN=velero-restic-restore-helper
+	@$(MAKE) --no-print-directory container BIN=velero-restore-helper
 
 local: build-dirs
 # Add DEBUG=1 to enable debug locally
@@ -163,6 +164,7 @@ shell: build-dirs build-env
 	@# under $GOPATH).
 	@docker run \
 		-e GOFLAGS \
+		-e GOPROXY \
 		-i $(TTY) \
 		--rm \
 		-u $$(id -u):$$(id -g) \
@@ -185,6 +187,7 @@ endif
 	--output=type=$(BUILDX_OUTPUT_TYPE) \
 	--platform $(BUILDX_PLATFORMS) \
 	$(addprefix -t , $(IMAGE_TAGS)) \
+	$(addprefix -t , $(GCR_IMAGE_TAGS)) \
 	--build-arg=GOPROXY=$(GOPROXY) \
 	--build-arg=PKG=$(PKG) \
 	--build-arg=BIN=$(BIN) \
@@ -195,6 +198,12 @@ endif
 	--build-arg=RESTIC_VERSION=$(RESTIC_VERSION) \
 	-f $(VELERO_DOCKERFILE) .
 	@echo "container: $(IMAGE):$(VERSION)"
+ifeq ($(BUILDX_OUTPUT_TYPE)_$(REGISTRY), registry_velero)
+	docker pull $(IMAGE):$(VERSION)
+	rm -f $(BIN)-$(VERSION).tar
+	docker save $(IMAGE):$(VERSION) -o $(BIN)-$(VERSION).tar
+	gzip -f $(BIN)-$(VERSION).tar
+endif
 
 SKIP_TESTS ?=
 test: build-dirs
@@ -214,22 +223,12 @@ endif
 
 lint:
 ifneq ($(SKIP_TESTS), 1)
-	@$(MAKE) shell CMD="-c 'hack/lint.sh $(LINTERS)'"
+	@$(MAKE) shell CMD="-c 'hack/lint.sh'"
 endif
 
 local-lint:
 ifneq ($(SKIP_TESTS), 1)
-	@hack/lint.sh $(LINTERS)
-endif
-
-lint-all:
-ifneq ($(SKIP_TESTS), 1)
-	@$(MAKE) shell CMD="-c 'hack/lint.sh $(LINTERS) true'"
-endif
-
-local-lint-all:
-ifneq ($(SKIP_TESTS), 1)
-	@hack/lint.sh $(LINTERS) true
+	@hack/lint.sh
 endif
 
 update:

README.md

@@ -1,7 +1,7 @@
 ![100]
 
 [![Build Status][1]][2] [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3811/badge)](https://bestpractices.coreinfrastructure.org/projects/3811)
-
+![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/vmware-tanzu/velero)
 
 ## Overview
 
@@ -38,14 +38,12 @@ See [the list of releases][6] to find out about feature changes.
 
 The following is a list of the supported Kubernetes versions for each Velero version.
 
-| Velero version | Expected Kubernetes version compatibility| Tested on Kubernetes version|
-|----------------|--------------------|--------------------|
-| 1.9            | 1.16-latest        |  1.20.5, 1.21.2, 1.22.5, 1.23, and 1.24 |
-| 1.8            | 1.16-latest        |  |
-| 1.6.3-1.7.1    | 1.12-latest        ||
-| 1.60-1.6.2     | 1.12-1.21          ||
-| 1.5            | 1.12-1.21          ||
-| 1.4            | 1.10-1.21          | |
+| Velero version | Expected Kubernetes version compatibility | Tested on Kubernetes version           |
+|----------------|-------------------------------------------|----------------------------------------|
+| 1.11           | 1.18-latest                               | 1.23.10, 1.24.9, 1.25.5, and 1.26.1    |
+| 1.10           | 1.18-latest                               | 1.22.5, 1.23.8, 1.24.6 and 1.25.1      |
+| 1.9            | 1.18-latest                               | 1.20.5, 1.21.2, 1.22.5, 1.23, and 1.24 |
+| 1.8            | 1.18-latest                               |                                        |
 
 Velero supports IPv4, IPv6, and dual stack environments. Support for this was tested against Velero v1.8.
 
@@ -53,6 +51,8 @@ The Velero maintainers are continuously working to expand testing coverage, but
 
 If you are interested in using a different version of Kubernetes with a given Velero version, we'd recommend that you perform testing before installing or upgrading your environment. For full information around capabilities within a release, also see the Velero [release notes](https://github.com/vmware-tanzu/velero/releases) or Kubernetes [release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG). See the Velero [support page](https://velero.io/docs/latest/support-process/) for information about supported versions of Velero.
 
+For each release, Velero maintainers run the test to ensure the upgrade path from n-2 minor release.  For example, before the release of v1.10.x, the test will verify that the backup created by v1.9.x and v1.8.x can be restored using the build to be tagged as v1.10.x.
+
 [1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg
 [2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI"
 [4]: https://github.com/vmware-tanzu/velero/issues

Tiltfile

@@ -7,7 +7,7 @@ k8s_yaml([
     'config/crd/v1/bases/velero.io_downloadrequests.yaml',
     'config/crd/v1/bases/velero.io_podvolumebackups.yaml',
     'config/crd/v1/bases/velero.io_podvolumerestores.yaml',
-    'config/crd/v1/bases/velero.io_resticrepositories.yaml',
+    'config/crd/v1/bases/velero.io_backuprepositories.yaml',
     'config/crd/v1/bases/velero.io_restores.yaml',
     'config/crd/v1/bases/velero.io_schedules.yaml',
     'config/crd/v1/bases/velero.io_serverstatusrequests.yaml',
@@ -17,7 +17,7 @@ k8s_yaml([
 # default values
 settings = {
     "default_registry": "docker.io/velero",
-    "enable_restic": False,
+    "use_node_agent": False,
     "enable_debug": False,
     "debug_continue_on_start": True,  # Continue the velero process by default when in debug mode
     "create_backup_locations": False,
@@ -34,9 +34,9 @@ k8s_yaml(kustomize('tilt-resources'))
 k8s_yaml('tilt-resources/deployment.yaml')
 if settings.get("enable_debug"):
     k8s_resource('velero', port_forwards = '2345')
-    # TODO: Need to figure out how to apply port forwards for all restic pods
-if settings.get("enable_restic"):
-    k8s_yaml('tilt-resources/restic.yaml')
+    # TODO: Need to figure out how to apply port forwards for all node-agent pods
+if settings.get("use_node_agent"):
+    k8s_yaml('tilt-resources/node-agent.yaml')
 if settings.get("create_backup_locations"):
     k8s_yaml('tilt-resources/velero_v1_backupstoragelocation.yaml')
 if settings.get("setup-minio"):
@@ -50,7 +50,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip(
 
 tilt_helper_dockerfile_header = """
 # Tilt image
-FROM golang:1.19.8 as tilt-helper
+FROM golang:1.20.6 as tilt-helper
 
 # Support live reloading with Tilt
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \
@@ -103,7 +103,7 @@ local_resource(
 
 local_resource(
     "restic_binary",
-    cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=linux GOARCH=amd64 RESTIC_VERSION=0.13.1 OUTPUT_DIR=_tiltbuild/restic ./hack/download-restic.sh',
+    cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=linux GOARCH=amd64 GOARM="" RESTIC_VERSION=0.13.1 OUTPUT_DIR=_tiltbuild/restic ./hack/build-restic.sh',
 )
 
 # Note: we need a distro with a bash shell to exec into the Velero container

changelogs/CHANGELOG-1.10.md

@@ -0,0 +1,190 @@
+  ## v1.10.0
+### 2022-11-23
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.10.0
+
+### Container Image
+`velero/velero:v1.10.0`
+
+### Documentation
+https://velero.io/docs/v1.10/
+
+### Upgrading
+https://velero.io/docs/v1.10/upgrade-to-1.10/
+
+### Highlights
+
+#### Unified Repository and Kopia integration
+In this release, we introduced the Unified Repository architecture to build a data path where data movers and the backup repository are decoupled and a unified backup repository could serve various data movement activities.
+
+In this release, we also deeply integrate Velero with Kopia, specifically, Kopia's uploader modules are isolated as a generic file system uploader; Kopia's repository modules are encapsulated as the unified backup repository.
+
+For more information, refer to the [design document](https://github.com/vmware-tanzu/velero/blob/v1.10.0/design/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md).
+
+#### File system backup refactor
+Velero's file system backup (a.k.s. pod volume backup or formerly restic backup) is refactored as the first user of the Unified Repository architecture. Specifically, we added a new path, the Kopia path, besides the existing Restic path. While Restic path is still available and set as default, you can opt in Kopia path by specifying the `uploader-type` parameter at installation time. Meanwhile, you are free to restore from existing backups under either path, Velero dynamically switches to the correct path to process the restore.
+
+Because of the new path, we renamed some modules and parameters, refer to the Break Changes section for more details.
+
+For more information, visit the [file system backup document](https://velero.io/docs/v1.10/file-system-backup/) and [v1.10 upgrade guide document](https://velero.io/docs/v1.10/upgrade-to-1.10/).
+
+Meanwhile, we've created a performance guide for both Restic path and Kopia path, which helps you to choose between the two paths and provides you the best practice to configure them under different scenarios. Please note that the results in the guide are based on our testing environments, you may get different results when testing in your own ones. For more information, visit the [performance guide document](https://velero.io/docs/v1.10/performance-guidance/).
+
+#### Plugin versioning V1 refactor
+In this release, Velero moves plugins BackupItemAction, RestoreItemAction and VolumeSnapshotterAction to version v1, this allows future plugin changes that do not support backward compatibility, so is a preparation for various complex tasks, for example, data movement tasks.
+For more information, refer to the [plugin versioning design document](https://github.com/vmware-tanzu/velero/blob/v1.10.0/design/plugin-versioning.md).
+
+#### Refactor the controllers using Kubebuilder v3
+In this release we continued our code modernization work, rewriting some controllers using Kubebuilder v3. This work is ongoing and we will continue to make progress in future releases.
+
+#### Add credentials to volume snapshot locations
+In this release, we enabled dedicate credentials options to volume snapshot locations so that you can specify credentials per volume snapshot location as same as backup storage location.
+
+For more information, please visit the [locations document](https://velero.io/docs/v1.10/locations/).
+
+#### CSI snapshot enhancements
+In this release we added several changes to enhance the robustness of CSI snapshot procedures, for example, some protection code for error handling, and a mechanism to skip exclusion checks so that CSI snapshot works with various backup resource filters.
+
+#### Backup schedule pause/unpause
+In this release, Velero supports to pause/unpause a backup schedule during or after its creation. Specifically:
+
+At creation time, you can specify `–paused` flag to `velero schedule create` command, if so, you will create a paused schedule that will not run until it is unpaused
+After creation, you can run `velero schedule pause` or `velero schedule unpause` command to pause/unpause a schedule
+
+#### Runtime and dependencies
+In order to fix CVEs, we changed Velero's runtime and dependencies as follows:
+
+Bump go runtime to v1.18.8
+Bump some core dependent libraries to newer versions
+Compile Restic (v0.13.1) with go 1.18.8 instead of packaging the official binary
+
+
+#### Breaking changes
+Due to file system backup refactor, below modules and parameters name have been changed in this release:
+
+`restic` daemonset is renamed to `node-agent`
+`resticRepository` CR is renamed to `backupRepository`
+`velero restic repo` command is renamed to `velero repo`
+`velero-restic-credentials` secret is renamed to `velero-repo-credentials`
+`default-volumes-to-restic` parameter is renamed to `default-volumes-to-fs-backup`
+`restic-timeout` parameter is renamed to `fs-backup-timeout`
+`default-restic-prune-frequency` parameter is renamed to `default-repo-maintain-frequency`
+
+#### Upgrade
+Due to the major changes of file system backup, the old upgrade steps are not suitable any more. For the new upgrade steps, visit [v1.10 upgrade guide document](https://velero.io/docs/v1.10/upgrade-to-1.10/).
+
+#### Limitations/Known issues
+In this release, Kopia backup repository (so the Kopia path of file system backup) doesn't support self signed certificate for S3 compatible storage. To track this problem, refer to this [Velero issue](https://github.com/vmware-tanzu/velero/issues/5123) or [Kopia issue](https://github.com/kopia/kopia/issues/1443). 
+
+Due to the code change in Velero, there will be some code change required in vSphere plugin, without which the functionality may be impacted.  Therefore, if you are using vSphere plugin in your workflow, please hold the upgrade until the issue [#485](https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/485) is fixed in vSphere plugin.
+
+### All changes
+  
+  * Restore ClusterBootstrap before Cluster otherwise a new default ClusterBootstrap object is create for the cluster  (#5616, @ywk253100)
+  * Add compile restic binary for CVE fix  (#5574, @qiuming-best)
+  * Fix controller problematic log output  (#5572, @qiuming-best)
+  * Enhance the restore priorities list to support specifying the low prioritized resources that need to be restored in the last (#5535, @ywk253100)
+  * fix restic backup progress error (#5534, @qiuming-best)
+  * fix restic backup failure with self-signed certification backend storage (#5526, @qiuming-best)
+  * Add credential store in backup deletion controller to support VSL credential. (#5521, @blackpiglet)
+  * Fix issue 5505: the pod volume backups/restores except the first one fail under the kopia path if "AZURE_CLOUD_NAME" is specified (#5512, @Lyndon-Li)
+  * After Pod Volume Backup/Restore refactor, remove all the unreasonable appearance of "restic" word from documents (#5499, @Lyndon-Li)
+  * Refactor Pod Volume Backup/Restore doc to match the new behavior (#5484, @Lyndon-Li)
+  * Remove redundancy code block left by #5388. (#5483, @blackpiglet)
+  * Issue fix 5477: create the common way to support S3 compatible object storages that work for both Restic and Kopia; Keep the resticRepoPrefix parameter for compatibility (#5478, @Lyndon-Li)
+  * Update the k8s.io dependencies to 0.24.0.
+This also required an update to github.com/bombsimon/logrusr/v3.
+Removed the `WithClusterName` method
+as it is a "legacy field that was
+always cleared by the system and never used" as per upstream k8s
+https://github.com/kubernetes/apimachinery/blob/release-1.24/pkg/apis/meta/v1/types.go#L257-L259 (#5471, @kcboyle)
+  * Add v1.10 velero upgrade doc (#5468, @qiuming-best)
+  * Upgrade velero docker image to use go 1.18 and upgrade golangci-lint to 1.45.0 (#5459, @Lyndon-Li)
+  * Add VolumeSnapshot client back. (#5449, @blackpiglet)
+  * Change subcommand `velero restic repo` to `velero repo` (#5446, @allenxu404)
+  * Remove irrational "Restic" names in Velero code after the PVBR refactor (#5444, @Lyndon-Li)
+  * moved RIA execute input/output structs back to velero package (#5441, @sseago)
+  * Rename Velero pod volume restore init helper from "velero-restic-restore-helper" to "velero-restore-helper" (#5432, @Lyndon-Li)
+  * Skip the exclusion check for additional resources returned by BIA (#5429, @reasonerjt)
+  * Change B/R describe CLI to support Kopia (#5412, @allenxu404)
+  * Add nil check before execution of csi snapshot delete (#5401, @shubham-pampattiwar)
+  * update velero using klog to version v2.9.0 (#5396, @blackpiglet)
+  * Fix Test_prepareBackupRequest_BackupStorageLocation UT failure. (#5394, @blackpiglet)
+  * Rename Velero daemonset from "restic" to "node-agent" (#5390, @Lyndon-Li)
+  * Add some corner cases checking for CSI snapshot in backup controller. (#5388, @blackpiglet)
+  * Fix issue 5386: Velero providers a full URL as the S3Url while the underlying minio client only accept the host part of the URL as the endpoint and the schema should be specified separately. (#5387, @Lyndon-Li)
+  * Fix restore error with flag namespace-mappings (#5377, @qiuming-best)
+  * Pod Volume Backup/Restore Refactor: Rename parameters in CRDs and commands to remove "Restic" word (#5370, @Lyndon-Li)
+  * Added backupController's UT to test the prepareBackupRequest() method BackupStorageLocation processing logic (#5362, @niulechuan)
+  * Fix a repoEnsurer problem introduced by the refactor - The repoEnsurer didn't check "" state of BackupRepository, as a result, the function GetBackupRepository always returns without an error even though the ensreReady is specified. (#5359, @Lyndon-Li)
+  * Add E2E test for schedule backup (#5355, @danfengliu)
+  * Add useOwnerReferencesInBackup field doc for schedule. (#5353, @cleverhu)
+  * Clarify the help message for the default value of parameter --snapshot-volumes, when it's not set. (#5350, @blackpiglet)
+  * Fix restore cmd extraflag overwrite bug (#5347, @qiuming-best)
+  * Resolve gopkg.in/yaml.v3 vulnerabilities by upgrading gopkg.in/yaml.v3 to v3.0.1 (#5344, @kaovilai)
+  * Increase ensure restic repository timeout to 5m (#5335, @shubham-pampattiwar)
+  * Add opt-in and opt-out PersistentVolume backup to E2E tests (#5331, @danfengliu)
+  * Cancel downloadRequest when timeout without downloadURL (#5329, @kaovilai)
+  * Fix PVB finds wrong parent snapshot (#5322, @qiuming-best)
+  * Fix issue 4874 and 4752: check the daemonset pod is running in the node where the workload pod resides before running the PVB for the pod (#5319, @Lyndon-Li)
+  * plugin versioning v1 refactor for VolumeSnapshotter (#5318, @sseago)
+  * Change the status of restore to completed from partially failed when restore empty backup (#5314, @allenxu404)
+  * RestoreItemAction v1 refactoring for plugin api versioning (#5312, @sseago)
+  * Refactor the repoEnsurer code to use controller runtime client and wrap some common BackupRepository operations to share with other modules (#5308, @Lyndon-Li)
+  * Remove snapshot related lister, informer and client from backup controller. (#5299, @jxun)
+  * Remove github.com/apex/log logger. (#5297, @blackpiglet)
+  * change CSISnapshotTimeout from pointer to normal variables. (#5294, @cleverhu)
+  * Optimize code for restore exists resources. (#5293, @cleverhu)
+  * Add more detailed comments for labels columns. (#5291, @cleverhu)
+  * Add backup status checking in schedule controller. (#5283, @blackpiglet)
+  * Add changes for problems/enhancements found during smoking test for Kopia pod volume backup/restore (#5282, @Lyndon-Li)
+  * Support pause/unpause schedules (#5279, @ywk253100)
+  *  plugin/clientmgmt refactoring for BackupItemAction v1 (#5271, @sseago)
+  * Don't move velero v1 plugins to new proto dir (#5263, @sseago)
+  * Fill gaps for Kopia path of PVBR: integrate Repo Manager with Unified Repo; pass UploaderType to PVBR backupper and restorer; pass RepositoryType to BackupRepository controller and Repo Ensurer (#5259, @Lyndon-Li)
+  * Add csiSnapshotTimeout for describe backup (#5252, @cleverhu)
+  * equip gc controller with configurable frequency (#5248, @allenxu404)
+  * Fix nil pointer panic when restoring StatefulSets (#5247, @divolgin)
+  * Controller refactor code modifications. (#5241, @jxun)
+  * Fix edge cases for already exists resources (#5239, @shubham-pampattiwar)
+  * Check for empty ns list before checking nslist[0] (#5236, @sseago)
+  * Remove reference to non-existent doc (#5234, @reasonerjt)
+  * Add changes for Kopia Integration: Kopia Lib - method implementation. Add changes to write Kopia Repository logs to Velero log (#5233, @Lyndon-Li)
+  * Add changes for Kopia Integration: Kopia Lib - initialize Kopia repo (#5231, @Lyndon-Li)
+  * Uploader Implementation: Kopia backup and restore (#5221, @qiuming-best)
+  * Migrate backup sync controller from code-generator to kubebuilder. (#5218, @jxun)
+  * check vsc null pointer (#5217, @lilongfeng0902)
+  * Refactor GCController with kubebuilder (#5215, @allenxu404)
+  * Uploader Implementation: Restic backup and restore (#5214, @qiuming-best)
+  * Add parameter "uploader-type" to velero server (#5212, @reasonerjt)
+  * Add annotation "pv.kubernetes.io/migrated-to" for CSI checking. (#5181, @jxun)
+  * Add changes for Kopia Integration: Unified Repository Provider - method implementation (#5179, @Lyndon-Li)
+  * Treat namespaces with exclude label as excludedNamespaces
+Related issue: #2413 (#5178, @allenxu404)
+  * Reduce CRD size. (#5174, @jxun)
+  * Fix restic backups to multiple backup storage locations bug (#5172, @qiuming-best)
+  * Add changes for Kopia Integration: Unified Repository Provider - Repo Password (#5167, @Lyndon-Li)
+  * Skip registering "crd-remap-version" plugin when feature flag "EnableAPIGroupVersions" is set (#5165, @reasonerjt)
+  * Kopia uploader integration on shim progress uploader module (#5163, @qiuming-best)
+  * Add labeled and unlabeled events for PR changelog check action. (#5157, @jxun)
+  * VolumeSnapshotLocation refactor with kubebuilder. (#5148, @jxun)
+  * Delay CA file deletion in PVB controller. (#5145, @jxun)
+  * This commit splits the pkg/restic package into several packages to support Kopia integration works (#5143, @ywk253100)
+  * Kopia Integration: Add the Unified Repository Interface definition. Kopia Integration: Add the changes for Unified Repository storage config. Related Issues; #5076, #5080 (#5142, @Lyndon-Li)
+  * Update the CRD for kopia integration (#5135, @reasonerjt)
+  * Let "make shell xxx" respect GOPROXY (#5128, @reasonerjt)
+  * Modify BackupStoreGetter to avoid BSL spec changes (#5122, @sseago)
+  * Dump stack trace when the plugin server handles panic (#5110, @reasonerjt)
+  * Make CSI snapshot creation timeout configurable. (#5104, @jxun)
+  *  Fix bsl validation bug: the BSL is validated continually and doesn't respect the validation period configured (#5101, @ywk253100)
+  * Exclude "csinodes.storage.k8s.io" and "volumeattachments.storage.k8s.io" from restore by default. (#5064, @jxun)
+  * Move 'velero.io/exclude-from-backup' label string to const (#5053, @niulechuan)
+  * Modify Github actions. (#5052, @jxun)
+  * Fix typo in doc, in https://velero.io/docs/main/restore-reference/ "Restore order" section, "Mamespace" should be "Namespace". (#5051, @niulechuan)
+  * Delete opened issues triage action. (#5041, @jxun)
+  * When spec.RestoreStatus is empty, don't restore status (#5008, @sseago)
+  * Added DownloadTargetKindCSIBackupVolumeSnapshots for retrieving the signed URL to download only the `<backup name>`-csi-volumesnapshots.json.gz  and DownloadTargetKindCSIBackupVolumeSnapshotContents to download only `<backup name>`-csi-volumesnapshotcontents.json.gz in the DownloadRequest CR structure. These files are already present in the backup layout.  (#4980, @anshulahuja98)
+  * Refactor BackupItemAction proto and related code to backupitemaction/v1 package.  This is part of implementation of the plugin version design https://github.com/vmware-tanzu/velero/blob/main/design/plugin-versioning.md (#4943, @phuongatemc)
+  * Unified Repository Design (#4926, @Lyndon-Li)
+  * Add credentials to volume snapshot locations (#4864, @sseago)

changelogs/CHANGELOG-1.11.md

@@ -0,0 +1,156 @@
+## v1.11.1
+### 2023-07-19
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.11.1
+
+### Container Image
+`velero/velero:v1.11.1`
+
+### Documentation
+https://velero.io/docs/v1.11/
+
+### Upgrading
+https://velero.io/docs/v1.11/upgrade-to-1.11/
+
+### All changes
+  * Add support for OpenStack CSI drivers topology keys (#6488, @kayrus)
+  * Enhance the code because of #6297, the return value of GetBucketRegion is not recorded, as a result, when it fails, we have no way to get the cause (#6477, @Lyndon-Li)
+  * Fixed a bug where status.progress is not getting updated for backups. (#6324, @blackpiglet)
+  * Restore Endpoints before Services (#6316, @ywk253100)
+  * Fix issue #6182. If pod is not running, don't treat it as an error, let it go and leave a warning. (#6189, @Lyndon-Li)
+
+
+## v1.11
+### 2023-04-07
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0
+
+### Container Image
+`velero/velero:v1.11.0`
+
+### Documentation
+https://velero.io/docs/v1.11/
+
+### Upgrading
+https://velero.io/docs/v1.11/upgrade-to-1.11/
+
+### Highlights
+
+#### BackupItemAction v2
+This feature implements the BackupItemAction v2. BIA v2 has two new methods: Progress() and Cancel() and modifies the Execute() return value.
+
+The API change is needed to facilitate long-running BackupItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running BackupItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item.
+
+#### RestoreItemAction v2
+This feature implemented the RestoreItemAction v2. RIA v2 has three new methods: Progress(), Cancel(), and AreAdditionalItemsReady(), and it modifies RestoreItemActionExecuteOutput() structure in the RIA return value.
+
+The Progress() and Cancel() methods are needed to facilitate long-running RestoreItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running RestoreItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. The AreAdditionalItemsReady() method is needed to allow plugins to tell Velero to wait until the returned additional items have been restored and are ready for use in the cluster before restoring the current item.
+
+#### Plugin Progress Monitoring
+This is intended as a replacement for the previously-approved Upload Progress Monitoring design ([Upload Progress Monitoring](https://github.com/vmware-tanzu/velero/blob/main/design/upload-progress.md)) to expand the supported use cases beyond snapshot upload to include what was previously called Async Backup/Restore Item Actions.
+
+#### Flexible resource policy that can filter volumes to skip in the backup
+This feature provides a flexible policy to filter volumes in the backup without requiring patching any labels or annotations to the pods or volumes. This policy is configured as k8s ConfigMap and maintained by the users themselves, and it can be extended to more scenarios in the future. By now, the policy rules out volumes from backup depending on the CSI driver, NFS setting, volume size, and StorageClass setting. Please refer to [Resource policies rules](https://velero.io/docs/v1.11/resource-filtering/#resource-policies) for the policy's ConifgMap format. It is not guaranteed to work on unofficial third-party plugins as it may not follow the existing backup workflow code logic of Velero.
+
+#### Resource Filters that can distinguish cluster scope and namespace scope resources
+This feature adds four new resource filters for backup. The new filters are separated into cluster scope and namespace scope. Before this feature, Velero could not filter cluster scope resources precisely. This feature provides the ability and refactors existing resource filter parameters.
+
+#### New parameter in installation to customize the serviceaccount name
+The `velero install` sub-command now includes a new parameter,`--service-account-name`, which allows users to specify the ServiceAccountName for the Velero and node-agent pods. This feature may be particularly useful for users who utilize IRSA (IAM Roles for Service Accounts) in Amazon EKS (Elastic Kubernetes Service)."
+
+#### Add a parameter for setting the Velero server connection with the k8s API server's timeout
+In Velero, some code pieces need to communicate with the k8s API server. Before v1.11, these code pieces used hard-code timeout settings. This feature adds a resource-timeout parameter in the velero server binary to make it configurable.
+
+#### Add resource list in the output of the restore describe command
+Before this feature, Velero restore didn't have a restored resources list as the Velero backup. It's not convenient for users to learn what is restored. This feature adds the resources list and the handling result of the resources (including created, updated, failed, and skipped).
+
+#### Support JSON format output of backup describe command 
+Before the Velero v1.11 release, users could not choose Velero's backup describe command's output format. The command output format is friendly for human reading, but it's not a structured output, and it's not easy for other programs to get information from it. Velero v1.11 adds a JSON format output for the backup describe command.
+
+#### Refactor controllers with controller-runtime
+In v1.11, Backup Controller and Restore controller are refactored with controller-runtime. Till v1.11, all Velero controllers use the controller-runtime framework.
+
+#### Runtime and dependencies
+To fix CVEs and keep pace with Golang, Velero made changes as follows:
+* Bump Golang runtime to v1.19.8.
+* Bump several dependent libraries to new versions.
+* Compile Restic (v0.15.0) with Golang v1.19.8 instead of packaging the official binary.
+
+
+### Breaking changes
+* The Velero CSI plugin now determines whether to restore Volume's data from snapshots on the restore's restorePVs setting. Before v1.11, the CSI plugin doesn't check the restorePVs parameter setting. 
+
+
+### Limitations/Known issues
+* The Flexible resource policy that can filter volumes to skip in the backup is not guaranteed to work on unofficial third-party plugins because the plugins may not follow the existing backup workflow code logic of Velero. The ConfigMap used as the policy is supposed to be maintained by users.
+
+
+### All Changes
+* Ignore not found error during patching managedFields (#6110, @ywk253100)
+* Modify new scope resource filters name. (#6089, @blackpiglet)
+* Make Velero not exits when EnableCSI is on and CSI snapshot not installed (#6062, @blackpiglet)
+* Restore Services before Clusters (#6057, @ywk253100)
+* Fixed backup deletion bug related to async operations (#6041, @sseago)
+* Update Golang version to v1.19 for branch main. (#6039, @blackpiglet)
+* Fix issue #5972, don't assume errorField as error type when dealing with logger.WithError (#6028, @Lyndon-Li)
+* distinguish between New and InProgress operations (#6012, @sseago)
+* Modify golangci.yaml file. Resolve found lint issues. (#6008, @blackpiglet)
+* Remove Reference of itemsnapshotter (#5997, @reasonerjt)
+* minor fixes for backup_operations_controller (#5996, @sseago)
+* RIAv2 async operations controller work (#5993, @sseago)
+* Follow-on fixes for BIAv2 controller work (#5971, @sseago)
+* Refactor backup controller based on the controller-runtime framework. (#5969, @qiuming-best)
+* Fix client wait problem after async operation change, velero backup/restore --wait should check a full list of the terminal status (#5964, @Lyndon-Li)
+* Fix issue #5935, refactor the logics for backup/restore persistent log, so as to remove the contest to gzip writer (#5956, @Lyndon-Li)
+* Switch the base image to distroless/base-nossl-debian11 to reduce the CVE triage efforts (#5939, @ywk253100)
+* Wait for additional items to be ready before restoring current item (#5933, @sseago)
+* Add configurable server setting for default timeouts (#5926, @eemcmullan)
+* Add warning/error result to cmd `velero backup describe` (#5916, @allenxu404)
+* Fix Dependabot alerts. Use 1.18 and 1.19 golang instead of patch image in dockerfile. Add release-1.10 and release-1.9 in Trivy daily scan. (#5911, @blackpiglet)
+* Update client-go to v0.25.6 (#5907, @kaovilai)
+* Limit the concurrent number for backup's VolumeSnapshot operation. (#5900, @blackpiglet)
+* Fix goreleaser issue for resolving tags and updated it's version. (#5899, @anshulahuja98)
+* This is to fix issue 5881, enhance the PVB tracker in two modes, Track and Taken (#5894, @Lyndon-Li)
+* Add labels for velero installed namespace to support PSA. (#5873, @blackpiglet)
+* Add restored resource list in the restore describe command (#5867, @ywk253100)
+* Add a json output to cmd velero backup describe  (#5865, @allenxu404)
+* Make restore controller adopting the controller-runtime framework. (#5864, @blackpiglet)
+* Replace k8s.io/apimachinery/pkg/util/clock with k8s.io/utils/clock (#5859, @hezhizhen)
+* Restore finalizer and managedFields of metadata during the restoration (#5853, @ywk253100)
+* BIAv2 async operations controller work (#5849, @sseago)
+* Add secret restore item action to handle service account token secret (#5843, @ywk253100)
+* Add new resource filters can separate cluster and namespace scope resources. (#5838, @blackpiglet)
+* Correct PVB/PVR Failed Phase patching during startup (#5828, @kaovilai)
+* bump up golang net to fix CVE-2022-41721 (#5812, @Lyndon-Li)
+* Update CRD descriptions for SnapshotVolumes and restorePVs (#5807, @shubham-pampattiwar)
+* Add mapped selected-node existence check (#5806, @blackpiglet)
+* Add option "--service-account-name" to install cmd (#5802, @reasonerjt)
+* Enable staticcheck linter. (#5788, @blackpiglet)
+* Set Kopia IgnoreUnknownTypes in ErrorHandlingPolicy to True for ignoring backup unknown file type (#5786, @qiuming-best)
+* Bump up Restic version to 0.15.0  (#5784, @qiuming-best)
+* Add File system backup related matrics to Grafana dashboard
+  - Add metrics backup_warning_total for record of total warnings
+  - Add metrics backup_last_status for record of last status of the backup  (#5779, @allenxu404)
+* Design for Handling backup of volumes by resources filters (#5773, @qiuming-best)
+* Add PR container build action, which will not push image. Add GOARM parameter. (#5771, @blackpiglet)
+* Fix issue 5458, track pod volume backup until the CR is submitted in case it is skipped half way (#5769, @Lyndon-Li)
+* Fix issue 5226, invalidate the related backup repositories whenever the backup storage info change in BSL (#5768, @Lyndon-Li)
+* Add Restic builder in Dockerfile, and keep the used built Golang image version in accordance with upstream Restic. (#5764, @blackpiglet)
+* Fix issue 5043, after the restore pod is scheduled, check if the node-agent pod is running in the same node. (#5760, @Lyndon-Li)
+* Remove restore controller's redundant client. (#5759, @blackpiglet)
+* Define itemoperations.json format and update DownloadRequest API (#5752, @sseago)
+* Add Trivy nightly scan. (#5740, @jxun)
+* Fix issue 5696, check if the repo is still openable before running the prune and forget operation, if not, try to reconnect the repo (#5715, @Lyndon-Li)
+* Fix error with Restic backup empty volumes (#5713, @qiuming-best)
+* new backup and restore phases to support async plugin operations:
+  - WaitingForPluginOperations
+  - WaitingForPluginOperationsPartiallyFailed (#5710, @sseago)
+* Prevent nil panic on exec restore hooks (#5675, @dymurray)
+* Fix CVEs scanned by trivy (#5653, @qiuming-best)
+* Publish backupresults json to enhance error info during backups. (#5576, @anshulahuja98)
+* RestoreItemAction v2 API implementation (#5569, @sseago)
+* add new RestoreItemAction of "velero.io/change-image-name" to handle the issue mentioned at #5519 (#5540, @wenterjoy)
+* BackupItemAction v2 API implementation (#5442, @sseago)
+* Proposal to separate resource filter into cluster scope and namespace scope (#5333, @blackpiglet)

changelogs/CHANGELOG-1.9.md

@@ -1,159 +1,3 @@
-## v1.9.7
-### 2023-04-14
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.7
-
-### Container Image
-`velero/velero:v1.9.7`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-  * Bump Golang version to v1.19.8 (#6148, @blackpiglet)
-
-## v1.9.6
-### 2023-02-21
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.6
-
-### Container Image
-`velero/velero:v1.9.6`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-  * Bump up Golang version and fix CVEs. (#5884, @blackpiglet)
-  * Add labels for velero installed namespace to support PSA. (#5887, @blackpiglet)
-  * Fix Dockerfile issue. (#5761, @blackpiglet)
-  * Add PR container build action, which will not push image. Add GOARM parameter. (#5777, @blackpiglet)
-  * Correct PVB/PVR Failed Phase patching during startup (#5829, @kaovilai)
-
-## v1.9.5
-### 2022-12-19
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.5
-
-### Container Image
-`velero/velero:v1.9.5`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-  * Add Restic builder in Dockerfile, and keep the used built Golang image version in accordance with upstream Restic. (#5685, @blackpiglet)
-
-## v1.9.4
-### 2022-11-30
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.4
-
-### Container Image
-`velero/velero:v1.9.4`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-  * Fix CVE for trivy scan (#5642, @qiuming-best)
-  * Remove old kubernetes versions from kind CI (#5627, @Lyndon-Li))
-  * Restore ClusterBootstrap before Cluster (#5617, @ywk253100)
-
-## v1.9.3
-### 2022-11-03
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.3
-
-### Container Image
-`velero/velero:v1.9.3`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-  * Fix controller problematic log output (#5570, @qiuming-best) 
-  * Add compile restic binary for CVE fix (#5564, @qiuming-best) 
-  * Bump up golang version to 1.18.8 (#5558, @qiuming-best) 
-  * Enhance the restore priorities list to support specifying the low prioritized resources that need to be restored in the last (#5529, @ywk253100)
-  * Fix v1.9.3 CSI VolumeSnapshot status duplicate issue. (#5518, @blackpiglet)
-  * Bump up the distroless image to the latest version (#5500, @ywk253100)
-  * Add some corner cases checking for CSI snapshot in backup controller. (#5482, @blackpiglet)
-  * Skip the exclusion check for additional resources returned by BIA (#5406, @reasonerjt)
-  * Exclude "csinodes.storage.k8s.io" and "volumeattachments.storage.k8s.io" from restore by default. (#5448, @jxun)
-  * Update the k8s.io dependencies to 0.24.0 and Removed the `WithClusterName` method as it is a "legacy field that was always cleared by the system and never used" as per upstream k8s. (#5472, @kcboyle)
-
-## v1.9.2
-### 2022-09-14
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.2
-
-### Container Image
-`velero/velero:v1.9.2`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-
-  * Fix CVE-2022-1962 by bumping up golang version to 1.17.13 (#5286, @qiuming-best)
-  * Fix code spell check fail (#5300, @qiuming-best)
-  * Fix nil pointer panic when restoring StatefulSets (#5301, @divolgin)
-  * Check for empty ns list before checking nslist[0] (#5302, @sseago)
-  * check vsc null pointer (#5303, @lilongfeng0902)
-  * Fix edge cases for already exists resources (#5304, @shubham-pampattiwar)
-  * Increase ensure restic repository timeout to 5m (#5336, @shubham-pampattiwar)
-  * Added DownloadTargetKindCSIBackupVolumeSnapshots for retrieving the signed URL to download only the `<backup name>`-csi-volumesnapshots.json.gz  and DownloadTargetKindCSIBackupVolumeSnapshotContents to download only `<backup name>`-csi-volumesnapshotcontents.json.gz in the DownloadRequest CR structure. These files are already present in the backup layout. (#5307, @anshulahuja98)
-## v1.9.1
-### 2022-08-03
-
-### Download
-https://github.com/vmware-tanzu/velero/releases/tag/v1.9.1
-
-### Container Image
-`velero/velero:v1.9.1`
-
-### Documentation
-https://velero.io/docs/v1.9/
-
-### Upgrading
-https://velero.io/docs/v1.9/upgrade-to-1.9/
-
-### All changes
-
-  * Fix bsl validation bug: the BSL is validated continually and doesn't respect the validation period configured (#5112, @ywk253100)
-  * Modify BackupStoreGetter to avoid BSL spec changes (#5134, @sseago)
-  * Delay CA file deletion in PVB controller. (#5150, @jxun)
-  * Skip registering "crd-remap-version" plugin when feature flag "EnableAPIGroupVersions" is set (#5173, @reasonerjt)
-  * Fix restic backups to multiple backup storage locations bug (#5175, @qiuming-best)
-  * Make CSI snapshot creation timeout configurable. (#5189, @jxun)
-  * Add annotation "pv.kubernetes.io/migrated-to" for CSI checking. (#5186, @jxun)
-  * Bump up base image and package version to fix CVEs. (#5202, @ywk253100)
-
 ## v1.9.0
 ### 2022-06-13
 
@@ -258,4 +102,3 @@ With bumping up the API to v1 in CSI plugin, the v0.3.0 CSI plugin will only wor
   * Fix E2E test [Backups][Deletion][Restic] on GCP. (#4968, @jxun)
   * Disable status as sub resource in CRDs (#4972, @ywk253100)
   * Add more information for failing to get path or snapshot in restic backup and restore. (#4988, @jxun)
-  * When spec.RestoreStatus is empty, don't restore status (#5015, @sseago)

changelogs/unreleased/6775-blackpiglet

@@ -1 +0,0 @@
-Add PSA audit and warn labels.

cmd/velero-restore-helper/velero-restore-helper.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"time"
@@ -34,18 +33,16 @@ func main() {
 	defer ticker.Stop()
 
 	for {
-		select {
-		case <-ticker.C:
-			if done() {
-				fmt.Println("All restic restores are done")
-				err := removeFolder()
-				if err != nil {
-					fmt.Println(err)
-				} else {
-					fmt.Println("Done cleanup .velero folder")
-				}
-				return
+		<-ticker.C
+		if done() {
+			fmt.Println("All restic restores are done")
+			err := removeFolder()
+			if err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Println("Done cleanup .velero folder")
 			}
+			return
 		}
 	}
 }
@@ -54,7 +51,7 @@ func main() {
 // within the .velero/ subdirectory whose name is equal to os.Args[1], or
 // false otherwise
 func done() bool {
-	children, err := ioutil.ReadDir("/restores")
+	children, err := os.ReadDir("/restores")
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "ERROR reading /restores directory: %s\n", err)
 		return false
@@ -84,7 +81,7 @@ func done() bool {
 
 // remove .velero folder
 func removeFolder() error {
-	children, err := ioutil.ReadDir("/restores")
+	children, err := os.ReadDir("/restores")
 	if err != nil {
 		return err
 	}

cmd/velero/velero.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/velero"

config/crd/v1/bases/velero.io_backuprepositories.yaml

@@ -6,20 +6,23 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
-  name: resticrepositories.velero.io
+  name: backuprepositories.velero.io
 spec:
   group: velero.io
   names:
-    kind: ResticRepository
-    listKind: ResticRepositoryList
-    plural: resticrepositories
-    singular: resticrepository
+    kind: BackupRepository
+    listKind: BackupRepositoryList
+    plural: backuprepositories
+    singular: backuprepository
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .spec.repositoryType
+      name: Repository Type
+      type: string
     name: v1
     schema:
       openAPIV3Schema:
@@ -37,7 +40,7 @@ spec:
           metadata:
             type: object
           spec:
-            description: ResticRepositorySpec is the specification for a ResticRepository.
+            description: BackupRepositorySpec is the specification for a BackupRepository.
             properties:
               backupStorageLocation:
                 description: BackupStorageLocation is the name of the BackupStorageLocation
@@ -47,12 +50,19 @@ spec:
                 description: MaintenanceFrequency is how often maintenance should
                   be run.
                 type: string
+              repositoryType:
+                description: RepositoryType indicates the type of the backend repository
+                enum:
+                - kopia
+                - restic
+                - ""
+                type: string
               resticIdentifier:
                 description: ResticIdentifier is the full restic-compatible string
                   for identifying this repository.
                 type: string
               volumeNamespace:
-                description: VolumeNamespace is the namespace this restic repository
+                description: VolumeNamespace is the namespace this backup repository
                   contains pod volume backups for.
                 type: string
             required:
@@ -62,7 +72,7 @@ spec:
             - volumeNamespace
             type: object
           status:
-            description: ResticRepositoryStatus is the current status of a ResticRepository.
+            description: BackupRepositoryStatus is the current status of a BackupRepository.
             properties:
               lastMaintenanceTime:
                 description: LastMaintenanceTime is the last time maintenance was
@@ -72,10 +82,10 @@ spec:
                 type: string
               message:
                 description: Message is a message about the current status of the
-                  ResticRepository.
+                  BackupRepository.
                 type: string
               phase:
-                description: Phase is the current state of the ResticRepository.
+                description: Phase is the current state of the BackupRepository.
                 enum:
                 - New
                 - Ready

config/crd/v1/bases/velero.io_backups.yaml

@@ -42,10 +42,36 @@ spec:
                   CSI VolumeSnapshot status turns to ReadyToUse during creation, before
                   returning error as timeout. The default value is 10 minute.
                 type: string
-              defaultVolumesToRestic:
-                description: DefaultVolumesToRestic specifies whether restic should
-                  be used to take a backup of all pod volumes by default.
+              defaultVolumesToFsBackup:
+                description: DefaultVolumesToFsBackup specifies whether pod volume
+                  file system backup should be used for all volumes by default.
+                nullable: true
                 type: boolean
+              defaultVolumesToRestic:
+                description: "DefaultVolumesToRestic specifies whether restic should
+                  be used to take a backup of all pod volumes by default. \n Deprecated:
+                  this field is no longer used and will be removed entirely in future.
+                  Use DefaultVolumesToFsBackup instead."
+                nullable: true
+                type: boolean
+              excludedClusterScopedResources:
+                description: ExcludedClusterScopedResources is a slice of cluster-scoped
+                  resource type names to exclude from the backup. If set to "*", all
+                  cluster-scoped resource types are excluded. The default value is
+                  empty.
+                items:
+                  type: string
+                nullable: true
+                type: array
+              excludedNamespaceScopedResources:
+                description: ExcludedNamespaceScopedResources is a slice of namespace-scoped
+                  resource type names to exclude from the backup. If set to "*", all
+                  namespace-scoped resource types are excluded. The default value
+                  is empty.
+                items:
+                  type: string
+                nullable: true
+                type: array
               excludedNamespaces:
                 description: ExcludedNamespaces contains a list of namespaces that
                   are not included in the backup.
@@ -251,6 +277,23 @@ spec:
                   resources should be included for consideration in the backup.
                 nullable: true
                 type: boolean
+              includedClusterScopedResources:
+                description: IncludedClusterScopedResources is a slice of cluster-scoped
+                  resource type names to include in the backup. If set to "*", all
+                  cluster-scoped resource types are included. The default value is
+                  empty, which means only related cluster-scoped resources are included.
+                items:
+                  type: string
+                nullable: true
+                type: array
+              includedNamespaceScopedResources:
+                description: IncludedNamespaceScopedResources is a slice of namespace-scoped
+                  resource type names to include in the backup. The default value
+                  is "*".
+                items:
+                  type: string
+                nullable: true
+                type: array
               includedNamespaces:
                 description: IncludedNamespaces is a slice of namespace names to include
                   objects from. If empty, all namespaces are included.
@@ -265,6 +308,11 @@ spec:
                   type: string
                 nullable: true
                 type: array
+              itemOperationTimeout:
+                description: ItemOperationTimeout specifies the time used to wait
+                  for asynchronous BackupItemAction operations The default value is
+                  1 hour.
+                type: string
               labelSelector:
                 description: LabelSelector is a metav1.LabelSelector to filter with
                   when adding individual objects to the backup. If empty or nil, all
@@ -378,14 +426,35 @@ spec:
                 additionalProperties:
                   type: string
                 description: OrderedResources specifies the backup order of resources
-                  of specific Kind. The map key is the Kind name and value is a list
-                  of resource names separated by commas. Each resource name has format
-                  "namespace/resourcename".  For cluster resources, simply use "resourcename".
+                  of specific Kind. The map key is the resource name and value is
+                  a list of object names separated by commas. Each resource name has
+                  format "namespace/objectname".  For cluster resources, simply use
+                  "objectname".
                 nullable: true
                 type: object
+              resourcePolicy:
+                description: ResourcePolicy specifies the referenced resource policies
+                  that backup should follow
+                properties:
+                  apiGroup:
+                    description: APIGroup is the group for the resource being referenced.
+                      If APIGroup is not specified, the specified Kind must be in
+                      the core API group. For any other third-party types, APIGroup
+                      is required.
+                    type: string
+                  kind:
+                    description: Kind is the type of resource being referenced
+                    type: string
+                  name:
+                    description: Name is the name of resource being referenced
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               snapshotVolumes:
-                description: SnapshotVolumes specifies whether to take cloud snapshots
-                  of any PV's referenced in the set of objects included in the Backup.
+                description: SnapshotVolumes specifies whether to take snapshots of
+                  any PV's referenced in the set of objects included in the Backup.
                 nullable: true
                 type: boolean
               storageLocation:
@@ -406,6 +475,20 @@ spec:
           status:
             description: BackupStatus captures the current status of a Velero backup.
             properties:
+              backupItemOperationsAttempted:
+                description: BackupItemOperationsAttempted is the total number of
+                  attempted async BackupItemAction operations for this backup.
+                type: integer
+              backupItemOperationsCompleted:
+                description: BackupItemOperationsCompleted is the total number of
+                  successfully completed async BackupItemAction operations for this
+                  backup.
+                type: integer
+              backupItemOperationsFailed:
+                description: BackupItemOperationsFailed is the total number of async
+                  BackupItemAction operations for this backup which ended with an
+                  error.
+                type: integer
               completionTimestamp:
                 description: CompletionTimestamp records the time a backup was completed.
                   Completion time is recorded even on failed backups. Completion time
@@ -446,6 +529,10 @@ spec:
                 - New
                 - FailedValidation
                 - InProgress
+                - WaitingForPluginOperations
+                - WaitingForPluginOperationsPartiallyFailed
+                - Finalizing
+                - FinalizingPartiallyFailed
                 - Completed
                 - PartiallyFailed
                 - Failed

config/crd/v1/bases/velero.io_downloadrequests.yaml

@@ -46,10 +46,13 @@ spec:
                     - BackupLog
                     - BackupContents
                     - BackupVolumeSnapshots
-                    - BackupItemSnapshots
+                    - BackupItemOperations
                     - BackupResourceList
+                    - BackupResults
                     - RestoreLog
                     - RestoreResults
+                    - RestoreResourceList
+                    - RestoreItemOperations
                     - CSIBackupVolumeSnapshots
                     - CSIBackupVolumeSnapshotContents
                     type: string

config/crd/v1/bases/velero.io_podvolumebackups.yaml

@@ -37,9 +37,13 @@ spec:
       jsonPath: .spec.volume
       name: Volume
       type: string
-    - description: Restic repository identifier for this backup
+    - description: Backup repository identifier for this backup
       jsonPath: .spec.repoIdentifier
-      name: Restic Repo
+      name: Repository ID
+      type: string
+    - description: The type of the uploader to handle data transfer
+      jsonPath: .spec.uploaderType
+      name: Uploader Type
       type: string
     - description: Name of the Backup Storage Location where this backup should be
         stored
@@ -70,7 +74,7 @@ spec:
             properties:
               backupStorageLocation:
                 description: BackupStorageLocation is the name of the backup storage
-                  location where the restic repository is stored.
+                  location where the backup repository is stored.
                 type: string
               node:
                 description: Node is the name of the node that the Pod is running
@@ -114,7 +118,7 @@ spec:
                     type: string
                 type: object
               repoIdentifier:
-                description: RepoIdentifier is the restic repository identifier.
+                description: RepoIdentifier is the backup repository identifier.
                 type: string
               tags:
                 additionalProperties:
@@ -122,6 +126,14 @@ spec:
                 description: Tags are a map of key-value pairs that should be applied
                   to the volume backup as tags.
                 type: object
+              uploaderType:
+                description: UploaderType is the type of the uploader to handle the
+                  data transfer.
+                enum:
+                - kopia
+                - restic
+                - ""
+                type: string
               volume:
                 description: Volume is the name of the volume within the Pod to be
                   backed up.

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -25,6 +25,10 @@ spec:
       jsonPath: .spec.pod.name
       name: Pod
       type: string
+    - description: The type of the uploader to handle data transfer
+      jsonPath: .spec.uploaderType
+      name: Uploader Type
+      type: string
     - description: Name of the volume to be restored
       jsonPath: .spec.volume
       name: Volume
@@ -67,7 +71,7 @@ spec:
             properties:
               backupStorageLocation:
                 description: BackupStorageLocation is the name of the backup storage
-                  location where the restic repository is stored.
+                  location where the backup repository is stored.
                 type: string
               pod:
                 description: Pod is a reference to the pod containing the volume to
@@ -107,11 +111,23 @@ spec:
                     type: string
                 type: object
               repoIdentifier:
-                description: RepoIdentifier is the restic repository identifier.
+                description: RepoIdentifier is the backup repository identifier.
                 type: string
               snapshotID:
                 description: SnapshotID is the ID of the volume snapshot to be restored.
                 type: string
+              sourceNamespace:
+                description: SourceNamespace is the original namespace for namaspace
+                  mapping.
+                type: string
+              uploaderType:
+                description: UploaderType is the type of the uploader to handle the
+                  data transfer.
+                enum:
+                - kopia
+                - restic
+                - ""
+                type: string
               volume:
                 description: Volume is the name of the volume within the Pod to be
                   restored.
@@ -121,6 +137,7 @@ spec:
             - pod
             - repoIdentifier
             - snapshotID
+            - sourceNamespace
             - volume
             type: object
           status:

config/crd/v1/bases/velero.io_schedules.yaml

@@ -32,6 +32,9 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .spec.paused
+      name: Paused
+      type: boolean
     name: v1
     schema:
       openAPIV3Schema:
@@ -53,6 +56,9 @@ spec:
           spec:
             description: ScheduleSpec defines the specification for a Velero schedule
             properties:
+              paused:
+                description: Paused specifies whether the schedule is paused or not
+                type: boolean
               schedule:
                 description: Schedule is a Cron expression defining when to run the
                   Backup.
@@ -66,10 +72,36 @@ spec:
                       for CSI VolumeSnapshot status turns to ReadyToUse during creation,
                       before returning error as timeout. The default value is 10 minute.
                     type: string
-                  defaultVolumesToRestic:
-                    description: DefaultVolumesToRestic specifies whether restic should
-                      be used to take a backup of all pod volumes by default.
+                  defaultVolumesToFsBackup:
+                    description: DefaultVolumesToFsBackup specifies whether pod volume
+                      file system backup should be used for all volumes by default.
+                    nullable: true
                     type: boolean
+                  defaultVolumesToRestic:
+                    description: "DefaultVolumesToRestic specifies whether restic
+                      should be used to take a backup of all pod volumes by default.
+                      \n Deprecated: this field is no longer used and will be removed
+                      entirely in future. Use DefaultVolumesToFsBackup instead."
+                    nullable: true
+                    type: boolean
+                  excludedClusterScopedResources:
+                    description: ExcludedClusterScopedResources is a slice of cluster-scoped
+                      resource type names to exclude from the backup. If set to "*",
+                      all cluster-scoped resource types are excluded. The default
+                      value is empty.
+                    items:
+                      type: string
+                    nullable: true
+                    type: array
+                  excludedNamespaceScopedResources:
+                    description: ExcludedNamespaceScopedResources is a slice of namespace-scoped
+                      resource type names to exclude from the backup. If set to "*",
+                      all namespace-scoped resource types are excluded. The default
+                      value is empty.
+                    items:
+                      type: string
+                    nullable: true
+                    type: array
                   excludedNamespaces:
                     description: ExcludedNamespaces contains a list of namespaces
                       that are not included in the backup.
@@ -280,6 +312,24 @@ spec:
                       resources should be included for consideration in the backup.
                     nullable: true
                     type: boolean
+                  includedClusterScopedResources:
+                    description: IncludedClusterScopedResources is a slice of cluster-scoped
+                      resource type names to include in the backup. If set to "*",
+                      all cluster-scoped resource types are included. The default
+                      value is empty, which means only related cluster-scoped resources
+                      are included.
+                    items:
+                      type: string
+                    nullable: true
+                    type: array
+                  includedNamespaceScopedResources:
+                    description: IncludedNamespaceScopedResources is a slice of namespace-scoped
+                      resource type names to include in the backup. The default value
+                      is "*".
+                    items:
+                      type: string
+                    nullable: true
+                    type: array
                   includedNamespaces:
                     description: IncludedNamespaces is a slice of namespace names
                       to include objects from. If empty, all namespaces are included.
@@ -294,6 +344,11 @@ spec:
                       type: string
                     nullable: true
                     type: array
+                  itemOperationTimeout:
+                    description: ItemOperationTimeout specifies the time used to wait
+                      for asynchronous BackupItemAction operations The default value
+                      is 1 hour.
+                    type: string
                   labelSelector:
                     description: LabelSelector is a metav1.LabelSelector to filter
                       with when adding individual objects to the backup. If empty
@@ -407,14 +462,34 @@ spec:
                     additionalProperties:
                       type: string
                     description: OrderedResources specifies the backup order of resources
-                      of specific Kind. The map key is the Kind name and value is
-                      a list of resource names separated by commas. Each resource
-                      name has format "namespace/resourcename".  For cluster resources,
-                      simply use "resourcename".
+                      of specific Kind. The map key is the resource name and value
+                      is a list of object names separated by commas. Each resource
+                      name has format "namespace/objectname".  For cluster resources,
+                      simply use "objectname".
                     nullable: true
                     type: object
+                  resourcePolicy:
+                    description: ResourcePolicy specifies the referenced resource
+                      policies that backup should follow
+                    properties:
+                      apiGroup:
+                        description: APIGroup is the group for the resource being
+                          referenced. If APIGroup is not specified, the specified
+                          Kind must be in the core API group. For any other third-party
+                          types, APIGroup is required.
+                        type: string
+                      kind:
+                        description: Kind is the type of resource being referenced
+                        type: string
+                      name:
+                        description: Name is the name of resource being referenced
+                        type: string
+                    required:
+                    - kind
+                    - name
+                    type: object
                   snapshotVolumes:
-                    description: SnapshotVolumes specifies whether to take cloud snapshots
+                    description: SnapshotVolumes specifies whether to take snapshots
                       of any PV's referenced in the set of objects included in the
                       Backup.
                     nullable: true

config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml

@@ -13,6 +13,8 @@ spec:
     kind: VolumeSnapshotLocation
     listKind: VolumeSnapshotLocationList
     plural: volumesnapshotlocations
+    shortNames:
+    - vsl
     singular: volumesnapshotlocation
   scope: Namespaced
   versions:
@@ -43,6 +45,24 @@ spec:
                   type: string
                 description: Config is for provider-specific configuration fields.
                 type: object
+              credential:
+                description: Credential contains the credential information intended
+                  to be used with this location
+                properties:
+                  key:
+                    description: The key of the secret to select from.  Must be a
+                      valid secret key.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                required:
+                - key
+                type: object
               provider:
                 description: Provider is the provider of the volume storage.
                 type: string

config/rbac/role.yaml

@@ -24,6 +24,26 @@ rules:
   - pods
   verbs:
   - get
+- apiGroups:
+  - velero.io
+  resources:
+  - backuprepositories
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - velero.io
+  resources:
+  - backuprepositories/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - velero.io
   resources:
@@ -31,6 +51,19 @@ rules:
   verbs:
   - create
   - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - velero.io
+  resources:
+  - backups/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - velero.io
   resources:
@@ -134,7 +167,7 @@ rules:
 - apiGroups:
   - velero.io
   resources:
-  - resticrepositories
+  - restores
   verbs:
   - create
   - delete
@@ -146,11 +179,17 @@ rules:
 - apiGroups:
   - velero.io
   resources:
-  - resticrepositories/status
+  - restores/status
   verbs:
   - get
   - patch
   - update
+- apiGroups:
+  - velero.io
+  resources:
+  - restorestoragelocations
+  verbs:
+  - get
 - apiGroups:
   - velero.io
   resources:
@@ -191,3 +230,15 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - velero.io
+  resources:
+  - volumesnapshotlocations
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

design/CLI/PoC/overlays/plugins/node-agent.yaml

@@ -5,22 +5,22 @@ metadata:
   creationTimestamp: null
   labels:
     component: velero
-  name: restic
+  name: node-agent
   namespace: velero
 spec:
   selector:
     matchLabels:
-      name: restic
+      name: node-agent
   template:
     metadata:
       creationTimestamp: null
       labels:
         component: velero
-        name: restic
+        name: node-agent
     spec:
       containers:
         - args:
-            - restic
+            - node-agent
             - server
           command:
             - /velero
@@ -43,7 +43,7 @@ spec:
               value: /credentials/cloud
           image: velero/velero:latest
           imagePullPolicy: Always
-          name: restic
+          name: node-agent
           resources: {}
           volumeMounts:
             - mountPath: /host_pods

design/Implemented/biav2-design.md

@@ -0,0 +1,103 @@
+# Design for BackupItemAction v2 API
+
+## Abstract
+This design includes the changes to the BackupItemAction (BIA) api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature.
+The BIA v2 interface will have two new methods, and the Execute() return signature will be modified.
+If there are any additional BIA API changes that are needed in the same Velero release cycle as this change, those can be added here as well.
+
+## Background
+This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns.
+It is an optional feature, so plugins which don't need this feature can simply return an empty operation ID and the new methods can be no-ops.
+This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc.
+
+## Goals
+- Allow for BIA Execute() to optionally initiate a long-running operation and report on operation status.
+
+## Non Goals
+- Allowing velero control over when the long-running operation begins.
+
+
+## High-Level Design
+As per the [Plugin Versioning](plugin-versioning.md) design, a new BIAv2 plugin `.proto` file will be created to define the GRPC interface.
+v2 go files will also be created in `plugin/clientmgmt/backupitemaction` and `plugin/framework/backupitemaction`, and a new PluginKind will be created.
+The velero Backup process will be modified to reference v2 plugins instead of v1 plugins.
+An adapter will be created so that any existing BIA v1 plugin can be executed as a v2 plugin when executing a backup.
+
+## Detailed Design
+
+### proto changes (compiled into golang by protoc)
+
+The v2 BackupItemAction.proto will be like the current v1 version with the following changes:
+ExecuteResponse gets a new field:
+```
+message ExecuteResponse {
+    bytes item = 1;
+    repeated generated.ResourceIdentifier additionalItems = 2;
+    string operationID = 3;
+    repeated generated.ResourceIdentifier itemsToUpdate = 4;
+}
+```
+The BackupItemAction service gets two new rpc methods:
+```
+service BackupItemAction {
+    rpc AppliesTo(BackupItemActionAppliesToRequest) returns (BackupItemActionAppliesToResponse);
+    rpc Execute(ExecuteRequest) returns (ExecuteResponse);
+    rpc Progress(BackupItemActionProgressRequest) returns (BackupItemActionProgressResponse);
+    rpc Cancel(BackupItemActionCancelRequest) returns (google.protobuf.Empty);
+}
+```
+To support these new rpc methods, we define new request/response message types:
+```
+message BackupItemActionProgressRequest {
+    string plugin = 1;
+    string operationID = 2;
+    bytes backup = 3;
+}
+
+message BackupItemActionProgressResponse {
+    generated.OperationProgress progress = 1;
+}
+
+message BackupItemActionCancelRequest {
+    string plugin = 1;
+    string operationID = 2;
+    bytes backup = 3;
+}
+
+```
+One new shared message type will be added, as this will also be needed for v2 RestoreItemAction and VolmeSnapshotter:
+```
+message OperationProgress {
+    bool completed = 1;
+    string err = 2;
+    int64 nCompleted = 3;
+    int64 nTotal = 4;
+    string operationUnits = 5;
+    string description = 6;
+    google.protobuf.Timestamp started = 7;
+    google.protobuf.Timestamp updated = 8;
+}
+```
+
+In addition to the two new rpc methods added to the BackupItemAction interface, there is also a new `Name()` method. This one is only actually used internally by Velero to get the name that the plugin was registered with, but it still must be defined in a plugin which implements BackupItemActionV2 in order to implement the interface. It doesn't really matter what it returns, though, as this particular method is not delegated to the plugin via RPC calls. The new (and modified) interface methods for `BackupItemAction` are as follows:
+```
+type BackupItemAction interface {
+...
+	Name() string
+...
+	Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error)
+	Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error)
+	Cancel(operationID string, backup *api.Backup) error
+...
+}
+```
+
+A new PluginKind, `BackupItemActionV2`, will be created, and the backup process will be modified to use this plugin kind.
+See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc.
+
+
+## Compatibility
+The included v1 adapter will allow any existing BackupItemAction plugin to work as expected, with an empty operation ID returned from Execute() and no-op Progress() and Cancel() methods.
+
+## Implementation
+This will be implemented during the Velero 1.11 development cycle.

design/Implemented/cluster-scope-resource-filter.md

@@ -0,0 +1,402 @@
+# Proposal to add resource filters for backup can distinguish whether resource is cluster-scoped or namespace-scoped.
+
+- [Proposal to add resource filters for backup can distinguish whether resource is cluster-scoped or namespace-scoped.](#proposal-to-add-resource-filters-for-backup-can-distinguish-whether-resource-is-cluster-scoped-or-namespace-scoped)
+	- [Abstract](#abstract)
+	- [Background](#background)
+	- [Goals](#goals)
+	- [Non Goals](#non-goals)
+	- [High-Level Design](#high-level-design)
+		- [Parameters Rules](#parameters-rules)
+		- [Using scenarios:](#using-scenarios)
+			- [no namespace-scoped resources + some cluster-scoped resources](#no-namespace-scoped-resources--some-cluster-scoped-resources)
+			- [no namespace-scoped resources + all cluster-scoped resources](#no-namespace-scoped-resources--all-cluster-scoped-resources)
+			- [some namespace-scoped resources + no cluster-scoped resources](#some-namespace-scoped-resources--no-cluster-scoped-resources)
+				- [scenario 1](#scenario-1)
+				- [scenario 2](#scenario-2)
+				- [scenario 3](#scenario-3)
+				- [scenario 4](#scenario-4)
+			- [some namespace-scoped resources + only related cluster-scoped resources](#some-namespace-scoped-resources--only-related-cluster-scoped-resources)
+				- [scenario 1](#scenario-1-1)
+				- [scenario 2](#scenario-2-1)
+				- [scenario 3](#scenario-3-1)
+			- [some namespace-scoped resources + some additional cluster-scoped resources](#some-namespace-scoped-resources--some-additional-cluster-scoped-resources)
+				- [scenario 1](#scenario-1-2)
+				- [scenario 2](#scenario-2-2)
+				- [scenario 3](#scenario-3-2)
+				- [scenario 4](#scenario-4-1)
+			- [some namespace-scoped resources + all cluster-scoped resources](#some-namespace-scoped-resources--all-cluster-scoped-resources)
+				- [scenario 1](#scenario-1-3)
+				- [scenario 2](#scenario-2-3)
+				- [scenario 3](#scenario-3-3)
+			- [all namespace-scoped resources + no cluster-scoped resources](#all-namespace-scoped-resources--no-cluster-scoped-resources)
+			- [all namespace-scoped resources + some additional cluster-scoped resources](#all-namespace-scoped-resources--some-additional-cluster-scoped-resources)
+			- [all namespace-scoped resources + all cluster-scoped resources](#all-namespace-scoped-resources--all-cluster-scoped-resources)
+			- [describe command change](#describe-command-change)
+	- [Detailed Design](#detailed-design)
+	- [Alternatives Considered](#alternatives-considered)
+	- [Security Considerations](#security-considerations)
+	- [Compatibility](#compatibility)
+	- [Implementation](#implementation)
+	- [Open Issues](#open-issues)
+
+## Abstract
+The current filter (IncludedResources/ExcludedResources + IncludeClusterResources flag) is not enough for some special cases, e.g. all namespace-scoped resources + some kind of cluster-scoped resource and all namespace-scoped resources + cluster-scoped resource excludes.
+Propose to add a new group of resource filtering parameters, which can distinguish cluster-scoped and namespace-scoped resources.
+
+## Background
+There are two sets of resource filters for Velero: `IncludedNamespaces/ExcludedNamespaces` and `IncludedResources/ExcludedResources`. 
+`IncludedResources` means only including the resource types specified in the parameter. Both cluster-scoped and namespace-scoped resources are handled in this parameter by now.
+The k8s resources are separated into cluster-scoped and namespace-scoped.
+As a result, it's hard to include all resources in one group and only including specified resource in the other group.
+
+## Goals
+- Make Velero can support more complicated namespace-scoped and cluster-scoped resources filtering scenarios in backup.
+
+## Non Goals
+- Enrich the resource filtering rules, for example, advanced PV filtering and filtering by resource names.
+
+
+## High-Level Design
+Four new parameters are added into command `velero backup create`: `--include-cluster-scoped-resources`, `--exclude-cluster-scoped-resources`, `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources`. 
+`--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` are used to filter cluster-scoped resources included or excluded in backup per resource type.
+`--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` are used to filter namespace-scoped resources included or excluded in backup per resource type.
+Restore and other code pieces also use resource filtering will be handled in future releases.
+
+### Parameters Rules
+
+* `--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources` valid value include `*` and comma separated string. Each element of the CSV string should a k8s resource name. The format should be `resource.group`, such as `storageclasses.storage.k8s.io.`.
+
+* `--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources` parameters are mutual exclusive with `--include-cluster-resources`, `--include-resources` and `--exclude-resources` parameters. If both sets of parameters are provisioned, validation failure should be returned.
+
+* `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` should only contain cluster-scoped resource type names. If namespace-scoped resource type names are included, they are ignored.
+
+* If there are conflicts between `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` specified resources type lists, `--exclude-cluster-scoped-resources` parameter has higher priority.
+
+* `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` should only contain namespace-scoped resource type names. If cluster-scoped resource type names are included, they are ignored.
+
+* If there are conflicts between `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` specified resources type lists, `--exclude-namespace-scoped-resources` parameter has higher priority.
+
+* If  `--include-namespace-scoped-resources` is not present, it means all namespace-scoped resources are included per resource type.
+
+* If both `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` are not present, it means no additional cluster-scoped resource is included per resource type, just as the existing `--include-cluster-resources` parameter not setting value. Cluster-scoped resources are related to the namespace-scoped resources, which means those are returned in the namespace-scoped resources' BackupItemAction's result AdditionalItems array, are still included in backup by default. Taking backing up PVC scenario as an example, PVC is namespace-scoped, PV is cluster-scoped. PVC's BIA will include PVC related PV into backup too.
+
+### Using scenarios:
+Please notice, if the scenario give the example of using old filtering parameters (`--include-cluster-resources`, `--include-resources` and `--exclude-resources`), that means the old parameters also work for this case. If old parameters example is not given, that means they don't work for this scenario, only new parameters (`--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources`) work.
+
+#### no namespace-scoped resources + some cluster-scoped resources
+The following command means backup no namespace-scoped resources and some cluster-scoped resources.
+
+``` bash
+velero backup create <backup-name>
+	--exclude-namespace-scoped-resources=*
+	--include-cluster-scoped-resources=storageclass
+```
+
+#### no namespace-scoped resources + all cluster-scoped resources
+The following command means backup no namespace-scoped resources and all cluster-scoped resources.
+
+``` bash
+velero backup create <backup-name>
+	--exclude-namespace-scoped-resources=*
+	--include-cluster-scoped-resources=*
+```
+
+#### some namespace-scoped resources + no cluster-scoped resources
+##### scenario 1
+The following commands mean backup all resources in namespaces default and kube-system, and no cluster-scoped resources.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--exclude-cluster-scoped-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-cluster-resources=false
+```
+##### scenario 2
+The following commands mean backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and no cluster-scoped resources. Although PVC's related PV should be included, due to no cluster-scoped resources are included, so they are ruled out too.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--exclude-cluster-scope-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--include-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--include-cluster-resources=false
+```
+##### scenario 3
+The following commands mean backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in namespace default and kube-system, and no cluster-scoped resources. Although PVC's related PV should be included, due to no cluster-scoped resources are included, so they are ruled out too.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--exclude-cluster-scope-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--include-cluster-resources=false
+```
+##### scenario 4
+The following commands mean backup all resources except Ingress type resources in all namespaces, and no cluster-scoped resources.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--exclude-namespace-scoped-resources=ingress
+	--exclude-cluster-scoped-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--exclude-resources=ingress
+	--include-cluster-resources=false
+```
+
+#### some namespace-scoped resources + only related cluster-scoped resources
+##### scenario 1
+This means backup all resources in namespaces default and kube-system, and related cluster-scoped resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+```
+
+##### scenario 2
+This means backup pods and configmaps in namespaces default and kube-system, and related cluster-scoped resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-namespace-scoped-resources=pods,configmaps
+```
+
+##### scenario 3
+This means backup all resources except Ingress type resources in all namespaces, and related cluster-scoped resources.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--exclude-namespace-scoped-resources=ingress
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--exclude-resources=ingress
+```
+
+#### some namespace-scoped resources + some additional cluster-scoped resources
+##### scenario 1
+This means backup all resources in namespace in default, kube-system, and related cluster-scoped resources, plus all StorageClass resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-cluster-scoped-resources=storageclass
+```
+
+##### scenario 2
+This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and related cluster-scoped resources, plus all StorageClass resources, and PVC related PV.
+``` bash
+velero backup create <backup-name>
+	--include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--include-cluster-scoped-resources=storageclass
+```
+
+##### scenario 3
+This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and related cluster-scoped resources, plus all StorageClass resources, and PVC related PV.
+``` bash
+velero backup create <backup-name>
+	--include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--include-namespaces=default,kube-system
+	--include-cluster-scoped-resources=storageclass
+```
+
+##### scenario 4
+This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and related cluster-scoped resources, plus all cluster-scoped resources except StorageClass type resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset
+	--include-namespaces=default,kube-system
+	--exclude-cluster-scoped-resources=storageclass
+```
+
+#### some namespace-scoped resources + all cluster-scoped resources
+##### scenario 1
+The following commands mean backup all resources in namespace in default, kube-system, and all cluster-scoped resources.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-cluster-scoped-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-cluster-resources=true
+```
+
+##### scenario 2
+This means backup Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and all cluster-scoped resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespace-scoped-resources=deployment,service,endpoint,pod,replicaset
+	--include-cluster-scoped-resources=*
+```
+
+##### scenario 3
+This means backup Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and all cluster-scoped resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=default,kube-system
+	--include-namespace-scoped-resources=deployment,service,endpoint,pod,replicaset
+	--include-cluster-scoped-resources=*
+```
+
+#### all namespace-scoped resources + no cluster-scoped resources
+The following commands all mean backup all namespace-scoped resources and no cluster-scoped resources.
+
+Example of new parameters:
+``` bash
+velero backup create <backup-name>
+	--exclude-cluster-scoped-resources=*
+```
+
+Example of old parameters:
+``` bash
+velero backup create <backup-name>
+	--include-cluster-resources=false
+```
+
+#### all namespace-scoped resources + some additional cluster-scoped resources
+This command means backup all namespace-scoped resources, and related cluster-scoped resources, plus all PersistentVolume resources.
+``` bash
+velero backup create <backup-name>
+	--include-namespaces=*
+	--include-cluster-scoped-resources=persistentvolume
+```
+
+#### all namespace-scoped resources + all cluster-scoped resources
+The following commands have the same meaning: backup all namespace-scoped resources, and all cluster-scoped resources.
+``` bash
+velero backup create <backup-name>
+	--include-cluster-scoped-resources=*
+```
+
+``` bash
+velero backup create <backup-name>
+	--include-cluster-resources=true
+```
+
+#### describe command change
+In `velero backup describe` command, the four new parameters should be outputted too.
+``` bash
+ velero backup describe <backup-name>
+......
+
+Namespaces:
+  Included:  ns2
+  Excluded:  <none>
+
+Resources:
+  Included cluster-scoped:    StorageClass,PersistentVolume
+  Excluded cluster-scoped:    <none>
+  Included namespace-scoped:  default
+  Excluded namespace-scoped:  <none>
+......
+```
+
+**Note:** `velero restore` command doesn't support those four new parameter in Velero v1.11, but `velero schedule` supports the four new parameters through backup specification.
+
+## Detailed Design
+With adding `IncludedNamespaceScopedResources`, `ExcludedNamespaceScopedResources`, `IncludedClusterScopedResources` and `ExcludedClusterScopedResources`, the `BackupSpec` looks like:
+``` go
+type BackupSpec struct {
+    ......
+	// IncludedResources is a slice of resource names to include
+	// in the backup. If empty, all resources are included.
+	// +optional
+	// +nullable
+	IncludedResources []string `json:"includedResources,omitempty"`
+
+	// ExcludedResources is a slice of resource names that are not
+	// included in the backup.
+	// +optional
+	// +nullable
+	ExcludedResources []string `json:"excludedResources,omitempty"`
+
+	// IncludeClusterResources specifies whether cluster-scoped resources
+	// should be included for consideration in the backup.
+	// +optional
+	// +nullable
+	IncludeClusterResources *bool `json:"includeClusterResources,omitempty"`
+
+	// IncludedClusterScopedResources is a slice of cluster-scoped
+	// resource type names to include in the backup.
+	// If set to "*", all cluster scope resource types are included.
+	// The default value is empty, which means only related cluster 
+	// scope resources are included.
+	// +optional
+	// +nullable
+	IncludedClusterScopedResources []string `json:"includedClusterScopedResources,omitempty"`
+
+	// ExcludedClusterScopedResources is a slice of cluster-scoped 
+	// resource type names to exclude from the backup.
+	// If set to "*", all cluster scope resource types are excluded.
+	// +optional
+	// +nullable
+	ExcludedClusterScopedResources []string `json:"excludedClusterScopedResources,omitempty"`
+
+	// IncludedNamespaceScopedResources is a slice of namespace-scoped
+	// resource type names to include in the backup.
+	// The default value is "*".
+	// +optional
+	// +nullable
+	IncludedNamespaceScopedResources []string `json:"includedNamespaceScopedResources,omitempty"`
+
+	// ExcludedNamespaceScopedResources is a slice of namespace-scoped
+	// resource type names to exclude from the backup.
+	// If set to "*", all namespace scope resource types are excluded.
+	// +optional
+	// +nullable
+	ExcludedNamespaceScopedResources []string `json:"excludedNamespaceScopedResources,omitempty"`
+    ......
+}
+```
+
+## Alternatives Considered
+Proposal from Jibu Data [Issue 5120](https://github.com/vmware-tanzu/velero/issues/5120#issue-1304534563)
+
+## Security Considerations
+No security impact.
+
+## Compatibility
+The four new parameters cannot be mixed with existing resource filter parameters: `IncludedResources`, `ExcludedResources` and `IncludeClusterResources`.
+If the new parameters and old parameters both appears in command line, or are specified in backup spec, the command line and the backup should fail.
+
+## Implementation
+This change should be included into Velero v1.11.
+New parameters will coexist with `IncludedResources`, `ExcludedResources` and `IncludeClusterResources`.
+Plan to deprecate `IncludedResources`, `ExcludedResources` and `IncludeClusterResources` in future releases, but also open to the community's feedback.
+
+## Open Issues
+`LabelSelector/OrLabelSelectors` apply to namespace-scoped resources.
+It may be reasonable to make them also working on cluster-scoped resources.
+An issue is created to trace this topic [resource label selector not work for cluster-scoped resources](https://github.com/vmware-tanzu/velero/issues/5787)

design/Implemented/general-progress-monitoring.md

@@ -0,0 +1,579 @@
+# Plugin Progress Monitoring
+
+This is intended as a replacement for the previously-approved Upload Progress Monitoring design
+([Upload Progress Monitoring](upload-progress.md)) in order to expand the supported use cases beyond
+snapshot uploads to include what was previously called Async Backup/Restore Item Actions. This
+updated design should handle the combined set of use cases for those previously separate designs.
+
+Volume snapshotter plug-in are used by Velero to take snapshots of persistent volume contents. 
+Depending on the underlying storage system, those snapshots may be available to use immediately, 
+they may be uploaded to stable storage internally by the plug-in or they may need to be uploaded after
+the snapshot has been taken. We would like for Velero to continue on to the next part of the backup as quickly
+as possible but we would also like the backup to not be marked as complete until it is a usable backup.  We'd also
+eventually like to bring the control of upload under the control of Velero and allow the user to make decisions
+about the ultimate destination of backup data independent of the storage system they're using.
+
+We would also like any internal or third party Backup or Restore Item Action to have the option of
+making use of this same ability to run some external process without blocking the current backup or
+restore operation. Beyond Volume Snapshotters, this is also needed for data mover operations on both
+backup and restore, and potentially useful for other third party operations -- for example
+in-cluster registry image backup or restore could make use of this feature in a third party plugin).
+
+### Glossary
+- <b>BIA</b>: BackupItemAction
+- <b>RIA</b>: RestoreItemAction
+
+## Examples
+- AWS - AWS snapshots return quickly, but are then uploaded in the background and cannot be used until EBS moves
+the data into S3 internally.
+
+- vSphere - The vSphere plugin takes a local snapshot and then the vSphere plugin uploads the data to S3.  The local
+snapshot is usable before the upload completes.
+
+- Restic - Does not go through the volume snapshot path.  Restic backups will block Velero progress
+until completed. However, with the more generalized approach in the revised design, restic/kopia
+backup and restore *could* make use of this framework if their actions are refactored as
+Backup/RestoreItemActions.
+
+- Data Movers
+    - Data movers are asynchronous processes executed inside backup/restore item actions that applies to a specific kubernetes resources. A common use case for data mover is to backup/restore PVCs whose data we want to move to some form of backup storage outside of using velero kopia/restic implementations.
+    - Workflow
+        - User takes velero backup of PVC A
+        - BIA plugin applies to PVCs with compatible storage driver
+        - BIA plugin triggers data mover
+	  - Most common use case would be for the plugin action to create a new CR which would
+            trigger an external controller action
+	  - Another possible use case would be for the plugin to run its own async action in a
+            goroutine, although this would be less resilient to plugin container restarts.
+        - BIA plugin returns
+        - Velero backup process continues
+        - Main velero backup process monitors running BIA threads via gRPC to determine if process is done and healthy
+
+
+## Primary changes from the original Upload Progress Monitoring design
+
+The most fundamental change here is that rather than proposing a new special-purpose
+SnapshotItemAction, the existing BackupItemAction plugin will be modified to accommodate an optional
+snapshot (or other item operation) ID return. The primary reasons for this change are as follows:
+
+1. The intended scope has moved beyond snapshot processing, so it makes sense to support
+asynchronous operations in other backup or restore item actions.
+
+2. We expect to have plugin API versioning implemented in Velero 1.10, making it feasible to
+implement changes in the existing plugin APIs now.
+
+3. We will need this feature on both backup and restore, meaning that if we took the "new plugin
+type" approach, we'd need two new plugin types.
+
+4. Other than the snapshot/operation ID return, the rest of the plugin processing is identical to
+Backup/RestoreItemActions. With separate plugin types, we'd have to repeat all of that logic
+(including dealing with additional items, etc.) twice.
+
+The other major change is that we will be applying this to both backups and restores, although the
+Volume Snapshotter use case only needs this on backup. This means that everything we're doing around
+backup phase and workflow will also need to be done for restore.
+
+Then there are various minor changes around terminology to make things more generic. Instead of
+"snapshotID", we'll have "operationID" (which for volume snapshotters will be a snapshot
+ID).
+
+## Goals
+
+- Enable monitoring of backup/restore item action operations that continue after snapshotting and other operations have completed
+- Keep non-usable backups and restores (upload/persistence has not finished, etc.) from appearing as completed
+- Make use of plugin API versioning functionality to manage changes to Backup/RestoreItemAction interfaces
+- Enable vendors to plug their own data movers into velero using BIA/RIA plugins
+
+## Non-goals
+- Today, Velero is unable to recover from an in progress backup when the velero server crashes (pod is deleted). This has an impact on running asynchronous processes, but it’s not something we intend to solve in this design.
+
+## Models
+
+### Internal configuration and management
+In this model, movement of the snapshot to stable storage is under the control of the snapshot
+plug-in.  Decisions about where and when the snapshot gets moved to stable storage are not
+directly controlled by Velero.  This is the model for the current VolumeSnapshot plugins.
+
+### Velero controlled management
+In this model, the snapshot is moved to external storage under the control of Velero.  This
+enables Velero to move data between storage systems.  This also allows backup partners to use
+Velero to snapshot data and then move the data into their backup repository.
+
+## Backup and Restore phases
+
+Velero currently has backup/restore phases "InProgress" and "Completed".  A backup moves to the
+Completed phase when all of the volume snapshots have completed and the Kubernetes metadata has been
+written into the object store.  However, the actual data movement may be happening in the background
+after the backup has been marked "Completed".  The backup is not actually a stable backup until the
+data has been persisted properly.  In some cases (e.g. AWS) the backup cannot be restored from until
+the snapshots have been persisted.
+
+Once the snapshots have been taken, however, it is possible for additional backups or restores (as
+long as they don't use not-yet-completed backups) to be made without interference.  Waiting until
+all data has been moved before starting the next backup will slow the progress of the system without
+adding any actual benefit to the user.
+
+New backup/restore phases, "WaitingForPluginOperations" and
+"WaitingForPluginOperationsPartiallyFailed" will be introduced.  When a backup or restore has
+entered one of these phases, Velero is free to start another backup/restore.  The backup/restore
+will remain in the "WaitingForPluginOperations" phase until all BIA/RIA operations have completed
+(for example, for a volume snapshotter, until all data has been successfully moved to persistent
+storage).  The backup/restore will not fail once it reaches this phase, although an error return
+from a plugin could cause a backup or restore to move to "PartiallyFailed".  If the backup is
+deleted (cancelled), the plug-ins will attempt to delete the snapshots and stop the data movement -
+this may not be possible with all storage systems.
+
+In addition, for backups (but not restores), there will also be two additional phases, "Finalizing"
+and "FinalizingPartiallyFailed", which will handle any steps required after plugin operations have
+all completed. Initially, this will just include adding any required resources to the backup that
+might have changed during asynchronous operation execution, although eventually other cleanup
+actions could be added to this phase.
+
+### State progression
+
+![image](AsyncActionFSM.png)
+### New
+When a backup/restore request is initially created, it is in the "New" phase.  
+
+The next state is either "InProgress" or "FailedValidation"
+
+### FailedValidation
+If the backup/restore request is incorrectly formed, it goes to the "FailedValidation" phase and
+terminates
+
+### InProgress
+When work on the backup/restore begins, it moves to the "InProgress" phase.  It remains in the
+"InProgress" phase until all pre/post execution hooks have been executed, all snapshots have been
+taken and the Kubernetes metadata and backup/restore info is safely written to the object store
+plug-in.
+
+In the current implementation, Restic backups will move data during the "InProgress" phase.  In the
+future, it may be possible to combine a snapshot with a Restic (or equivalent) backup which would
+allow for data movement to be handled in the "WaitingForPluginOperations" phase,
+
+The next phase would be "WaitingForPluginOperations" for backups or restores which have unfinished
+asynchronous plugin operations and no errors so far, "WaitingForPluginOperationsPartiallyFailed" for
+backups or restores which have unfinished asynchronous plugin operations at least one error,
+"Completed" for restores with no unfinished asynchronous plugin operations and no errors,
+"PartiallyFailed" for restores with no unfinished asynchronous plugin operations and at least one
+error, "Finalizing" for backups with no unfinished asynchronous plugin operations and no errors,
+"FinalizingPartiallyFailed" for backups with no unfinished asynchronous plugin operations and at
+least one error, or "PartiallyFailed".  Backups/restores which would have a final phase of
+"Completed" or "PartiallyFailed" may move to the "WaitingForPluginOperations" or
+"WaitingForPluginOperationsPartiallyFailed" state.  A backup/restore which will be marked "Failed"
+will go directly to the "Failed" phase.  Uploads may continue in the background for snapshots that
+were taken by a "Failed" backup/restore, but no progress will not be monitored or updated. If there
+are any operations in progress when a backup is moved to the "Failed" phase (although with the
+current workflow, that shouldn't happen), Cancel() should be called on these operations. When a
+"Failed" backup is deleted, all snapshots will be deleted and at that point any uploads still in
+progress should be aborted.
+
+### WaitingForPluginOperations (new)
+The "WaitingForPluginOperations" phase signifies that the main part of the backup/restore, including
+snapshotting has completed successfully and uploading and any other asynchronous BIA/RIA plugin
+operations are continuing.  In the event of an error during this phase, the phase will change to
+WaitingForPluginOperationsPartiallyFailed.  On success, the phase changes to
+"Finalizing" for backups and "Completed" for restores.  Backups cannot be
+restored from when they are in the WaitingForPluginOperations state.
+
+### WaitingForPluginOperationsPartiallyFailed (new)
+The "WaitingForPluginOperationsPartiallyFailed" phase signifies that the main part of the
+backup/restore, including snapshotting has completed, but there were partial failures either during
+the main part or during any async operations, including snapshot uploads.  Backups cannot be
+restored from when they are in the WaitingForPluginOperationsPartiallyFailed state.
+
+### Finalizing (new)
+The "Finalizing" phase signifies that asynchronous backup operations have all completed successfully
+and Velero is currently backing up any resources indicated by asynchronous plugins as items to back
+up after operations complete. Once this is done, the phase changes to Completed.  Backups cannot be
+restored from when they are in the Finalizing state.
+
+### FinalizingPartiallyFailed (new)
+
+The "FinalizingPartiallyFailed" phase signifies that, for a backup which had errors during initial
+processing or asynchronous plugin operation, asynchronous backup operations have all completed and
+Velero is currently backing up any resources indicated by asynchronous plugins as items to back up
+after operations complete. Once this is done, the phase changes to PartiallyFailed.  Backups cannot
+be restored from when they are in the FinalizingPartiallyFailed state.
+
+### Failed
+When a backup/restore has had fatal errors it is marked as "Failed" Backups in this state cannot be
+restored from.
+
+### Completed
+The "Completed" phase signifies that the backup/restore has completed, all data has been transferred
+to stable storage (or restored to the cluster) and any backup in this state is ready to be used in a
+restore.  When the Completed phase has been reached for a backup it is safe to remove any of the
+items that were backed up.
+
+### PartiallyFailed
+The "PartiallyFailed" phase signifies that the backup/restore has completed and at least part of the
+backup/restore is usable.  Restoration from a PartiallyFailed backup will not result in a complete
+restoration but pieces may be available.
+
+## Workflow
+
+When a Backup or Restore Action is executed, any BackupItemAction, RestoreItemAction, or
+VolumeSnapshot plugins will return operation IDs (snapshot IDs or other plugin-specific
+identifiers).  The plugin should be able to provide status on the progress for the snapshot and
+handle cancellation of the operation/upload if the snapshot is deleted.  If the plugin is restarted,
+the operation ID should remain valid.
+
+When all snapshots have been taken and Kubernetes resources have been persisted to the ObjectStorePlugin
+the backup will either have fatal errors or will be at least partially usable.
+
+If the backup/restore has fatal errors it will move to the "Failed" state and finish. If a
+backup/restore fails, the upload or other operation will not be cancelled but it will not be
+monitored either.  For backups in any phase, all snapshots will be deleted when the backup is
+deleted.  Plugins will cancel any data movement or other operations and remove snapshots and other
+associated resources when the VolumeSnapshotter DeleteSnapshot method or DeleteItemAction Execute
+method is called.
+
+Velero will poll the plugins for status on the operations when the backup/restore exits the
+"InProgress" phase and has no fatal errors.
+
+If any operations are not complete, the backup/restore will move to either WaitingForPluginOperations
+or WaitingForPluginOperationsPartiallyFailed or Failed.
+
+Post-snapshot and other operations may take a long time and Velero and its plugins may be restarted
+during this time.  Once a backup/restore has moved into the WaitingForPluginOperations or
+WaitingForPluginOperationsPartiallyFailed phase, another backup/restore may be started.
+
+While in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase, the
+snapshots and item actions will be periodically polled.  When all of the snapshots and item actions
+have reported success, restores will move directly to the Completed or PartiallyFailed phase, and
+backups will move to the Finalizing or FinalizingPartiallyFailed phase, depending on whether the
+backup/restore was in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed
+phase.
+
+While in the Finalizing or FinalizingPartiallyFailed phase, Velero will update the backup with any
+resources indicated by plugins that they must be added to the backup after operations are completed,
+and then the backup will move to the Completed or PartiallyFailed phase, depending on whether there
+are any backup errors.
+
+The Backup resources will be written to object storage at the time the backup leaves the InProgress
+phase, but it will not be synced to other clusters (or usable for restores in the current cluster)
+until the backup has entered a final phase: Completed, Failed or PartiallyFailed. During the
+Finalizing phases, a the backup resources will be updated with any required resources related to
+asynchronous plugins.
+
+## Reconciliation of InProgress backups
+
+InProgress backups will not have a `velero-backup.json` present in the object store.  During
+reconciliation, backups which do not have a `velero-backup.json` object in the object store will be
+ignored.
+
+## Plug-in API changes
+
+### OperationProgress struct
+
+    type OperationProgress struct {
+        Completed bool                          // True when the operation has completed, either successfully or with a failure
+        Err string                              // Set when the operation has failed
+        NCompleted, NTotal int64                // Quantity completed so far and the total quantity associated with the operation in operationUnits
+                                                // For data mover and volume snapshotter use cases, this would be in bytes
+                                                // On successful completion, completed and total should be the same.
+        OperationUnits string                   // Units represented by completed and total -- for data mover and item
+	                                        // snapshotters, this will usually be bytes.
+        Description string                      // Optional description of operation progress
+        Started, Updated time.Time              // When the upload was started and when the last update was seen.  Not all
+                                                // systems retain when the upload was begun, return Time 0 (time.Unix(0, 0))
+                                                // if unknown.
+    }
+
+### VolumeSnapshotter changes
+
+Two new methods will be added to the VolumeSnapshotter interface:
+
+    Progress(snapshotID string) (OperationProgress, error)
+    Cancel(snapshotID string) (error)
+
+Progress will report the current status of a snapshot upload.  This should be callable at
+any time after the snapshot has been taken.  In the event a plug-in is restarted, if the operationID
+(snapshot ID) continues to be valid it should be possible to retrieve the progress.
+
+`error` is set if there is an issue retrieving progress.  If the snapshot is has encountered an
+error during the upload, the error should be returned in OperationProgress and error should be nil.
+
+### BackupItemAction and RestoreItemAction plug-in changes
+
+Currently CSI snapshots and the Velero Plug-in for vSphere are implemented as BackupItemAction
+plugins.  While the majority of BackupItemAction plugins do not take snapshots or upload data, this
+functionality is useful for any longstanding plugin operation managed by an external
+process/controller so we will modify BackupItemAction and RestoreItemAction to optionally return an
+operationID in addition to the modified item.
+
+Velero can attempt to cancel an operation by calling the Cancel API call on the BIA/RIA. The plugin
+can then take any appropriate action as needed. Cancel will be called for unfinished operations on
+backup deletion, and possibly reaching timeout. Cancel is not intended to be used to delete/remove
+the results of completed actions and will have no effect on a completed action. Cancel has no return
+value apart from the standard Error return, but this should only be used for unexpected
+failures. Under normal operations, Cancel will simply return a nil error (and nothing else), whether
+or not the plugin is able to cancel the operation.
+
+_AsyncOperationsNotSupportedError_ should only be returned (by Progress) if the
+Backup/RestoreItemAction plugin should not be handling the item.  If the Backup/RestoreItemAction
+plugin should handle the item but, for example, the item/snapshot ID cannot be found to report
+progress, Progress will return an InvalidOperationIDError error rather than a populated
+OperationProgress struct. If the item action does not start an asynchronous operation, then
+operationID will be empty.
+
+Three new methods will be added to the BackupItemAction interface, and the Execute() return signature
+will be modified:
+
+	// Name returns the name of this BIA. Plugins which implement this interface must defined Name,
+	// but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure
+	// will implement this directly rather than delegating to the RPC plugin in order to return the name
+	// that the plugin was registered under. The plugins must implement the method to complete the interface.
+	Name() string
+	// Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up,
+	// including mutating the item itself prior to backup. The item (unmodified or modified)
+	// should be returned, along with an optional slice of ResourceIdentifiers specifying
+	// additional related items that should be backed up now, an optional operationID for actions which
+	// initiate asynchronous actions, and a second slice of ResourceIdentifiers specifying related items
+	// which should be backed up after all asynchronous operations have completed. This last field will be
+	// ignored if operationID is empty, and should not be filled in unless the resource must be updated in the
+	// backup after async operations complete (i.e. some of the item's kubernetes metadata will be updated
+	// during the asynch operation which will be required during restore)
+	Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error)
+    	
+    	// Progress  
+	Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error)
+    	// Cancel  
+	Cancel(operationID string, backup *api.Backup) error
+
+Three new methods will be added to the RestoreItemAction interface, and the
+RestoreItemActionExecuteOutput struct will be modified:
+
+	// Name returns the name of this RIA. Plugins which implement this interface must defined Name,
+	// but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure
+	// will implement this directly rather than delegating to the RPC plugin in order to return the name
+	// that the plugin was registered under. The plugins must implement the method to complete the interface.
+	Name() string
+	// Execute allows the ItemAction to perform arbitrary logic with the item being restored,
+	// including mutating the item itself prior to restore. The item (unmodified or modified)
+	// should be returned, an optional OperationID, along with an optional slice of ResourceIdentifiers
+	// specifying additional related items that should be restored, a warning (which will be
+	// logged but will not prevent the item from being restored) or error (which will be logged
+	// and will prevent the item from being restored) if applicable. If OperationID is specified
+    	// then velero will wait for this operation to complete before the restore is marked Completed.
+	Execute(input *RestoreItemActionExecuteInput) (*RestoreItemActionExecuteOutput, error)
+
+    	
+    	// Progress  
+	Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error)
+
+        // Cancel  
+	Cancel(operationID string, restore *api.Restore) error
+    
+    // RestoreItemActionExecuteOutput contains the output variables for the ItemAction's Execution function.
+    type RestoreItemActionExecuteOutput struct {
+        // UpdatedItem is the item being restored mutated by ItemAction.
+        UpdatedItem runtime.Unstructured
+
+        // AdditionalItems is a list of additional related items that should
+        // be restored.
+        AdditionalItems []ResourceIdentifier
+
+        // SkipRestore tells velero to stop executing further actions
+        // on this item, and skip the restore step. When this field's
+        // value is true, AdditionalItems will be ignored.
+        SkipRestore bool
+
+        // OperationID is an identifier which indicates an ongoing asynchronous action which Velero will
+        // continue to monitor after restoring this item. If left blank, then there is no ongoing operation
+        OperationID string
+    }
+
+## Changes in Velero backup format
+
+No changes to the existing format are introduced by this change.  As part of the backup workflow changes, a
+`<backup-name>-itemoperations.json.gz` file will be added that contains the items and operation IDs
+(snapshotIDs) returned by VolumeSnapshotter and BackupItemAction plugins.  Also, the creation of the
+`velero-backup.json` object will not occur until the backup moves to one of the terminal phases
+(_Completed_, _PartiallyFailed_, or _Failed_).  Reconciliation should ignore backups that do not
+have a `velero-backup.json` object.
+
+The Backup/RestoreItemAction plugin identifier as well as the ItemID and OperationID will be stored
+in the `<backup-name>-itemoperations.json.gz`.  When checking for progress, this info will be used
+to select the appropriate Backup/RestoreItemAction plugin to query for progress. Here's an example
+of what a record for a datamover plugin might look like:
+```
+    {
+        "spec": {
+            "backupName": "backup-1",
+            "backupUID": "f8c72709-0f73-46e1-a071-116bc4a76b07",
+            "backupItemAction": "velero.io/volumesnapshotcontent-backup",
+            "resourceIdentifier": {
+	        "Group": "snapshot.storage.k8s.io",
+                "Resource": "VolumeSnapshotContent",
+                "Namespace": "my-app",
+                "Name": "my-volume-vsc"
+            },
+            "operationID": "<DataMoverBackup objectReference>",
+            "itemsToUpdate": [
+	      {
+	        "Group": "velero.io",
+                "Resource": "VolumeSnapshotBackup",
+                "Namespace": "my-app",
+                "Name": "vsb-1"
+              }
+	    ]
+	},
+        "status": {
+            "operationPhase": "Completed",
+            "error": "",
+            "nCompleted": 12345,
+            "nTotal": 12345,
+            "operationUnits": "byte",
+            "description": "",
+            "Created": "2022-12-14T12:00:00Z",
+            "Started": "2022-12-14T12:01:00Z",
+            "Updated": "2022-12-14T12:11:02Z"
+        },
+    }
+```
+
+The cluster that is creating the backup will have the Backup resource present and will be able to
+manage the backup before the backup completes.
+
+If the Backup resource is removed (e.g. Velero is uninstalled) before a backup completes and writes
+its `velero-backup.json` object, the other objects in the object store for the backup will be
+effectively orphaned.  This can currently happen but the current window is much smaller.
+
+### `<backup-name>-itemoperations.json.gz`
+The itemoperations file is similar to the existing `<backup-name>-itemsnapshots.json.gz`  Each snapshot taken via
+BackupItemAction will have a JSON record in the file.  Exact format TBD.
+
+This file will be uploaded to object storage at the end of processing all of the items in the
+backup, before the phase moves away from `InProgress`.
+
+## Changes to Velero restores
+
+A `<restore-name>-itemoperations.json.gz` file will be added that contains the items and operation
+IDs returned by RestoreItemActions. The format will be the same as the
+`<backup-name>-itemoperations.json.gz` generated for backups.
+
+This file will be uploaded to object storage at the end of processing all of the items in the
+restore, before the phase moves away from `InProgress`.
+
+## CSI snapshots
+
+For systems such as EBS, a snapshot is not available until the storage system has transferred the
+snapshot to stable storage.  CSI snapshots expose the _readyToUse_ state that, in the case of EBS,
+indicates that the snapshot has been transferred to durable storage and is ready to be used.  The
+CSI BackupItemAction.Progress method will poll that field and when completed, return completion.
+
+## vSphere plug-in
+
+The vSphere Plug-in for Velero uploads snapshots to S3 in the background.  This is also a
+BackupItemAction plug-in, it will check the status of the Upload records for the snapshot and return
+progress.
+
+## Backup workflow changes
+
+The backup workflow remains the same until we get to the point where the `velero-backup.json` object
+is written.  At this point, Velero will
+run across all of the VolumeSnapshotter/BackupItemAction operations and call the _Progress_ method
+on each of them.
+
+If all backup item operations have finished (either successfully or failed), the backup will move to
+one of the finalize phases.
+
+If any of the snapshots or backup items are still being processed, the phase of the backup will be
+set to the appropriate phase (_WaitingForPluginOperations_ or
+_WaitingForPluginOperationsPartiallyFailed_), and the async backup operations controller will
+reconcile periodically and call Progress on any unfinished operations. In the event of any of the
+progress checks return an error, the phase will move to _WaitingForPluginOperationsPartiallyFailed_.
+
+Once all operations have completed, the backup will be moved to one of the finalize phases, and the
+backup finalizer controller will update the the `velero-backup.json`in the object store with any
+resources necessary after asynchronous operations are complete and the backup will move to the
+appropriate terminal phase.
+
+
+## Restore workflow changes
+
+The restore workflow remains the same until velero would currently move the backup into one of the
+terminal states. At this point, Velero will run across all of the RestoreItemAction operations and
+call the _Progress_ method on each of them.
+
+If all restore item operations have finished (either successfully or failed), the restore will be
+completed and the restore will move to the appropriate terminal phase and the restore will be
+complete.
+
+If any of the restore items are still being processed, the phase of the restore will be set to the
+appropriate phase (_WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_), and
+the async restore operations controller will reconcile periodically and call Progress on any
+unfinished operations. In the event of any of the progress checks return an error, the phase will
+move to _WaitingForPluginOperationsPartiallyFailed_. Once all of the operations have completed, the
+restore will be moved to the appropriate terminal phase.
+
+## Restart workflow
+
+On restart, the Velero server will scan all Backup/Restore resources.  Any Backup/Restore resources
+which are in the _InProgress_ phase will be moved to the _Failed_ phase.  Any Backup/Restore
+resources in the _WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_ phase
+will be treated as if they have been requeued and progress checked and the backup/restore will be
+requeued or moved to a terminal phase as appropriate.
+
+## Notes on already-merged code which may need updating
+
+Since this design is modifying a previously-approved design, there is some preparation work based on
+the earlier upload progress monitoring design that may need modification as a result of these
+updates. Here is a list of some of these items:
+
+1. Consts for the "Uploading" and "UploadingPartiallyFailed" phases have already been defined. These
+   will need to be removed when the "WaitingForPluginOperations" and
+   "WaitingForPluginOperationsPartiallyFailed" phases are defined.
+   - https://github.com/vmware-tanzu/velero/pull/3805
+1. Remove the ItemSnapshotter plugin APIs (and related code) since the revised design will reuse
+   VolumeSnapshotter and BackupItemAction plugins.
+   - https://github.com/vmware-tanzu/velero/pull/4077
+   - https://github.com/vmware-tanzu/velero/pull/4417
+1. UploadProgressFeatureFlag shouldn't be needed anymore. The current design won't really need a
+   feature flag here -- the new features will be added to V2 of the VolumeSnapshotter,
+   BackupItemAction, and RestoreItemAction plugins, and it will only be used if there are plugins which
+   return operation IDs.
+   - https://github.com/vmware-tanzu/velero/pull/4416
+1. Adds <backup-name>-itemsnapshots.gz file to backup (when provided) -- this is still part of the
+   revised design, so it should stay.
+   - https://github.com/vmware-tanzu/velero/pull/4429
+1. Upload Progress Monitoring and Item Snapshotter basic support: This PR is not yet merged, so
+   nothing will need to be reverted. While the implementation here will be useful in informing the
+   implementation of the new design, several things have changed in the design proposal since the PR
+   was written.
+   - https://github.com/vmware-tanzu/velero/pull/4467
+
+# Implementation tasks
+
+VolumeSnapshotter new plugin APIs  
+BackupItemAction new plugin APIs  
+RestoreItemAction new plugin APIs  
+New backup phases  
+New restore phases  
+Defer uploading `velero-backup.json`  
+AWS EBS plug-in Progress implementation  
+Operation monitoring  
+Implementation of `<backup-name>-itemoperations.json.gz` file  
+Implementation of `<restore-name>-itemoperations.json.gz` file
+Restart logic  
+Change in reconciliation logic to ignore backups/restores that have not completed  
+CSI plug-in BackupItemAction Progress implementation  
+vSphere plug-in BackupItemAction Progress implementation (vSphere plug-in team)  
+
+
+# Open Questions
+
+1. Do we need a Cancel operation for VolumeSnapshotter?
+   - From feedback, I'm thinking we probably don't need it. The only real purpose of Cancel
+     here is to tell the plugin that Velero won't be waiting anymore, so if there are any
+     required custom cancellation actions, now would be a good time to perform them. For snapshot
+     uploads that are already in proress, there's not really anything else to cancel.
+2. Should we actually write the backup *before* moving to the WaitingForPluginOperations or
+   WaitingForPluginOperationsPartiallyFailed phase rather than waiting until all operations
+   have completed? The operations in question won't affect what gets written to object storage
+   for the backup, and since we've already written the list of operations we're waiting for to
+   object storage, writing the backup now would make the process resilient to Velero restart if
+   it happens during WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed
+

design/Implemented/handle-backup-of-volumes-by-resources-filters.md

@@ -0,0 +1,324 @@
+# Handle backup of volumes by resources filters
+ 
+## Abstract
+Currently, Velero doesn't have one flexible way to handle volumes.
+ 
+If users want to skip the backup of volumes or only backup some volumes in different namespaces in batch, currently they need to use the opt-in and opt-out approach one by one, or use label-selector but if it has big different labels on each different related pod, which is cumbersome when they have lots of volumes to handle with. it would be convenient if Velero could provide policies to handle the backup of volumes just by `some specific volumes conditions`.
+ 
+## Background
+As of Today, Velero has lots of filters to handle (backup or skip backup) resources including resources filters like `IncludedNamespaces, ExcludedNamespaces`, label selectors like `LabelSelector, OrLabelSelectors`, annotation like `backup.velero.io/must-include-additional-items` etc. But it's not enough flexible to handle volumes, we need one generic way to handle volumes.
+ 
+## Goals
+- Introducing flexible policies to handle volumes, and do not patch any labels or annotations to the pods or volumes.
+ 
+## Non-Goals
+- We only handle volumes for backup and do not support restore.
+- Currently, only handles volumes, and does not support other resources.
+- Only environment-unrelated and platform-independent general volumes attributes are supported, do not support volumes attributes related to a specific environment.
+ 
+## Use-cases/Scenarios
+### Skip backup volumes by some attributes
+Users want to skip PV with the requirements:
+- option to skip all PV data
+- option to skip specified PV type (RBD, NFS)
+- option to skip specified PV size
+- option to skip specified storage-class
+ 
+## High-Level Design
+First, Velero will provide the user with one YAML file template and all supported volume policies will be in.
+
+Second, writing your own configuration file by imitating the YAML template, it could be partial volume policies from the template.
+
+Third, create one configmap from your own configuration file, and the configmap should be in Velero install namespace.
+
+Fourth, create a backup with the command `velero backup create --resource-policies-configmap $policiesConfigmap`, which will reference the current backup to your volume policies. At the same time, Velero will validate all volume policies user imported, the backup will fail if the volume policies are not supported or some items could not be parsed.
+
+Fifth, the current backup CR will record the reference of volume policies configmap.
+
+Sixth, Velero first filters volumes by other current supported filters, at last, it will apply the volume policies to the filtered volumes to get the final matched volume to handle.
+ 
+## Detailed Design
+The volume resources policies should contain a list of policies which is the combination of conditions and related `action`, when target volumes meet the conditions, the related `action` will take effection.
+
+Below is the API Design for the user configuration:
+
+### API Design
+```go
+type VolumeActionType string
+
+const Skip VolumeActionType = "skip"
+
+// Action defined as one action for a specific way of backup
+type Action struct {
+	// Type defined specific type of action, it could be 'file-system-backup', 'volume-snapshot', or 'skip' currently
+	Type VolumeActionType `yaml:"type"`
+	// Parameters defined map of parameters when executing a specific action
+	// +optional
+	// +nullable
+	Parameters map[string]interface{} `yaml:"parameters,omitempty"`
+}
+
+// VolumePolicy defined policy to conditions to match Volumes and related action to handle matched Volumes
+type VolumePolicy struct {
+	// Conditions defined list of conditions to match Volumes
+	Conditions map[string]interface{} `yaml:"conditions"`
+	Action     Action                 `yaml:"action"`
+}
+
+// ResourcePolicies currently defined slice of volume policies to handle backup
+type ResourcePolicies struct {
+  Version        string         `yaml:"version"`
+	VolumePolicies []VolumePolicy `yaml:"volumePolicies"`
+	// we may support other resource policies in the future, and they could be added separately
+  	// OtherResourcePolicies: []OtherResourcePolicy
+}
+```
+
+The policies YAML config file would look like this:
+```yaml
+version: v1
+volumePolicies:
+# it's a list and if the input item matches the first policy, the latters will be ignored
+# each policy consists of a list of conditions and an action
+
+# each key in the object is one condition, and one policy will apply to resources that meet ALL conditions
+- conditions:
+    # capacity condition matches the volumes whose capacity falls into the range
+    capacity: "0,100Gi"
+    csi:
+      driver: aws.ebs.csi.driver
+      fsType: ext4
+    storageClass:
+    - gp2
+    - ebs-sc
+  action:
+    type: volume-snapshot
+    parameters:
+      # optional parameters which are custom-defined parameters when doing an action
+      volume-snapshot-timeout: "6h"
+- conditions:
+    capacity: "0,100Gi"
+    storageClass:
+    - gp2
+    - ebs-sc
+  action:
+    type: file-system-backup
+- conditions:
+    nfs:
+      server: 192.168.200.90
+  action:
+    # type of file-system-backup could be defined a second time
+    type: file-system-backup
+- conditions:
+    nfs: {}
+  action:
+    type: skip
+- conditions:
+    csi:
+      driver: aws.efs.csi.driver
+  action:
+    type: skip
+```
+ 
+### Filter rules
+#### VolumePolicies
+The whole resource policies consist of groups of volume policies.
+
+For one specific volume policy which is a combination of one action and serval conditions. which means one action and serval conditions are the smallest unit of volume policy.
+
+Volume policies are a list and if the target volumes match the first policy, the latter will be ignored, which would reduce the complexity of matching volumes especially when there are multiple complex volumes policies.
+
+#### Action
+`Action` defined one action for a specific way of backup:
+  - if choosing `Kopia` or `Restic`, the action value would be `file-system-backup`.
+  - if choosing volume snapshot, the action value would be `volume-snapshot`.
+  - if choosing skip backup of volume, the action value would be `skip`, and it will skip backup of volume no matter is `file-system-backup` or `volume-snapshot`.
+
+The policies could be extended for later other ways of backup, which means it may have some other `Action` value that will be assigned in the future.
+
+Both `file-system-backup` `volume-snapshot`, and `skip` could be partially or fully configured in the YAML file. And configuration could take effect only for the related action.
+
+#### Conditions
+The conditions are serials of volume attributes, the matched Volumes should meet all the volume attributes in one conditions configuration.
+
+##### Supported conditions
+In Velero 1.11, we want to support the volume attributes listed below:
+- capacity: matching volumes have the capacity that falls within this `capacity` range.
+- storageClass: matching volumes those with specified `storageClass`, such as `gp2`, `ebs-sc` in eks.
+- matching volumes that used specified volume sources.
+##### Parameters
+Parameters are optional for one specific action. For example, it could be `csi-snapshot-timeout: 6h` for CSI snapshot.
+
+#### Special rule definitions:
+- One single condition in `Conditions` with a specific key and empty value, which means the value matches any value. For example, if the `conditions.nfs` is `{}`, it means if `NFS` is used as `persistentVolumeSource` in Persistent Volume will be skipped no matter what the NFS server or NFS Path is.
+ 
+- The size of each single filter value should limit to 256 bytes in case of an unfriendly long variable assignment.
+ 
+- For capacity for PV or size for Volume, the value should include the lower value and upper value concatenated by commas. And it has several combinations below:
+  - "0,5Gi" or "0Gi,5Gi" which means capacity or size matches from 0 to 5Gi, including value 0 and value 5Gi
+  - ",5Gi" which is equal to "0,5Gi"
+  - "5Gi," which means capacity or size matches larger than 5Gi, including value 5Gi
+  - "5Gi" which is not supported and will be failed in validating configuration.
+ 
+### Configmap Reference
+Currently, resources policies are defined in `BackupSpec` struct, it will be more and more bloated with adding more and more filters which makes the size of `Backup` CR bigger and bigger, so we want to store the resources policies in configmap, and `Backup` CRD reference to current configmap.
+
+the `configmap` user created would be like this:
+```yaml
+apiVersion: v1
+data:
+ policies.yaml:
+  ----
+  version: v1
+  volumePolicies:
+  - conditions:
+      capacity: "0,100Gi"
+      csi:
+        driver: aws.ebs.csi.driver
+        fsType: ext4
+      storageClass:
+      - gp2
+      - ebs-sc
+    action:
+      type: volume-snapshot
+      parameters:
+        volume-snapshot-timeout: "6h"
+kind: ConfigMap
+metadata:
+ creationTimestamp: "2023-01-16T14:08:12Z"
+ name: backup01
+ namespace: velero
+ resourceVersion: "17891025"
+ uid: b73e7f76-fc9e-4e72-8e2e-79db717fe9f1
+```
+
+A new variable `resourcePolices` would be added into `BackupSpec`, it's value is assigned with the current resources policy configmap
+```yaml
+apiVersion: velero.io/v1
+kind: Backup
+metadata:
+  name: backup-1
+  spec:
+    resourcePolices:
+      refType: Configmap
+      ref: backup01
+    ...
+```
+The configmap only stores those assigned values, not the whole resources policies.
+ 
+The name of the configmap is `$BackupName`, and it's in Velero install namespace.
+ 
+#### Resource policies configmap related
+The life cycle of resource policies configmap is managed by the user instead of Velero, which could make it more flexible and easy to maintain.
+- The resource policies configmap will remain in the cluster until the user deletes it.
+- Unlike backup, the resource policies configmap will not sync to the new cluster. So if the user wants to use one resource policies that do not sync to the new cluster, the backup will fail with resource policies not found.
+- One resource policies configmap could be used by multiple backups.
+- If the backup referenced resource policies configmap is been deleted, it won't affect the already existing backups, but if the user wants to reference the deleted configmap to create one new backup, it will fail with resource policies not found.
+
+#### Versioning
+We want to introduce the version field in the YAML data to contain break changes. Therefore, we won't follow a semver paradigm, for example in v1.11 the data look like this:
+```yaml
+version: v1
+volumePolicies:
+  ....
+```
+Hypothetically, in v1.12 we add new fields like clusterResourcePolicies, the version will remain as v1 b/c this change is backward compatible:
+```yaml
+version: v1
+volumePolicies:
+  ....
+clusterResourcePolicies:
+  ....
+```
+Suppose in v1.13, we have to introduce a break change, at this time we will bump up the version:
+```yaml
+version: v2
+# This is just an example, we should try to avoid break change
+volume-policies: 
+  ....
+```
+We only support one version in Velero, so it won't be recognized if backup using a former version of YAML data.
+
+#### Multiple versions supporting
+To manage the effort for maintenance, we will only support one version of the data in Velero. Suppose that there is one break change for the YAML data in Velero v1.13, we should bump up the config version to v2, and v2 is only supported in v1.13. For the existing data with version: v1, it should migrate them when the Velero startup, this won't hurt the existing backup schedule CR as it only references the configmap. To make the migration easier, the configmap for such resource filter policies should be labeled manually before Velero startup like this, Velero will migrate the labeled configmap.
+
+We only support migrating from the previous version to the current version in case of complexity in data format conversion, which users could regenerate configmap in the new YAML data version, and it is easier to do version control.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+# This label can be optional but if this is not set, the backup will fail after the breaking change and the user will need to update the data manually 
+    velero.io/resource-filter-policies: "true"
+  name: example
+  namespace: velero
+data:
+  .....
+```
+### Display of resources policies
+As the resource policies configmap is referenced by backup CR, the policies in configmap are not so intuitive, so we need to integrate policies in configmap to the output of the command `velero backup describe`, and make it more readable.
+ 
+## Compatibility
+Currently, we have these resources filters:
+- IncludedNamespaces
+- ExcludedNamespaces
+- IncludedResources
+- ExcludedResources
+- LabelSelector
+- OrLabelSelectors
+- IncludeClusterResources
+- UseVolumeSnapshots
+- velero.io/exclude-from-backup=true
+- backup.velero.io/backup-volumes-excludes
+- backup.velero.io/backup-volumes
+- backup.velero.io/must-include-additional-items
+ 
+So it should be careful with the combination of volumes resources policies and the above resources filters.
+- When volume resource policies conflict with the above resource filters, we should respect the above resource filters. For example, if the user used the opt-out approach to `backup.velero.io/backup-volumes-excludes` annotation on the pod and also defined include volume in volumes resources filters configuration, we should respect the opt-out approach to skip backup of the volume.
+- If volume resource policies conflict with themselves, the first matched policy will be respect.
+
+## Implementation
+This implementation should be included in Velero v1.11.0
+
+Currently, in Velero v1.11.0 we only support `Action`
+ `skip`, and support `file-system-backup` and `volume-snapshot` for the later version. And `Parameters` in `Action` is also not supported in v1.11.0, we will support in a later version.
+
+In Velero 1.11, we supported Conditions and format listed below:
+ - capacity
+    ```yaml
+    capacity: "10Gi,100Gi" // match volume has the size between 10Gi and 100Gi
+    ```
+ - storageClass
+    ```yaml
+    storageClass: // match volume has the storage class gp2 or ebs-sc
+     - gp2
+     - ebs-sc
+    ```
+- volume sources (currently only support below format and attributes)
+  1. Specify the volume source name, the name could be `nfs`, `rbd`, `iscsi`, `csi` etc.
+     ```yaml
+     nfs : {} // match any volume has nfs volume source
+     
+     csi : {} // match any volume has csi volume source
+     ```
+
+  2. Specify details for the related volume source (currently we only support csi driver filter and nfs server or path filter)
+     ```yaml
+     csi: // match volume has nfs volume source and using `aws.efs.csi.driver`
+       driver: aws.efs.csi.driver 
+     
+     nfs: // match volume has nfs volume source and using below server and path
+       server: 192.168.200.90
+       path: /mnt/nfs
+     ```
+The conditions also could be extended in later versions, such as we could further supporting filtering other volume source detail not only NFS and CSI.
+
+## Alternatives Considered
+### Configmap VS CRD
+Here we support the user define the YAML config file and storing the resources policies into configmap, also we could define one resource's policies CRD and store policies imported from the user-defined config file in the related CR.
+
+But CRD is more like one kind of resource with status, Kubernetes API Server handles the lifecycle of a CR and handles it in different statuses. Compared to CRD, Configmap is more focused to store data.
+
+## Open Issues
+Should we support more than one version of filter policies configmap?

design/Implemented/plugin-versioning.md

@@ -135,8 +135,11 @@ type ObjectStore interface {
 
 The proto service definitions of the plugins will also be versioned and arranged by their plugin kind.
 Currently, all the proto definitions reside under `pkg/plugin/proto` in a file corresponding to their plugin kind.
-These files will be rearranged to be grouped by kind and then versioned: `pkg/plugin/proto/<plugin_kind>/<version>`.
-The scripts to compile the proto service definitions will need to be updated to place the generated Go code under a matching directory structure.
+These files will be rearranged to be grouped by kind and then versioned: `pkg/plugin/proto/<plugin_kind>/<version>`,
+except for the current v1 plugins. Those will remain in their current package/location for backwards compatibility.
+This will allow plugin images built with earlier versions of velero to work with the latest velero (for v1 plugins
+only). The go_package option will be added to all proto service definitions to allow the proto compilation script
+to place the generated go code for each plugin api version in the proper go package directory.
 
 It is not possible to import an existing proto service into a new one, so any methods will need to be duplicated across versions if they are required by the new version.
 The message definitions can be shared however, so these could be extracted from the service definition files and placed in a file that can be shared across all versions of the service.

design/Implemented/riav2-design.md

@@ -0,0 +1,130 @@
+# Design for RestoreItemAction v2 API
+
+## Abstract
+This design includes the changes to the RestoreItemAction (RIA) api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature.
+It also includes changes as required by the [Wait For Additional Items](wait-for-additional-items.md) feature.
+The BIA v2 interface will have three new methods, and the RestoreItemActionExecuteOutput() struct in the return from Execute() will have three optional fields added.
+If there are any additional RIA API changes that are needed in the same Velero release cycle as this change, those can be added here as well.
+
+## Background
+This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns.
+It is an optional feature, so plugins which don't need this feature can simply return an empty operation ID and the new methods can be no-ops.
+This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc.
+The other change allows Velero to wait until newly-restored AdditionalItems returned by a RIA plugin are ready before moving on to restoring the current item.
+
+## Goals
+- Allow for RIA Execute() to optionally initiate a long-running operation and report on operation status.
+- Allow for RIA to allow Velero to call back into the plugin to wait until AdditionalItems are ready before continuing with restore.
+
+## Non Goals
+- Allowing velero control over when the long-running operation begins.
+
+
+## High-Level Design
+As per the [Plugin Versioning](plugin-versioning.md) design, a new RIAv2 plugin `.proto` file will be created to define the GRPC interface.
+v2 go files will also be created in `plugin/clientmgmt/restoreitemaction` and `plugin/framework/restoreitemaction`, and a new PluginKind will be created.
+Changes to RestoreItemActionExecuteOutput will be made to the existing struct.
+Since the new fields are optional elements of the struct, the new enlarged struct will work with both v1 and v2 plugins.
+The velero Restore process will be modified to reference v2 plugins instead of v1 plugins.
+An adapter will be created so that any existing RIA v1 plugin can be executed as a v2 plugin when executing a restore.
+
+## Detailed Design
+
+### proto changes (compiled into golang by protoc)
+
+The v2 RestoreItemAction.proto will be like the current v1 version with the following changes:
+RestoreItemActionExecuteOutput gets three new fields (defined in the current (v1) RestoreItemAction.proto file:
+```
+message RestoreItemActionExecuteResponse {
+    bytes item = 1;
+    repeated ResourceIdentifier additionalItems = 2;
+    bool skipRestore = 3;
+    string operationID = 4;
+    bool waitForAdditionalItems = 5;
+    google.protobuf.Duration additionalItemsReadyTimeout = 6;
+}
+
+```
+The RestoreItemAction service gets three new rpc methods:
+```
+service RestoreItemAction {
+    rpc AppliesTo(RestoreItemActionAppliesToRequest) returns (RestoreItemActionAppliesToResponse);
+    rpc Execute(RestoreItemActionExecuteRequest) returns (RestoreItemActionExecuteResponse);
+    rpc Progress(RestoreItemActionProgressRequest) returns (RestoreItemActionProgressResponse);
+    rpc Cancel(RestoreItemActionCancelRequest) returns (google.protobuf.Empty);
+    rpc AreAdditionalItemsReady(RestoreItemActionItemsReadyRequest) returns (RestoreItemActionItemsReadyResponse);
+}
+
+```
+To support these new rpc methods, we define new request/response message types:
+```
+message RestoreItemActionProgressRequest {
+    string plugin = 1;
+    string operationID = 2;
+    bytes restore = 3;
+}
+
+message RestoreItemActionProgressResponse {
+    generated.OperationProgress progress = 1;
+}
+
+message RestoreItemActionCancelRequest {
+    string plugin = 1;
+    string operationID = 2;
+    bytes restore = 3;
+}
+
+message RestoreItemActionItemsReadyRequest {
+    string plugin = 1;
+    bytes restore = 2;
+    repeated ResourceIdentifier additionalItems = 3;
+}
+message RestoreItemActionItemsReadyResponse {
+    bool ready = 1;
+}
+
+```
+One new shared message type will be needed, as defined in the v2 BackupItemAction design:
+```
+message OperationProgress {
+    bool completed = 1;
+    string err = 2;
+    int64 completed = 3;
+    int64 total = 4;
+    string operationUnits = 5;
+    string description = 6;
+    google.protobuf.Timestamp started = 7;
+    google.protobuf.Timestamp updated = 8;
+}
+```
+
+In addition to the three new rpc methods added to the RestoreItemAction interface, there is also a new `Name()` method. This one is only actually used internally by Velero to get the name that the plugin was registered with, but it still must be defined in a plugin which implements RestoreItemActionV2 in order to implement the interface. It doesn't really matter what it returns, though, as this particular method is not delegated to the plugin via RPC calls. The new (and modified) interface methods for `RestoreItemAction` are as follows:
+```
+type BackupItemAction interface {
+...
+	Name() string
+...
+	Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error)
+	Cancel(operationID string, backup *api.Restore) error
+	AreAdditionalItemsReady(AdditionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error)
+...
+}
+type RestoreItemActionExecuteOutput struct {
+	UpdatedItem            runtime.Unstructured
+	AdditionalItems        []ResourceIdentifier
+	SkipRestore            bool
+	OperationID            string
+	WaitForAdditionalItems bool
+}
+
+```
+
+A new PluginKind, `RestoreItemActionV2`, will be created, and the restore process will be modified to use this plugin kind.
+See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc.
+
+
+## Compatibility
+The included v1 adapter will allow any existing RestoreItemAction plugin to work as expected, with no-op AreAdditionalItemsReady(), Progress(), and Cancel() methods.
+
+## Implementation
+This will be implemented during the Velero 1.11 development cycle.

design/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md

@@ -0,0 +1,483 @@
+# Unified Repository & Kopia Integration Design
+
+## Glossary & Abbreviation
+
+**BR**: Backup & Restore  
+**Backup Storage**: The storage that meets BR requirements, for example, scalable, durable, cost-effective, etc., therefore, Backup Storage is usually implemented as Object storage or File System storage, it may be on-premise or in cloud. Backup Storage is not BR specific necessarily, so it usually doesn’t provide most of the BR related features. On the other hand, storage vendors may provide BR specific storages that include some BR features like deduplication, compression, encryption, etc. For a standalone BR solution (i.e. Velero), the Backup Storage is not part of the solution, it is provided by users, so the BR solution should not assume the BR related features are always available from the Backup Storage.  
+**Backup Repository**: Backup repository is layered between BR data movers and Backup Storage to provide BR related features. Backup Repository is a part of BR solution, so generally, BR solution by default leverages the Backup Repository to provide the features because Backup Repository is always available; when Backup Storage provides duplicated features, and the latter is more beneficial (i.e., performance is better), BR solution should have the ability to opt to use the Backup Storage’s implementation.  
+**Data Mover**: The BR module to read/write data from/to workloads, the aim is to eliminate the differences of workloads.  
+**TCO**: Total Cost of Ownership. This is a general criteria for products/solutions, but also means a lot for BR solutions. For example, this means what kind of backup storage (and its cost) it requires, the retention policy of backup copies, the ways to remove backup data redundancy, etc.   
+**RTO**: Recovery Time Objective. This is the duration of time that users’ business can recover after a disaster.  
+
+## Background
+
+As a Kubernetes BR solution, Velero is pursuing the capability to back up data from the volatile and limited production environment into the durable, heterogeneous and scalable backup storage. This relies on two parts:  
+
+- Move data from various production workloads. The data mover has this role. Depending on the type of workload, Velero needs different data movers. For example, file system data mover, block data mover, and data movers for specific applications. At present, Velero supports moving file system data from PVs through Restic, which plays the role of the File System Data Mover.  
+- Persist data in backup storage. For a BR solution, this is the responsibility of the backup repository. Specifically, the backup repository is required to:  
+  - Efficiently save data so as to reduce TCO. For example, deduplicate and compress the data before saving it
+  - Securely save data so as to meet security criteria. For example, encrypt the data on rest, make the data immutable after backup, and detect/protect from ransomware
+  - Efficiently retrieve data during restore so as to meet RTO. For example, restore a small unit of data or data associated with a small span of time 
+  - Effectively manage data from all kinds of data movers in all kinds of backup storage. This means 2 things: first, apparently, backup storages are different from each other; second, some data movers may save quite different data from others, for example, some data movers save a portion of the logical object for each backup and need to visit and manage the portions as an entire logic object, aka. incremental backup. The backup repository needs to provide unified functionalities to eliminate the differences from the both ends
+  - Provide scalabilities so that users could assign resources (CPU, memory, network, etc.) in a flexible way to the backup repository since backup repository contains resource consuming modules
+
+At present, Velero provides some of these capabilities by leveraging Restic (e.g., deduplication and  encryption on rest). This means that in addition to being a data mover for file system level data, Restic also plays the role of a backup repository, albeit one that is incomplete and limited:  
+
+- Restic is an inseparable unit made up of a file system data mover and a repository. This means that the repository capabilities are only available for Restic file system backup. We cannot provide the same capabilities to other data movers using Restic.
+- The backup storage Velero supports through our Restic backup path depends on the storage Restic supports. As a result, if there is a requirement to introduce backup storage that Restic doesn’t support, we have no way to make it.
+- There is no way to enhance or extend the repository capabilities, because of the same reason – Restic is an inseparable unit, we cannot insert one or more customized layers to make the enhancements and extensions.
+
+Moreover, as reflected by user-reported issues, Restic seems to have many performance issues on both the file system data mover side and the repository side.  
+
+On the other hand, based on a previous analysis and testing, we found that Kopia has better performance, with more features and more suitable to fulfill Velero’s repository targets (Kopia’s architecture divides modules more clearly according to their responsibilities, every module plays a complete role with clear interfaces. This makes it easier to take individual modules to Velero without losing critical functionalities).  
+
+## Goals
+
+- Define a Unified Repository Interface that various data movers could interact with. This is for below purposes:
+  - All kinds of data movers acquire the same set of backup repository capabilities very easily
+  - Provide the possibility to plugin in different backup repositories/backup storages without affecting the upper layers
+  - Provide the possibility to plugin in modules between data mover and backup repository, so as to extend the repository capabilities
+  - Provide the possibility to scale the backup repository without affecting the upper layers
+- Use Kopia repository to implement the Unified Repository
+- Use Kopia uploader as the file system data mover for Pod Volume Backup
+- Have Kopia uploader calling the Unified Repository Interface and save/retrieve data to/from the Unified Repository
+- Make Kopia uploader generic enough to move any file system data so that other data movement cases could use it
+- Use the existing logic or add new logic to manage the unified repository and Kopia uploader
+- Preserve the legacy Restic path, this is for the consideration of backward compatibility
+
+## Non-Goals
+
+- The Unified Repository supports all kinds of data movers to save logic objects into it. How these logic objects are organized for a specific data mover (for example, how a volume’s block data is organized and represented by a unified repository object) should be included in the related data mover design.
+- At present, Velero saves Kubernetes resources, backup metedata, debug logs separately. Eventually, we want to save them in the Unified Repository. How to organize these data into the Unified Repository should be included in a separate design.
+- For PodVolume BR, this design focuses on the data path only, other parts beyond the data read/write and data persistency are irrelevant and kept unchanged. 
+- Kopia uploader is made generic enough to move any file system data. How it is integrated in other cases, is irrelevant to this design. Take CSI snapshot backup for example, how the snapshot is taken and exposed to Kopia uploader should be included in the related data mover design.
+- The adanced modes of the Unified Repository, for example, backup repository/storage plugin, backup repository extension, etc. are not included in this design. We will have separate designs to cover them whenever necessary.
+
+## Architecture of Unified Repository
+
+Below shows the primary modules and their responsibilities:
+
+- Kopia uploader, as been well isolated, could move all file system data either from the production PV (as Velero’s PodVolume BR does), or from any kind of snapshot (i.e., CSI snapshot).
+- Unified Repository Interface, data movers call the Unified Repository Interface to write/read data to/from the Unified Repository.
+- Kopia repository layers, CAOS and CABS, work as the backup repository and expose the Kopia Repository interface.
+- A Kopia Repository Library works as an adapter between Unified Repository Interface and Kopia Repository interface. Specifically, it implements Unified Repository Interface and calls Kopia Repository interface.
+- At present, there is only one kind of backup repository -- Kopia Repository. If a new backup repository/storage is required, we need to create a new Library as an adapter to the Unified Repository Interface
+- At present, the Kopia Repository works as a single piece in the same process of the caller, in future, we may run its CABS into a dedicated process or node.
+- At present, we don’t have a requirement to extend the backup repository, if needed, an extra module could be added as an upper layer into the Unified Repository without changing the data movers.  
+
+Neither Kopia uploader nor Kopia Repository is invoked through CLI, instead, they are invoked through code interfaces, because we need to do lots of customizations.  
+
+The Unified Repository takes two kinds of data:
+- Unified Repository Object: This is the user's logical data, for example, files/directories, blocks of a volume, data of a database, etc.
+- Unified Repository Manifest: This could include all other data to maintain the object data, for example, snapshot information, etc.  
+
+For Unified Repository Object/Manifest, a brief guidance to data movers are as below:
+- Data movers treat the simple unit of data they recognize as an Object. For example, file system data movers treat a file or a directory as an Object; block data movers treat a volume as an Object. However, it is unnecessary that every data mover has a unique data format in the Unified Repository, to the opposite, it is recommended that data movers could share the data formats unless there is any reason not to, in this way, the data generated by one data mover could be used by other data movers.
+- Data movers don't need to care about the differences between full and incremental backups regarding the data organization. Data movers always have full views of their objects, if an object is partially written, they use the object writer's Seek function to skip the unchanged parts
+- Unified Repository may divide the data movers' logical Object into sub-objects or slices, or append internal metadata, but they are transparent to data movers 
+- Every Object has an unified identifier, in order to retrieve the Object later, data movers need to save the identifiers into the snapshot information. The snapshot information is saved as a Manifest.
+- Manifests could hold any kind of small piece data in a K-V manner. Inside the backup repository, these kinds of data may be processed differently from Object data, but it is transparent to data movers.
+- A Manifest also has an unified identifier, the Unified Repository provides the capabilities to list all the Manifests or a specified Manifest by its identifier, or a specified Manifest by its name, or a set of Manifests by their labels.
+ 
+![A Unified Repository Architecture](unified-repo.png)
+
+Velero by default uses the Unified Repository for all kinds of data movement, it is also able to integrate with other data movement paths from any party, for any purpose. Details are concluded as below:
+
+- Built-in Data Path: this is the default data movement path, which uses Velero built-in data movers to backup/restore workloads, the data is written to/read from the Unified Repository.
+- Data Mover Replacement: Any party could write its own data movers and plug them into Velero. Meanwhile, these plugin data movers could also write/read data to/from Velero’s Unified Repository so that these data movers could expose the same capabilities that provided by the Unified Repository. In order to do this, the data mover providers need to call the Unified Repository Interface from inside their plugin data movers.
+- Data Path Replacement: Some vendors may already have their own data movers and backup repository and they want to replace Velero’s entire data path (including data movers and backup repository). In this case, the providers only need to implement their plugin data movers, all the things downwards are a black box to Velero and managed by providers themselves (including API call, data transport, installation, life cycle management, etc.). Therefore, this case is out of the scope of Unified Repository.  
+![A Scope](scope.png)  
+
+# Detailed Design
+
+## The Unified Repository Interface  
+Below are the definitions of the Unified Repository Interface. All the functions are synchronization functions.   
+```
+// BackupRepoService is used to initialize, open or maintain a backup repository
+type BackupRepoService interface {
+	// Init creates a backup repository or connect to an existing backup repository.
+	// repoOption: option to the backup repository and the underlying backup storage.
+	// createNew: indicates whether to create a new or connect to an existing backup repository.
+	Init(ctx context.Context, repoOption RepoOptions, createNew bool) error
+
+	// Open opens an backup repository that has been created/connected.
+	// repoOption: options to open the backup repository and the underlying storage.
+	Open(ctx context.Context, repoOption RepoOptions) (BackupRepo, error)
+
+	// Maintain is periodically called to maintain the backup repository to eliminate redundant data.
+	// repoOption: options to maintain the backup repository.
+	Maintain(ctx context.Context, repoOption RepoOptions) error
+
+	// DefaultMaintenanceFrequency returns the defgault frequency of maintenance, callers refer this
+	// frequency to maintain the backup repository to get the best maintenance performance
+	DefaultMaintenanceFrequency() time.Duration
+}
+
+// BackupRepo provides the access to the backup repository
+type BackupRepo interface {
+	// OpenObject opens an existing object for read.
+	// id: the object's unified identifier.
+	OpenObject(ctx context.Context, id ID) (ObjectReader, error)
+
+	// GetManifest gets a manifest data from the backup repository.
+	GetManifest(ctx context.Context, id ID, mani *RepoManifest) error
+
+	// FindManifests gets one or more manifest data that match the given labels
+	FindManifests(ctx context.Context, filter ManifestFilter) ([]*ManifestEntryMetadata, error)
+
+	// NewObjectWriter creates a new object and return the object's writer interface.
+	// return: A unified identifier of the object on success.
+	NewObjectWriter(ctx context.Context, opt ObjectWriteOptions) ObjectWriter
+
+	// PutManifest saves a manifest object into the backup repository.
+	PutManifest(ctx context.Context, mani RepoManifest) (ID, error)
+
+	// DeleteManifest deletes a manifest object from the backup repository.
+	DeleteManifest(ctx context.Context, id ID) error
+
+	// Flush flushes all the backup repository data
+	Flush(ctx context.Context) error
+
+	// Time returns the local time of the backup repository. It may be different from the time of the caller
+	Time() time.Time
+
+	// Close closes the backup repository
+	Close(ctx context.Context) error
+
+type ObjectReader interface {
+	io.ReadCloser
+	io.Seeker
+
+	// Length returns the logical size of the object
+	Length() int64
+}
+ 
+type ObjectWriter interface {
+	io.WriteCloser
+
+	// Seeker is used in the cases that the object is not written sequentially
+	io.Seeker
+
+	// Checkpoint is periodically called to preserve the state of data written to the repo so far.
+	// Checkpoint returns a unified identifier that represent the current state.
+	// An empty ID could be returned on success if the backup repository doesn't support this.
+	Checkpoint() (ID, error)
+
+	// Result waits for the completion of the object write.
+	// Result returns the object's unified identifier after the write completes.
+	Result() (ID, error)
+}
+```
+ 
+Some data structure & constants used by the interfaces:  
+```
+type RepoOptions struct {
+	// StorageType is a repository specific string to identify a backup storage, i.e., "s3", "filesystem"
+	StorageType string
+	// RepoPassword is the backup repository's password, if any
+	RepoPassword string
+	// ConfigFilePath is a custom path to save the repository's configuration, if any
+	ConfigFilePath string
+	// GeneralOptions takes other repository specific options
+	GeneralOptions map[string]string
+	// StorageOptions takes storage specific options
+	StorageOptions map[string]string
+	// Description is a description of the backup repository/backup repository operation.
+	// It is for logging/debugging purpose only and doesn't control any behavior of the backup repository.
+	Description string
+}
+
+// ObjectWriteOptions defines the options when creating an object for write
+type ObjectWriteOptions struct {
+	FullPath    string // Full logical path of the object
+	DataType    int    // OBJECT_DATA_TYPE_*
+	Description string // A description of the object, could be empty
+	Prefix      ID     // A prefix of the name used to save the object
+	AccessMode  int    // OBJECT_DATA_ACCESS_*
+	BackupMode  int    // OBJECT_DATA_BACKUP_*
+}
+
+const (
+	// Below consts descrbe the data type of one object.
+	// Metadata: This type describes how the data is organized.
+	// For a file system backup, the Metadata describes a Dir or File.
+	// For a block backup, the Metadata describes a Disk and its incremental link.
+	ObjectDataTypeUnknown  int = 0
+	ObjectDataTypeMetadata int = 1
+	ObjectDataTypeData     int = 2
+
+	// Below consts defines the access mode when creating an object for write
+	ObjectDataAccessModeUnknown int = 0
+	ObjectDataAccessModeFile    int = 1
+	ObjectDataAccessModeBlock   int = 2
+
+	ObjectDataBackupModeUnknown int = 0
+	ObjectDataBackupModeFull    int = 1
+	ObjectDataBackupModeInc     int = 2
+)
+
+// ManifestEntryMetadata is the metadata describing one manifest data
+type ManifestEntryMetadata struct {
+	ID      ID                // The ID of the manifest data
+	Length  int32             // The data size of the manifest data
+	Labels  map[string]string // Labels saved together with the manifest data
+	ModTime time.Time         // Modified time of the manifest data
+}
+ 
+type RepoManifest struct {
+	Payload  interface{}            // The user data of manifest
+	Metadata *ManifestEntryMetadata // The metadata data of manifest
+}
+ 
+type ManifestFilter struct {
+    Labels map[string]string
+}
+```  
+
+## Workflow
+
+### Backup & Restore Workflow
+
+We preserve the bone of the existing BR workflow, that is:
+
+- Still use the Velero Server pod and VeleroNodeAgent daemonSet (originally called Restic daemonset) pods to hold the corresponding controllers and modules
+- Still use the Backup/Restore CR and BackupRepository CR (originally called ResticRepository CR) to drive the BR workflow
+
+The modules in gray color in below diagram are the existing modules and with no significant changes.  
+In the new design, we will have separate and independent modules/logics for backup repository and uploader (data mover), specifically:
+
+- Repository Provider provides functionalities to manage the backup repository. For example, initialize a repository, connect to a repository, manage the snapshots in the repository, maintain a repository, etc.
+- Uploader Provider provides functionalities to run a backup or restore.
+
+The Repository Provider and Uploader Provider use options to choose the path --- legacy path vs. new path (Kopia uploader + Unified Repository). Specifically, for legacy path, Repository Provider will manage Restic Repository only, otherwise, it manages Unified Repository only; for legacy path, Uploader Provider calls Restic to do the BR, otherwise, it calls Kopia uploader to do the BR.  
+
+In order to manage Restic Repository, the Repository Provider calls Restic Repository Provider, the latter invokes the existing Restic CLIs.  
+In order to manage Unified Repository, the Repository Provider calls Unified Repository Provider, the latter calls the Unified Repository module through the udmrepo.BackupRepoService interface. It doesn’t know how the Unified Repository is implemented necessarily.  
+In order to use Restic to do BR, the Uploader Provider calls Restic Uploader Provider, the latter invokes the existing Restic CLIs.  
+In order to use Kopia to do BR, the Uploader Provider calls Kopia Uploader Provider, the latter do the following things:
+
+- Call Unified Repository through the udmrepo.BackupRepoService interface to open the unified repository for read/write. Again, it doesn’t know how the Unified Repository is implemented necessarily. It gets a BackupRepo’s read/write handle after the call succeeds
+- Wrap the BackupRepo handle into a Kopia Shim which implements Kopia Repository interface
+- Call the Kopia Uploader. Kopia Uploader is a Kopia module without any change, so it only understands Kopia Repository interface
+- Kopia Uploader starts to backup/restore the corresponding PV’s file system data and write/read data to/from the provided Kopia Repository implementation, that is, Kopia Shim here
+- When read/write calls go into Kopia Shim, it in turn calls the BackupRepo handle for read/write
+- Finally, the read/write calls flow to Unified Repository module
+
+The Unified Repository provides all-in-one functionalities of a Backup Repository and exposes the Unified Repository Interface. Inside, Kopia Library is an adapter for Kopia Repository to translate the Unified Repository Interface calls to Kopia Repository interface calls.  
+Both Kopia Shim and Kopia Library rely on Kopia Repository interface, so we need to have some Kopia version control. We may need to change Kopia Shim and Kopia Library when upgrading Kopia to a new version and the Kopia Repository interface has some changes in the new version.
+![A BR Workflow](br-workflow.png)  
+The modules in blue color in below diagram represent the newly added modules/logics or reorganized logics.  
+The modules in yellow color in below diagram represent the called Kopia modules without changes.  
+
+### Delete Snapshot Workflow
+The Delete Snapshot workflow follows the similar manner with BR workflow, that is, we preserve the upper-level workflows until the calls reach to BackupDeletionController, then:
+- Leverage Repository Provider to switch between Restic implementation and Unified Repository implementation in the same mechanism as BR
+- For Restic implementation, the Restic Repository Provider invokes the existing “Forget” Restic CLI
+- For Unified Repository implementation, the Unified Repository Provider calls udmrepo.BackupRepo’s DeleteManifest to delete a snapshot
+![A Snapshot Deletion Workflow](snapshot-deletion-workflow.png) 
+
+### Maintenance Workflow
+Backup Repository/Backup Storage may need to periodically reorganize its data so that it could guarantee its QOS during the long-time service. Some Backup Repository/Backup Storage does this in background automatically, so the user doesn’t need to interfere; some others need the caller to explicitly call their maintenance interface periodically. Restic and Kopia both go with the second way, that is, Velero needs to periodically call their maintenance interface.  
+Velero already has an existing workflow to call Restic maintenance (it is called “Prune” in Restic, so Velero uses the same word). The existing workflow is as follows:
+- The Prune is triggered at the time of the backup
+- When a BackupRepository CR (originally called ResticRepository CR) is created by PodVolumeBackup/Restore Controller, the BackupRepository controller checks if it reaches to the Prune Due Time, if so, it calls PruneRepo  
+- In the new design, the Repository Provider implements PruneRepo call, it uses the same way to switch between Restic Repository Provider and Unified Repository Provider, then:
+  - For Restic Repository, Restic Repository Provider invokes the existing “Prune” CLI of Restic
+  - For Unified Repository, Unified Repository Provider calls udmrepo.BackupRepoService’s Maintain function
+
+Kopia has two maintenance modes – the full maintenance and quick maintenance. There are many differences between full and quick mode, but briefly speaking, quick mode only processes the hottest data (primarily, it is the metadata and index data), so quick maintenance is much faster than full maintenance. On the other hand, quick maintenance also scatters the burden of full maintenance so that the full maintenance could finish fastly and make less impact. We will also take this quick maintenance into Velero.  
+We will add a new Due Time to Velero, finally, we have two Prune Due Time:
+- Normal Due Time: For Restic, this will invoke Restic Prune; for Unified Repository, this will invoke udmrepo.BackupRepoService’s Maintain(full) call and finally call Kopia’s full maintenance
+- Quick Due Time: For Restic, this does nothing; for Unified Repository, this will invoke udmrepo.BackupRepoService’s Maintain(quick) call and finally call Kopia’s quick maintenance  
+
+We assign different values to Normal Due Time and Quick Due Time, as a result of which, the quick maintenance happens more frequently than full maintenance.  
+![A Maintenance Workflow](maintenance-workflow.png) 
+
+### Progress Update
+Because Kopia Uploader is an unchanged Kopia module, we need to find a way to get its progress during the BR.  
+Kopia Uploader accepts a Progress interface to update rich information during the BR, so the Kopia Uploader Provider will implement a Kopia’s Progress interface and then pass it to Kopia Uploader during its initialization.  
+In this way, Velero will be able to get the progress as shown in the diagram below.  
+![A Progress Update](progress-update.png) 
+
+### Logs
+In the current design, Velero is using two unchanged Kopia modules --- the Kopia Uploader and the Kopia Repository. Both will generate debug logs during their run. Velero will collect these logs in order to aid the debug.  
+Kopia’s Uploader and Repository both get the Logger information from the current GO Context, therefore, the Kopia Uploader Provider/Kopia Library could set the Logger interface into the current context and pass the context to Kopia Uploader/Kopia Repository.  
+Velero will set Logger interfaces separately for Kopia Uploader and Kopia Repository. In this way, the Unified Repository could serve other data movers without losing the debug log capability; and the Kopia Uploader could write to any repository without losing the debug log capability.  
+Kopia’s debug logs will be written to the same log file as Velero server or VeleroNodeAgent daemonset, so Velero doesn’t need to upload/download these debug logs separately.  
+![A Debug Log for Uploader](debug-log-uploader.png) 
+![A Debug Log for Repository](debug-log-repository.png) 
+
+## Path Switch & Coexist
+As mentioned above, There will be two paths. The related controllers need to identify the path during runtime and adjust its working mode.  
+According to the requirements, path changing is fulfilled at the backup/restore level. In order to let the controllers know the path, we need to add some option values. Specifically, there will be option/mode values for path selection in two places:
+- Add the “uploader-type” option as a parameter of the Velero server. The parameters will be set by the installation. Currently the option has two values, either "restic" or "kopia" (in future, we may add other file system uploaders, then we will have more values).
+- Add a "uploaderType" value in the PodVolume Backup/Restore CR and a "repositoryType" value in the BackupRepository CR. "uploaderType" currently has two values , either "restic" or "kopia";  "repositoryType" currently has two values, either "restic" or "kopia" (in future, the Unified Repository could opt among multiple backup repository/backup storage, so there may be more values. This is a good reason that repositoryType is a multivariate flag, however, in which way to opt among the backup repository/backup storage is not covered in this PR). If the values are missing in the CRs, it by default means "uploaderType=restic" and "repositoryType=restic", so the legacy CRs are handled correctly by Restic.  
+
+The corresponding controllers handle the CRs by checking the CRs' path value. Some examples are as below:
+- The PodVolume BR controller checks the "uploaderType" value from PodVolume CRs and decide its working path
+- The BackupRepository controller checks the "repositoryType" value from BackupRepository CRs and decide its working path
+- The Backup controller that runs in Velero server checks its “uploader-type” parameter to decide the path for the Backup it is going to create and then create the PodVolume Backup CR and BackupRepository CR
+- The Restore controller checks the Backup, from which it is going to restore, for the path and then create the PodVolume Restore CR and BackupRepository CR
+
+As described above, the “uploader-type” parameter of the Velero server is only used to decide the path when creating a new Backup, for other cases, the path selection is driven by the related CRs. Therefore, we only need to add this parameter to the Velero server.  
+
+## Velero CR Name Changes
+We will change below CRs' name to make them more generic:
+- "ResticRepository" CR to "BackupRepository" CR  
+
+This means, we add a new CR type and deprecate the old one. As a result, if users upgrade from the old release, the old CRs will be orphaned, Velero will neither refer to it nor manage it, users need to delete these CRs manually.  
+As a side effect, when upgrading from an old release, even though the path is not changed, the BackupRepository gets created all the time, because Velero will not refer to the old CR's status. This seems to cause the repository to initialize more than once, however, it won't happen. In the BackupRepository controller, before initializing a repository, it always tries to connect to the repository first, if it is connectable, it won't do the initialization.  
+When backing up with the new release, Velero always creates BackupRepository CRs instead of ResticRepository CRs.  
+When restoring from an old backup, Velero always creates BackupRepository CRs instead of ResticRepository CRs.  
+When there are already backups or restores running during the upgrade, since after upgrade, the Velero server pods and VeleroNodeAgent daemonset pods are restarted, the existing backups/restores will fail immediately. 
+
+## Storage Configuration
+The backup repository needs some parameters to connect to various backup storage. For example, for a S3 compatible storage, the parameters may include bucket name, region, endpoint, etc. Different backup storage have totally different parameters. BackupRepository CRs, PodVolume Backup CRs and PodVolume Restore CRs save these parameters in their spec, as a string called repoIdentififer. The format of the string is for S3 storage only, it meets Restic CLI's requirements but is not enough for other backup repository. On the other hand, the parameters that are used to generate the repoIdentififer all come from the BackupStorageLocation. The latter has a map structure that could take parameters from any storage kind.  
+Therefore, for the new path, Velero uses the information in the BackupStorageLocation directly. That is, whenever Velero needs to initialize/connect to the Unified Repository, it acquires the storage configuration from the corresponding BackupStorageLocation. Then no more elements will be added in BackupRepository CRs, PodVolume Backup CRs or PodVolume Restore CRs.  
+The legacy path will be kept as is. That is, Velero still sets/gets the repoIdentififer in BackupRepository CRs, PodVolume Backup CRs and PodVolume Restore CRs and then passes to Restic CLI.  
+
+## Installation
+ We will add a new flag "--pod-volume-backup-uploader" during installation. The flag has 3 meanings:
+ - It indicates PodVolume BR as the default method to protect PV data over other methods, i.e., durable snapshot. Therefore, the existing --use-restic option will be replaced
+ - It indicates the file system uploader to be used by PodVolume BR
+ - It implies the backup repository type manner, Restic if pod-volume-backup-uploader=restic, Unified Repository in all other cases
+
+ The flag has below two values:  
+ **"Restic"**: it means Velero will use Restic to do the pod volume backup. Therefore, the Velero server deployment will be created as below:
+ ```
+    spec:
+      containers:
+      - args:
+        - server
+        - --features=
+        - --uploader-type=restic
+        command:
+        - /velero
+```
+The BackupRepository CRs and PodVolume Backup/Restore CRs created in this case are as below:
+```
+spec:
+  backupStorageLocation: default
+  maintenanceFrequency: 168h0m0s
+  repositoryType: restic
+  volumeNamespace: nginx-example
+```
+```
+spec:
+  backupStorageLocation: default
+  node: aks-agentpool-27359964-vmss000000
+  pod:
+    kind: Pod
+    name: nginx-stateful-0
+    namespace: nginx-example
+    uid: 86aaec56-2b21-4736-9964-621047717133   
+  tags:
+    ...
+  uploaderType: restic
+  volume: nginx-log
+```
+```
+spec:
+  backupStorageLocation: default
+  pod:
+    kind: Pod
+    name: nginx-stateful-0
+    namespace: nginx-example
+    uid: e56d5872-3d94-4125-bfe8-8a222bf0fcf1
+  snapshotID: 1741e5f1
+  uploaderType: restic
+  volume: nginx-log
+```
+ **"Kopia"**: it means Velero will use Kopia uploader to do the pod volume backup (so it will use Unified Repository as the backup target). Therefore, the Velero server deployment will be created as below:
+  ```
+    spec:
+      containers:
+      - args:
+        - server
+        - --features=
+        - --uploader-type=kopia
+        command:
+        - /velero
+```
+The BackupRepository CRs created in this case are hard set with "kopia" at present, sice Kopia is the only option as a backup repository. The PodVolume Backup/Restore CRs are created with "kopia" as well:
+```
+spec:
+  backupStorageLocation: default
+  maintenanceFrequency: 168h0m0s
+  repositoryType: kopia
+  volumeNamespace: nginx-example
+```
+```
+spec:
+  backupStorageLocation: default
+  node: aks-agentpool-27359964-vmss000000
+  pod:
+    kind: Pod
+    name: nginx-stateful-0
+    namespace: nginx-example
+    uid: 86aaec56-2b21-4736-9964-621047717133   
+  tags:
+    ...
+  uploaderType: kopia
+  volume: nginx-log
+```
+```
+spec:
+  backupStorageLocation: default
+  pod:
+    kind: Pod
+    name: nginx-stateful-0
+    namespace: nginx-example
+    uid: e56d5872-3d94-4125-bfe8-8a222bf0fcf1
+  snapshotID: 1741e5f1
+  uploaderType: kopia
+  volume: nginx-log
+```
+We will add the flag for both CLI installation and Helm Chart Installation. Specifically:
+- Helm Chart Installation: add the "--pod-volume-backup-uploader" flag into its value.yaml and then generate the deployments according to the value. Value.yaml is the user-provided configuration file, therefore, users could set this value at the time of installation. The changes in Value.yaml are as below:
+```
+          command:
+            - /velero
+          args:
+            - server
+          {{- with .Values.configuration }}
+            {{- if .pod-volume-backup-uploader "restic" }}
+            - --legacy
+            {{- end }} 
+```     
+- CLI Installation: add the "--pod-volume-backup-uploader" flag into the installation command line, and then create the two deployments accordingly. Users could change the option at the time of installation. The CLI is as below:  
+```velero install --pod-volume-backup-uploader=restic```  
+```velero install --pod-volume-backup-uploader=kopia``` 
+
+## Upgrade
+For upgrade, we allow users to change the path by specifying "--pod-volume-backup-uploader" flag in the same way as the fresh installation. Therefore, the flag change should be applied to the Velero server after upgrade. Additionally, We need to add a label to Velero server to indicate the current path, so as to provide an easy for querying it.  
+Moreover, if users upgrade from the old release, we need to change the existing Restic Daemonset name to VeleroNodeAgent daemonSet. The name change should be applied after upgrade.  
+The recommended way for upgrade is to modify the related Velero resource directly through kubectl, the above changes will be applied in the same way. We need to modify the Velero doc for all these changes.  
+
+## CLI
+Below Velero CLI or its output needs some changes:  
+- ```Velero backup describe```: the output should indicate the path  
+- ```Velero restore describe```: the output should indicate the path  
+- ```Velero restic repo get```: the name of this CLI should be changed to a generic one, for example, "Velero repo get"; the output of this CLI should print all the backup repository if Restic repository and Unified Repository exist at the same time  
+
+At present, we don't have a requirement for selecting the path during backup, so we don't change the ```Velero backup create``` CLI for now. If there is a requirement in future, we could simply add a flag similar to "--pod-volume-backup-uploader" to select the path.  
+
+## CR Example
+Below sample files demonstrate complete CRs with all the changes mentioned above:
+- BackupRepository CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-backup-repository-sample-yaml
+- PodVolumeBackup CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-pvb-sample-yaml
+- PodVolumeRestore CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-pvr-sample-yaml
+
+## User Perspective
+This design aims to provide a flexible backup repository layer and a generic file system uploader, which are fundermental for PodVolume and other data movements. Although this will make Velero more capable, at present, we don't pursue to expose differentiated features end to end. Specifically:  
+- By default, Velero still uses Restic for PodVolume BR
+- Even when changing to the new path, Velero still allows users to restore from the data backed up by Restic
+- The capability of PodVolume BR under the new path is kept the same as it under Restic path and the same as the existing PodVolume BR
+- The operational experiences are kept the same as much as possible, the known changes are listed below
+
+Below user experiences are changed for this design:
+- Installation CLI change: a new option is added to the installation CLI, see the Installation section for details
+- CR change: One or more existing CRs have been renamed, see the Velero CR Changes section for details
+- Velero CLI name and output change, see the CLI section for details
+- Velero daemonset name change
+- Wording Alignment: as the existing situation, many places are using the word of "Restic", for example, "default-volume-to-restic" option, most of them are not accurate anymore, we will change these words and give a detailed list of the changes  

design/vsv2-design.md

@@ -0,0 +1,84 @@
+# Design for VolumeSnapshotter v2 API
+
+## Abstract
+This design includes the changes to the VolumeSnapshotter api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature.
+The VolumeSnapshotter v2 interface will have two new methods.
+If there are any additional VolumeSnapshotter API changes that are needed in the same Velero release cycle as this change, those can be added here as well.
+
+## Background
+This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns.
+The existing snapshotID returned by CreateSnapshot will be used as the operation ID.
+This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc.
+
+## Goals
+- Allow for VolumeSnapshotter CreateSnapshot() to initiate a long-running operation and report on operation status.
+
+## Non Goals
+- Allowing velero control over when the long-running operation begins.
+
+
+## High-Level Design
+As per the [Plugin Versioning](plugin-versioning.md) design, a new VolumeSnapshotterv2 plugin `.proto` file will be created to define the GRPC interface.
+v2 go files will also be created in `plugin/clientmgmt/volumesnapshotter` and `plugin/framework/volumesnapshotter`, and a new PluginKind will be created.
+The velero Backup process will be modified to reference v2 plugins instead of v1 plugins.
+An adapter will be created so that any existing VolumeSnapshotter v1 plugin can be executed as a v2 plugin when executing a backup.
+
+## Detailed Design
+
+### proto changes (compiled into golang by protoc)
+
+The v2 VolumeSnapshotter.proto will be like the current v1 version with the following changes:
+The VolumeSnapshotter service gets two new rpc methods:
+```
+service VolumeSnapshotter {
+    rpc Init(VolumeSnapshotterInitRequest) returns (Empty);
+    rpc CreateVolumeFromSnapshot(CreateVolumeRequest) returns (CreateVolumeResponse);
+    rpc GetVolumeInfo(GetVolumeInfoRequest) returns (GetVolumeInfoResponse);
+    rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse);
+    rpc DeleteSnapshot(DeleteSnapshotRequest) returns (Empty);
+    rpc GetVolumeID(GetVolumeIDRequest) returns (GetVolumeIDResponse);
+    rpc SetVolumeID(SetVolumeIDRequest) returns (SetVolumeIDResponse);
+    rpc Progress(VolumeSnapshotterProgressRequest) returns (VolumeSnapshotterProgressResponse);
+    rpc Cancel(VolumeSnapshotterCancelRequest) returns (google.protobuf.Empty);
+}
+```
+To support these new rpc methods, we define new request/response message types:
+```
+message VolumeSnapshotterProgressRequest {
+    string plugin = 1;
+    string snapshotID = 2;
+}
+
+message VolumeSnapshotterProgressResponse {
+    generated.OperationProgress progress = 1;
+}
+
+message VolumeSnapshotterCancelRequest {
+    string plugin = 1;
+    string operationID = 2;
+}
+
+```
+One new shared message type will be needed, as defined in the v2 BackupItemAction design:
+```
+message OperationProgress {
+    bool completed = 1;
+    string err = 2;
+    int64 completed = 3;
+    int64 total = 4;
+    string operationUnits = 5;
+    string description = 6;
+    google.protobuf.Timestamp started = 7;
+    google.protobuf.Timestamp updated = 8;
+}
+```
+
+A new PluginKind, `VolumeSnapshotterV2`, will be created, and the backup process will be modified to use this plugin kind.
+See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc.
+
+
+## Compatibility
+The included v1 adapter will allow any existing VolumeSnapshotter plugin to work as expected, with no-op Progress() and Cancel() methods.
+
+## Implementation
+This will be implemented during the Velero 1.11 development cycle.

go.mod

@@ -1,31 +1,30 @@
 module github.com/vmware-tanzu/velero
 
-go 1.18
+go 1.20
 
 require (
-	cloud.google.com/go/storage v1.10.0
+	cloud.google.com/go/storage v1.21.0
 	github.com/Azure/azure-pipeline-go v0.2.3
-	github.com/Azure/azure-sdk-for-go v61.4.0+incompatible
+	github.com/Azure/azure-sdk-for-go v67.2.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/Azure/go-autorest/autorest v0.11.21
+	github.com/Azure/go-autorest/autorest v0.11.27
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
 	github.com/Azure/go-autorest/autorest/to v0.3.0
-	github.com/apex/log v1.9.0
 	github.com/aws/aws-sdk-go v1.43.31
 	github.com/bombsimon/logrusr/v3 v3.0.0
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/fatih/color v1.13.0
 	github.com/gobwas/glob v0.2.3
-	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/joho/godotenv v1.3.0
+	github.com/kopia/kopia v0.10.7
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.18.1
+	github.com/onsi/gomega v1.20.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
 	github.com/robfig/cron v1.1.0
@@ -33,29 +32,37 @@ require (
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	github.com/vmware-tanzu/crash-diagnostics v0.3.7
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
 	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	google.golang.org/api v0.56.0
-	google.golang.org/grpc v1.40.0
-	k8s.io/api v0.24.2
+	google.golang.org/api v0.74.0
+	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.28.0
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.25.6
 	k8s.io/apiextensions-apiserver v0.24.2
-	k8s.io/apimachinery v0.24.2
+	k8s.io/apimachinery v0.25.6
 	k8s.io/cli-runtime v0.24.0
-	k8s.io/client-go v0.24.2
-	k8s.io/klog v1.0.0
+	k8s.io/client-go v0.25.6
+	k8s.io/klog/v2 v2.70.1
 	k8s.io/kube-aggregator v0.19.12
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	sigs.k8s.io/controller-runtime v0.12.2
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
-	cloud.google.com/go v0.93.3 // indirect
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/compute v1.5.0 // indirect
+	cloud.google.com/go/iam v0.1.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.14 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
@@ -63,69 +70,83 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
-	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-colorable v0.1.9 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.23 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/oklog/run v1.0.0 // indirect
+	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.34.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/rs/xid v1.3.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
 	github.com/vladimirvivien/gexe v0.1.1 // indirect
+	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.19.1 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
+	golang.org/x/exp v0.0.0-20210916165020-5cb4fee858ee // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/component-base v0.24.2 // indirect
-	k8s.io/klog/v2 v2.60.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 // indirect
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
-	sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )

go.sum

@@ -23,17 +23,29 @@ cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAV
 cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
 cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
 cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3 h1:wPBktZFzYBcCZVARvwVKqH1uEj+aLXofJEtrb4oOsio=
 cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
+cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
+cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
+cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
+cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U=
+cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y=
+cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
+cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
+cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM=
+cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/iam v0.1.1 h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68=
+cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -42,13 +54,20 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.21.0 h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14=
+cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
-github.com/Azure/azure-sdk-for-go v61.4.0+incompatible h1:BF2Pm3aQWIa6q9KmxyF1JYKYXtVw67vtvu2Wd54NGuY=
-github.com/Azure/azure-sdk-for-go v61.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v67.2.0+incompatible h1:Uu/Ww6ernvPTrpq31kITVTIm/I5jlJ1wjtEH/bmSB2k=
+github.com/Azure/azure-sdk-for-go v67.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk=
+github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo=
 github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM=
 github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -59,15 +78,16 @@ github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+B
 github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
 github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest v0.11.21 h1:w77zY/9RnUAWcIQyDC0Fc89mCvwftR8F+zsR/OH6enk=
-github.com/Azure/go-autorest/autorest v0.11.21/go.mod h1:Do/yuMSW/13ayUkcVREpsMHGG+MvV81uzSCFgYPj4tM=
+github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A=
+github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk=
-github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
+github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg=
+github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY=
@@ -79,8 +99,9 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
+github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
 github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8=
 github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA=
 github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4=
@@ -108,11 +129,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
-github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
-github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
-github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
-github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
-github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -120,10 +136,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.43.31 h1:yJZIr8nMV1hXjAvvOLUFqZRJcHV7udPQBfhJqawDzI0=
 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -146,6 +160,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
+github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -153,7 +169,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
@@ -182,9 +202,13 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -199,6 +223,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -212,8 +237,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
@@ -267,13 +292,17 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
-github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -331,6 +360,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -354,7 +384,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -368,8 +397,10 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
+github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE=
+github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
@@ -405,6 +436,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
@@ -424,7 +457,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -435,7 +467,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -452,14 +483,25 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
+github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kopia/kopia v0.10.7 h1:6s0ZIZW3Ge2ozzefddASy7CIUadp/5tF9yCDKQfAKKI=
+github.com/kopia/kopia v0.10.7/go.mod h1:0d9THPD+jwomPcXvPbCdmLyX6phQVP7AqcCcDEajfNA=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -467,6 +509,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -480,16 +523,14 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -499,8 +540,13 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4=
+github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -524,6 +570,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -531,6 +578,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
+github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -544,24 +593,22 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=
-github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
-github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
+github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
+github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
+github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -603,14 +650,16 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
 github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -620,10 +669,7 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
-github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -652,24 +698,19 @@ github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
-github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
-github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
-github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
-github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
-github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
-github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -691,6 +732,12 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -731,23 +778,24 @@ go.starlark.net v0.0.0-20201006213952-227f4aabceb5 h1:ApvY/1gw+Yiqb/FKeks3KnVPWp
 go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
-go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
+go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -759,6 +807,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -772,6 +821,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20210916165020-5cb4fee858ee h1:qlrAyYdKz4o7rWVUjiKqQJMa4PEpd55fqBU8jpsl4Iw=
+golang.org/x/exp v0.0.0-20210916165020-5cb4fee858ee/go.mod h1:a3o/VtDNHN+dCVLEpzjjUHOzR+Ln3DHX056ZPzoZGGA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -838,6 +889,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -847,16 +899,17 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -877,6 +930,7 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -967,10 +1021,17 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1069,6 +1130,7 @@ golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
 gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
@@ -1099,8 +1161,19 @@ google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtuk
 google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
 google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
 google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.56.0 h1:08F9XVYTLOGeSQb3xI9C0gXMuQanhdGed0cWFhDozbI=
+google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
+google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
+google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
+google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
+google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM=
+google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M=
+google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
+google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
+google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE=
+google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1166,8 +1239,28 @@ google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwy
 google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
+google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1194,8 +1287,11 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1226,6 +1322,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
+gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -1242,7 +1340,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -1262,16 +1359,18 @@ k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw=
 k8s.io/api v0.19.12/go.mod h1:EK+KvSq2urA6+CjVdZyAHEphXoLq2K2eW6lxOzTKSaY=
 k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8=
 k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I=
-k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI=
 k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg=
+k8s.io/api v0.25.6 h1:LwDY2H6kD/3R8TekJYYaJWOdekNdXDO44eVpX6sNtJA=
+k8s.io/api v0.25.6/go.mod h1:bVp01KUcl8VUHFBTJMOknWNo7XvR0cMbeTTuFg1zCUs=
 k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k=
 k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ=
 k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
 k8s.io/apimachinery v0.19.12/go.mod h1:9eb44nUQSsz9QZiilFRuMj3ZbTmoWolU8S2gnXoRMjo=
 k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
 k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
-k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM=
 k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
+k8s.io/apimachinery v0.25.6 h1:r6KIF2AHwLqFfZ0LcOA3I11SF62YZK83dxj1fn14NOQ=
+k8s.io/apimachinery v0.25.6/go.mod h1:1S2i1QHkmxc8+EZCIxe/fX5hpldVXk4gvnJInMEb8D4=
 k8s.io/apiserver v0.19.12/go.mod h1:ldZAZTNIKfMMv/UUEhk6UyTXC0/34iRdNFHo+MJOPc4=
 k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI=
 k8s.io/cli-runtime v0.22.2/go.mod h1:tkm2YeORFpbgQHEK/igqttvPTRIHFRz5kATlw53zlMI=
@@ -1281,8 +1380,9 @@ k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU=
 k8s.io/client-go v0.19.12/go.mod h1:BAGKQraZ6fDmXhT46pGXWZQQqN7P4E0BJux0+9O6Gt0=
 k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U=
 k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw=
-k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA=
 k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30=
+k8s.io/client-go v0.25.6 h1:CHxACHi0DijmlYyUR7ooZoXnD5P8jYLgBHcxp775x/U=
+k8s.io/client-go v0.25.6/go.mod h1:s9mMAGFYiH3Z66j7BESzu0GEradT9GQ2LjFf/YRrnyc=
 k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
 k8s.io/code-generator v0.19.12/go.mod h1:ADrDvaUQWGn4a8lX0ONtzb7uFmDRQOMSYIMk1qWIAx8=
 k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w=
@@ -1293,25 +1393,25 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
 k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
+k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8=
 k8s.io/kube-aggregator v0.19.12/go.mod h1:K76wPd03pSHEmS1FgJOcpryac5C3va4cbCvSu+4EmE0=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
-k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 h1:IyQ1DifCBk589JD4Cm2CT2poIdO3lfPzz3WwVh1Ugf8=
-k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8/go.mod h1:guXtiQW/y/AWAfPSOaI/1eY0TGBAmL5OygiIyUOKDRc=
+k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
+k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
 k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
+k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
@@ -1320,8 +1420,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lR
 sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE=
 sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
-sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 h1:2sgAQQcY0dEW2SsQwTXhQV4vO6+rSslYx8K3XmM5hqQ=
-sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
 sigs.k8s.io/kustomize/api v0.11.4/go.mod h1:k+8RsqYbgpkIrJ4p9jcdPqe8DprLxFUUO0yNOq8C+xI=
 sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM=
@@ -1330,8 +1430,9 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=

golangci.yaml

@@ -7,18 +7,11 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  timeout: 5m
+  timeout: 20m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
 
-  # include test files or not, default is true
-  tests: true
-
-  # list of build tags, all linters use it. Default is empty list.
-  #build-tags:
-  #  - mytag
-
   # which dirs to skip: issues from them won't be reported;
   # can use regexp here: generated.*, regexp is applied on full path;
   # default value is empty list, but default dirs are skipped independently
@@ -294,77 +287,23 @@ linters-settings:
     # Allow leading comments to be separated with empty liens
     allow-separated-leading-comment: false
 
-  # The custom section can be used to define linter plugins to be loaded at runtime. See README doc
-  #  for more info.
-  # custom:
-    # Each custom linter should have a unique name.
-    #  example:
-      # The path to the plugin *.so. Can be absolute or local. Required for each custom linter
-      # path: /path/to/example.so
-      # The description of the linter. Optional, just for documentation purposes.
-      # description: This is an example usage of a plugin linter.
-      # Intended to point to the repo location of the linter. Optional, just for documentation purposes.
-      # original-url: github.com/golangci/example-linter
-
 linters:
-#  enable:
-#    - megacheck
-#    - govet
-# disable:
-#    - maligned
-#    - prealloc
   disable-all: true
-  presets:
-#    - bugs
-#    - unused
+  enable:
+    - bodyclose
+    - goconst
+    - gofmt
+    - goheader
+    - goimports
+    - gosec
+    - misspell
+    - typecheck
+    - unparam
+    - unused
   fast: false
 
 
 issues:
-#  # List of regexps of issue texts to exclude, empty list by default.
-#  # But independently from this option we use default exclude patterns,
-#  # it can be disabled by `exclude-use-default: false`. To list all
-#  # excluded by default patterns execute `golangci-lint run --help`
-#  exclude:
-#    - abcdef
-#
-#  # Excluding configuration per-path, per-linter, per-text and per-source
-#  exclude-rules:
-#    # Exclude some linters from running on tests files.
-#    - path: _test\.go
-#      linters:
-#        - gocyclo
-#        - errcheck
-#        - dupl
-#        - gosec
-#
-#    # Exclude known linters from partially hard-vendored code,
-#    # which is impossible to exclude via "nolint" comments.
-#    - path: internal/hmac/
-#      text: "weak cryptographic primitive"
-#      linters:
-#        - gosec
-#
-#    # Exclude some staticcheck messages
-#    - linters:
-#        - staticcheck
-#      text: "SA9003:"
-#
-#    # Exclude lll issues for long lines with go:generate
-#    - linters:
-#        - lll
-#      source: "^//go:generate "
-
-  # Independently from option `exclude` we use default exclude patterns,
-  # it can be disabled by this option. To list all
-  # excluded by default patterns execute `golangci-lint run --help`.
-  # Default value for this option is true.
-  exclude-use-default: true
-
-  # The default value is false. If set to true exclude and exclude-rules
-  # regular expressions become case sensitive.
-  exclude-case-sensitive: false
-
   # The list of ids of default excludes to include or disable. By default it's empty.
   include:
     - EXC0002 # disable excluding of issues about comments from golint
@@ -375,19 +314,8 @@ issues:
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
-  # Show only new issues: if there are unstaged changes or untracked files,
-  # only those changes are analyzed, else only changes in HEAD~ are analyzed.
-  # It's a super-useful option for integration of golangci-lint into existing
-  # large codebase. It's not practical to fix all existing issues at the moment
-  # of integration: much better don't allow issues in new code.
-  # Default is false.
-  new: false
-
   # Show only new issues created after git revision `REV`
-  new-from-rev: REV
-
-  # Show only new issues created in git patch with set file path.
-  new-from-patch: path/to/patch/file
+  # new-from-rev: origin/main
 
 severity:
   # Default value is empty string.

hack/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.19.8
+FROM --platform=linux/amd64 golang:1.20.6-bullseye
 
 ARG GOPROXY
 
@@ -39,14 +39,18 @@ RUN go install golang.org/x/tools/cmd/goimports@11e9d9cc0042e6bd10337d4d2c3e5d92
 # get protoc compiler and golang plugin
 WORKDIR /root
 RUN apt-get update && apt-get install -y unzip
-RUN wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip && \
-    unzip protoc-3.9.1-linux-x86_64.zip && \
+RUN wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protoc-3.14.0-linux-x86_64.zip && \
+    unzip protoc-3.14.0-linux-x86_64.zip && \
     mv bin/protoc /usr/bin/protoc && \
+    mv include/google /usr/include && \
+    chmod a+x  /usr/include/google && \
+    chmod a+x  /usr/include/google/protobuf && \
+    chmod a+r -R /usr/include/google && \
     chmod +x /usr/bin/protoc
-RUN go install github.com/golang/protobuf/protoc-gen-go@v1.0.0
+RUN go install github.com/golang/protobuf/protoc-gen-go@v1.4.3
 
 # get goreleaser
-RUN wget --quiet https://github.com/goreleaser/goreleaser/releases/download/v0.120.8/goreleaser_Linux_x86_64.tar.gz && \
+RUN wget --quiet https://github.com/goreleaser/goreleaser/releases/download/v1.15.2/goreleaser_Linux_x86_64.tar.gz && \
     tar xvf goreleaser_Linux_x86_64.tar.gz && \
     mv goreleaser /usr/bin/goreleaser && \
     chmod +x /usr/bin/goreleaser

hack/build-restic.sh

@@ -50,7 +50,6 @@ fi
 mkdir ${build_path}/restic
 git clone -b v${RESTIC_VERSION} https://github.com/restic/restic.git ${build_path}/restic
 pushd ${build_path}/restic
-git apply /go/src/github.com/vmware-tanzu/velero/hack/modify_acces_denied_code.txt
 git apply /go/src/github.com/vmware-tanzu/velero/hack/fix_restic_cve.txt
 go run build.go --goos "${GOOS}" --goarch "${GOARCH}" --goarm "${GOARM}" -o ${restic_bin}
 chmod +x ${restic_bin}

hack/ci/build_util.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+set -e
+
+function uploader {
+    gsutil cp $1 gs://$2/$1
+    gsutil -D setacl public-read gs://$2/$1 &> /dev/null
+}

hack/crd-gen/v1/main.go

@@ -24,7 +24,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"text/template"
@@ -41,7 +40,7 @@ package crds
 import (
 	"bytes"
 	"compress/gzip"
-	"io/ioutil"
+	"io"
 
 	apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -65,7 +64,7 @@ func crds() []*apiextv1.CustomResourceDefinition {
 		if err != nil {
 			panic(err)
 		}
-		bytes, err := ioutil.ReadAll(gzr)
+		bytes, err := io.ReadAll(gzr)
 		if err != nil {
 			panic(err)
 		}
@@ -87,7 +86,7 @@ type templateData struct {
 }
 
 func main() {
-	headerBytes, err := ioutil.ReadFile(goHeaderFile)
+	headerBytes, err := os.ReadFile(goHeaderFile)
 	if err != nil {
 		log.Fatalln(err)
 	}
@@ -97,7 +96,7 @@ func main() {
 	}
 
 	// This is relative to config/crd/crds
-	manifests, err := ioutil.ReadDir("../bases")
+	manifests, err := os.ReadDir("../bases")
 	if err != nil {
 		log.Fatalln(err)
 	}

hack/fix_restic_cve.txt

@@ -1,97 +1,60 @@
 diff --git a/go.mod b/go.mod
-index d819a6be7..4ec9d9bf1 100644
+index 5f939c481..6f281b45d 100644
 --- a/go.mod
 +++ b/go.mod
-@@ -35,12 +35,12 @@ require (
- 	github.com/spf13/cobra v1.5.0
+@@ -25,12 +25,12 @@ require (
+ 	github.com/spf13/cobra v1.6.1
  	github.com/spf13/pflag v1.0.5
- 	golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
--	golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c
+ 	golang.org/x/crypto v0.5.0
+-	golang.org/x/net v0.5.0
 +	golang.org/x/net v0.7.0
- 	golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2
- 	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
--	golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
--	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035
--	golang.org/x/text v0.3.7
+ 	golang.org/x/oauth2 v0.4.0
+ 	golang.org/x/sync v0.1.0
+-	golang.org/x/sys v0.4.0
+-	golang.org/x/term v0.4.0
+-	golang.org/x/text v0.6.0
 +	golang.org/x/sys v0.5.0
 +	golang.org/x/term v0.5.0
 +	golang.org/x/text v0.7.0
- 	google.golang.org/api v0.93.0
- 	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
- 	gopkg.in/ini.v1 v1.67.0 // indirect
+ 	google.golang.org/api v0.106.0
+ )
+ 
 diff --git a/go.sum b/go.sum
-index 959651048..da200f0c5 100644
+index 026e1d2fa..da35b7a6c 100644
 --- a/go.sum
 +++ b/go.sum
-@@ -319,6 +319,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
- github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
- github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
- github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
- go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
- go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
- go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-@@ -373,6 +374,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
- golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
- golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
- golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
- golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
- golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
- golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-@@ -418,8 +420,8 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug
- golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
- golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
- golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
--golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes=
--golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+@@ -189,8 +189,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
+ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+-golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
  golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
- golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
- golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-@@ -454,6 +456,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
- golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
- golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
- golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
- golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
- golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
- golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-@@ -522,13 +525,12 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
+ golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
+ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
+@@ -214,17 +214,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
+ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
  golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
  golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
- golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
--golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
--golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U=
--golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+-golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
  golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
- golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
--golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
--golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+-golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+-golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
 +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
- golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
- golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-@@ -537,8 +539,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
- golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
- golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
  golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
--golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
- golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+-golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
- golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
- golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
- golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-@@ -593,6 +596,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
- golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
- golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
- golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
- golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
- golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
- golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

hack/lint.sh

@@ -14,23 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-LINTERS=${1:-"gosec,goconst,gofmt,goimports,unparam"}
-ALL=${2:-false}
-
 HACK_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 # Printing out cache status
 golangci-lint cache status
 
-if [[ $ALL == true ]] ; then
-  action=""
-else
-  action="-n"
-fi
-
 # Enable GL_DEBUG line below for debug messages for golangci-lint
 # export GL_DEBUG=loader,gocritic,env
-CMD="golangci-lint run -E ${LINTERS} $action -c  $HACK_DIR/../golangci.yaml"
+CMD="golangci-lint run -c $HACK_DIR/../golangci.yaml"
 echo "Running $CMD"
 
 eval $CMD

hack/modify_acces_denied_code.txt

@@ -1,13 +0,0 @@
-diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go
-index 0b3816c06..eec10f9c7 100644
---- a/internal/backend/s3/s3.go
-+++ b/internal/backend/s3/s3.go
-@@ -164,7 +164,7 @@ func isAccessDenied(err error) bool {
- 	debug.Log("isAccessDenied(%T, %#v)", err, err)
- 
- 	var e minio.ErrorResponse
--	return errors.As(err, &e) && e.Code == "Access Denied"
-+	return errors.As(err, &e) && e.Code == "AccessDenied"
- }
- 
- // IsNotExist returns true if the error is caused by a not existing file.

hack/release-tools/chk_version.go

@@ -29,7 +29,7 @@ import (
 //	minor
 //	patch
 //	prerelease (this will be alpha/beta/rc followed by a ".", followed by 1 or more digits (alpha.5)
-var release_regex *regexp.Regexp = regexp.MustCompile("^v(?P<major>[[:digit:]]+)\\.(?P<minor>[[:digit:]]+)\\.(?P<patch>[[:digit:]]+)(-{1}(?P<prerelease>(alpha|beta|rc)\\.[[:digit:]]+))*")
+var release_regex *regexp.Regexp = regexp.MustCompile(`^v(?P<major>[[:digit:]]+)\.(?P<minor>[[:digit:]]+)\.(?P<patch>[[:digit:]]+)(-{1}(?P<prerelease>(alpha|beta|rc)\.[[:digit:]]+))*`)
 
 // This small program exists because checking the VELERO_VERSION rules in bash is difficult, and difficult to test for correctness.
 // Calling it with --verify will verify whether or not the VELERO_VERSION environment variable is a valid version string, without parsing for its components.

hack/release-tools/goreleaser.sh

@@ -46,12 +46,12 @@ fi
 if [[ "${PUBLISH:-}" != "TRUE" ]]; then
     echo "Not set to publish"
     goreleaser release \
-        --rm-dist \
+        --clean \
         --release-notes="${RELEASE_NOTES_FILE}" \
         --skip-publish
 else
     echo "Getting ready to publish"
     goreleaser release \
-        --rm-dist \
+        --clean \
         --release-notes="${RELEASE_NOTES_FILE}"
 fi

hack/restore-crd-patch-v1.json

@@ -1,3 +0,0 @@
-[
-    { "op": "replace", "path": "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/hooks/properties/resources/items/properties/postHooks/items/properties/init/properties/initContainers/items/properties/ports/items/required", "value": [ "containerPort", "protocol"] }
-]

hack/restore-crd-patch-v1beta1.json

@@ -1,3 +0,0 @@
-[
-    { "op": "replace", "path": "/spec/validation/openAPIV3Schema/properties/spec/properties/hooks/properties/resources/items/properties/postHooks/items/properties/init/properties/initContainers/items/properties/ports/items/required", "value": [ "containerPort", "protocol"] }
-]

hack/test.sh

@@ -25,7 +25,6 @@ TARGETS=(
   ./cmd/...
   ./pkg/...
   ./internal/...
-  ./test/...
 )
 
 if [[ ${#@} -ne 0 ]]; then

hack/update-2proto.sh

@@ -18,6 +18,7 @@ HACK_DIR=$(dirname "${BASH_SOURCE}")
 
 echo "Updating plugin proto"
 
-protoc pkg/plugin/proto/*.proto --go_out=plugins=grpc:pkg/plugin/generated/ -I pkg/plugin/proto/
+echo protoc --version
+protoc pkg/plugin/proto/*.proto pkg/plugin/proto/*/*/*.proto --go_out=plugins=grpc:pkg/plugin/generated/ --go_opt=module=github.com/vmware-tanzu/velero/pkg/plugin/generated -I pkg/plugin/proto/ -I /usr/include
 
 echo "Updating plugin proto - done!"

hack/update-3generated-crd-code.sh

@@ -48,7 +48,7 @@ ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \
 # Generate apiextensions.k8s.io/v1
 # Generate manifests e.g. CRD, RBAC etc.
 controller-gen \
-  crd:crdVersions=v1\
+  crd:crdVersions=v1 \
   paths=./pkg/apis/velero/v1/... \
   rbac:roleName=velero-perms \
   paths=./pkg/controller/... \
@@ -56,13 +56,4 @@ controller-gen \
   object \
   paths=./pkg/apis/velero/v1/...
 
-# this is a super hacky workaround for https://github.com/kubernetes/kubernetes/issues/91395
-# which a result of fixing the validation on CRD objects. The validation ensures the fields that are list map keys, are either marked
-# as required or have default values to ensure merging of list map items work as expected.
-# With "containerPort" and "protocol" being considered as x-kubernetes-list-map-keys in the container ports, and "protocol" was not
-# a required field, the CRD would fail validation with errors similar to the one reported in https://github.com/kubernetes/kubernetes/issues/91395.
-# once controller-gen (above) is able to generate CRDs with `protocol` as a required field, this hack can be removed.
-kubectl patch -f config/crd/v1/bases/velero.io_restores.yaml -p "$(cat hack/restore-crd-patch-v1.json)" --type=json --local=true -o yaml > /tmp/velero.io_restores-yaml.patched
-mv /tmp/velero.io_restores-yaml.patched config/crd/v1/bases/velero.io_restores.yaml
-
 go generate ./config/crd/v1/crds

hack/verify-fmt.sh

@@ -15,4 +15,4 @@
 # limitations under the License.
 
 HACK_DIR=$(dirname "${BASH_SOURCE[0]}")
-"${HACK_DIR}"/update-fmt.sh --verify
+"${HACK_DIR}"/update-1fmt.sh --verify

hack/verify-generated-crd-code.sh

@@ -16,7 +16,7 @@
 
 HACK_DIR=$(dirname "${BASH_SOURCE}")
 
-${HACK_DIR}/update-generated-crd-code.sh
+${HACK_DIR}/update-3generated-crd-code.sh
 
 # ensure no changes to generated CRDs
 if ! git diff --exit-code config/crd/v1/crds/crds.go >/dev/null; then

hack/verify-generated-issue-template.sh

@@ -27,7 +27,7 @@ cleanup() {
 }
 
 echo "Verifying generated Github issue template"
-${HACK_DIR}/update-generated-issue-template.sh ${OUT_TMP_FILE} > /dev/null
+${HACK_DIR}/update-4generated-issue-template.sh ${OUT_TMP_FILE} > /dev/null
 output=$(echo "`diff ${ISSUE_TEMPLATE_FILE} ${OUT_TMP_FILE}`")
 
 if [[ -n "${output}" ]] ; then

internal/credentials/getter.go

@@ -0,0 +1,24 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package credentials
+
+// CredentialGetter is a collection of interfaces for interacting with credentials
+// that are stored in different targets
+type CredentialGetter struct {
+	FromFile   FileStore
+	FromSecret SecretStore
+}

internal/credentials/mocks/FileStore.go

@@ -0,0 +1,49 @@
+// Code generated by mockery v2.14.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	v1 "k8s.io/api/core/v1"
+)
+
+// FileStore is an autogenerated mock type for the FileStore type
+type FileStore struct {
+	mock.Mock
+}
+
+// Path provides a mock function with given fields: selector
+func (_m *FileStore) Path(selector *v1.SecretKeySelector) (string, error) {
+	ret := _m.Called(selector)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(*v1.SecretKeySelector) string); ok {
+		r0 = rf(selector)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*v1.SecretKeySelector) error); ok {
+		r1 = rf(selector)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+type mockConstructorTestingTNewFileStore interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewFileStore creates a new instance of FileStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewFileStore(t mockConstructorTestingTNewFileStore) *FileStore {
+	mock := &FileStore{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

internal/credentials/mocks/SecretStore.go

@@ -0,0 +1,49 @@
+// Code generated by mockery v2.14.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	v1 "k8s.io/api/core/v1"
+)
+
+// SecretStore is an autogenerated mock type for the SecretStore type
+type SecretStore struct {
+	mock.Mock
+}
+
+// Get provides a mock function with given fields: selector
+func (_m *SecretStore) Get(selector *v1.SecretKeySelector) (string, error) {
+	ret := _m.Called(selector)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(*v1.SecretKeySelector) string); ok {
+		r0 = rf(selector)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*v1.SecretKeySelector) error); ok {
+		r1 = rf(selector)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+type mockConstructorTestingTNewSecretStore interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewSecretStore creates a new instance of SecretStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewSecretStore(t mockConstructorTestingTNewSecretStore) *SecretStore {
+	mock := &SecretStore{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

internal/credentials/secret_store.go

@@ -0,0 +1,56 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package credentials
+
+import (
+	"github.com/pkg/errors"
+	corev1api "k8s.io/api/core/v1"
+	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+)
+
+// SecretStore defines operations for interacting with credentials
+// that are stored in Secret.
+type SecretStore interface {
+	// Get returns the secret key defined by the given selector
+	Get(selector *corev1api.SecretKeySelector) (string, error)
+}
+
+type namespacedSecretStore struct {
+	client    kbclient.Client
+	namespace string
+}
+
+// NewNamespacedSecretStore returns a SecretStore which can interact with credentials
+// for the given namespace.
+func NewNamespacedSecretStore(client kbclient.Client, namespace string) (SecretStore, error) {
+	return &namespacedSecretStore{
+		client:    client,
+		namespace: namespace,
+	}, nil
+}
+
+// Buffer returns the secret key defined by the given selector.
+func (n *namespacedSecretStore) Get(selector *corev1api.SecretKeySelector) (string, error) {
+	creds, err := kube.GetSecretKey(n.client, n.namespace, selector)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to get key for secret")
+	}
+
+	return string(creds), nil
+}

internal/delete/delete_item_action_handler.go

@@ -48,7 +48,7 @@ type Context struct {
 func InvokeDeleteActions(ctx *Context) error {
 	var err error
 	resolver := framework.NewDeleteItemActionResolver(ctx.Actions)
-	ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper)
+	ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper, ctx.Log)
 	// No actions installed and no error means we don't have to continue;
 	// just do the backup deletion without worrying about plugins.
 	if len(ctx.resolvedActions) == 0 && err == nil {

internal/hook/item_hook_handler.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"time"
 
-	uuid "github.com/gofrs/uuid"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
@@ -36,7 +36,7 @@ import (
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/podexec"
-	"github.com/vmware-tanzu/velero/pkg/restic"
+	"github.com/vmware-tanzu/velero/pkg/restorehelper"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
@@ -121,20 +121,20 @@ func (i *InitContainerRestoreHookHandler) HandleRestoreHooks(
 	}
 
 	initContainers := []corev1api.Container{}
-	// If this pod had pod volumes backed up using restic, then we want to the pod volumes restored prior to
+	// If this pod is backed up with data movement, then we want to the pod volumes restored prior to
 	// running the restore hook init containers. This allows the restore hook init containers to prepare the
 	// restored data to be consumed by the application container(s).
-	// So if there is a "restic-wait" init container already on the pod at index 0, we'll preserve that and run
+	// So if there is a "restore-wait" init container already on the pod at index 0, we'll preserve that and run
 	// it before running any other init container.
-	if len(pod.Spec.InitContainers) > 0 && pod.Spec.InitContainers[0].Name == restic.InitContainer {
+	if len(pod.Spec.InitContainers) > 0 && pod.Spec.InitContainers[0].Name == restorehelper.WaitInitContainer {
 		initContainers = append(initContainers, pod.Spec.InitContainers[0])
 		pod.Spec.InitContainers = pod.Spec.InitContainers[1:]
 	}
 
-	hooksFromAnnotations := getInitRestoreHookFromAnnotation(kube.NamespaceAndName(pod), metadata.GetAnnotations(), log)
-	if hooksFromAnnotations != nil {
+	initContainerFromAnnotations := getInitContainerFromAnnotation(kube.NamespaceAndName(pod), metadata.GetAnnotations(), log)
+	if initContainerFromAnnotations != nil {
 		log.Infof("Handling InitRestoreHooks from pod annotations")
-		initContainers = append(initContainers, hooksFromAnnotations.InitContainers...)
+		initContainers = append(initContainers, *initContainerFromAnnotations)
 	} else {
 		log.Infof("Handling InitRestoreHooks from RestoreSpec")
 		// pod did not have the annotations appropriate for restore hooks
@@ -155,7 +155,22 @@ func (i *InitContainerRestoreHookHandler) HandleRestoreHooks(
 			}
 			for _, hook := range rh.RestoreHooks {
 				if hook.Init != nil {
-					initContainers = append(initContainers, hook.Init.InitContainers...)
+					containers := make([]corev1api.Container, 0)
+					for _, raw := range hook.Init.InitContainers {
+						container := corev1api.Container{}
+						err := ValidateContainer(raw.Raw)
+						if err != nil {
+							log.Errorf("invalid Restore Init hook: %s", err.Error())
+							return nil, err
+						}
+						err = json.Unmarshal(raw.Raw, &container)
+						if err != nil {
+							log.Errorf("fail to Unmarshal hook Init into container: %s", err.Error())
+							return nil, errors.WithStack(err)
+						}
+						containers = append(containers, container)
+					}
+					initContainers = append(initContainers, containers...)
 				}
 			}
 		}
@@ -258,11 +273,25 @@ func (h *DefaultItemHookHandler) HandleHooks(
 	return nil
 }
 
+// NoOpItemHookHandler is the an itemHookHandler for the Finalize controller where hooks don't run
+type NoOpItemHookHandler struct{}
+
+func (h *NoOpItemHookHandler) HandleHooks(
+	log logrus.FieldLogger,
+	groupResource schema.GroupResource,
+	obj runtime.Unstructured,
+	resourceHooks []ResourceHook,
+	phase hookPhase,
+) error {
+
+	return nil
+}
+
 func phasedKey(phase hookPhase, key string) string {
 	if phase != "" {
 		return fmt.Sprintf("%v.%v", phase, key)
 	}
-	return string(key)
+	return key
 }
 
 func getHookAnnotation(annotations map[string]string, key string, phase hookPhase) string {
@@ -350,7 +379,7 @@ type ResourceRestoreHook struct {
 	RestoreHooks []velerov1api.RestoreResourceHook
 }
 
-func getInitRestoreHookFromAnnotation(podName string, annotations map[string]string, log logrus.FieldLogger) *velerov1api.InitRestoreHook {
+func getInitContainerFromAnnotation(podName string, annotations map[string]string, log logrus.FieldLogger) *corev1api.Container {
 	containerImage := annotations[podRestoreHookInitContainerImageAnnotationKey]
 	containerName := annotations[podRestoreHookInitContainerNameAnnotationKey]
 	command := annotations[podRestoreHookInitContainerCommandAnnotationKey]
@@ -362,7 +391,7 @@ func getInitRestoreHookFromAnnotation(podName string, annotations map[string]str
 		log.Infof("RestoreHook init container for pod %s is using container's default entrypoint", podName, containerImage)
 	}
 	if containerName == "" {
-		uid, err := uuid.NewV4()
+		uid, err := uuid.NewRandom()
 		uuidStr := "deadfeed"
 		if err != nil {
 			log.Errorf("Failed to generate UUID for container name")
@@ -373,15 +402,13 @@ func getInitRestoreHookFromAnnotation(podName string, annotations map[string]str
 		log.Infof("Pod %s has no %s annotation, using generated name %s for initContainer", podName, podRestoreHookInitContainerNameAnnotationKey, containerName)
 	}
 
-	return &velerov1api.InitRestoreHook{
-		InitContainers: []corev1api.Container{
-			{
-				Image:   containerImage,
-				Name:    containerName,
-				Command: parseStringToCommand(command),
-			},
-		},
+	initContainer := corev1api.Container{
+		Image:   containerImage,
+		Name:    containerName,
+		Command: parseStringToCommand(command),
 	}
+
+	return &initContainer
 }
 
 // GetRestoreHooksFromSpec returns a list of ResourceRestoreHooks from the restore Spec.
@@ -406,7 +433,7 @@ func GetRestoreHooksFromSpec(hooksSpec *velerov1api.RestoreHooks) ([]ResourceRes
 		if rs.LabelSelector != nil {
 			ls, err := metav1.LabelSelectorAsSelector(rs.LabelSelector)
 			if err != nil {
-				return nil, errors.WithStack(err)
+				return []ResourceRestoreHook{}, errors.WithStack(err)
 			}
 			rh.Selector.LabelSelector = ls
 		}
@@ -526,3 +553,17 @@ func GroupRestoreExecHooks(
 
 	return byContainer, nil
 }
+
+// ValidateContainer validate whether a map contains mandatory k8s Container fields.
+// mandatory fields include name, image and commands.
+func ValidateContainer(raw []byte) error {
+	container := corev1api.Container{}
+	err := json.Unmarshal(raw, &container)
+	if err != nil {
+		return err
+	}
+	if len(container.Command) <= 0 || len(container.Name) <= 0 || len(container.Image) <= 0 {
+		return fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field")
+	}
+	return nil
+}

internal/hook/item_hook_handler_test.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -40,15 +39,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 )
 
-type mockItemHookHandler struct {
-	mock.Mock
-}
-
-func (h *mockItemHookHandler) HandleHooks(log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, resourceHooks []ResourceHook, phase hookPhase) error {
-	args := h.Called(log, groupResource, obj, resourceHooks, phase)
-	return args.Error(0)
-}
-
 func TestHandleHooksSkips(t *testing.T) {
 	tests := []struct {
 		name          string
@@ -1191,11 +1181,11 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 	}
 }
 
-func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
+func TestGetInitContainerFromAnnotations(t *testing.T) {
 	testCases := []struct {
 		name             string
 		inputAnnotations map[string]string
-		expected         velerov1api.InitRestoreHook
+		expected         *corev1api.Container
 		expectNil        bool
 	}{
 		{
@@ -1223,12 +1213,8 @@ func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 				podRestoreHookInitContainerNameAnnotationKey:    "",
 				podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator /user-data full",
 			},
-			expected: velerov1api.InitRestoreHook{
-				InitContainers: []corev1api.Container{
-					*builder.ForContainer("restore-init1", "busy-box").
-						Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
-				},
-			},
+			expected: builder.ForContainer("restore-init1", "busy-box").
+				Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
 		},
 		{
 			name:      "should generate container name when container name is missing",
@@ -1237,22 +1223,14 @@ func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 				podRestoreHookInitContainerImageAnnotationKey:   "busy-box",
 				podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator /user-data full",
 			},
-			expected: velerov1api.InitRestoreHook{
-				InitContainers: []corev1api.Container{
-					*builder.ForContainer("restore-init1", "busy-box").
-						Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
-				},
-			},
+			expected: builder.ForContainer("restore-init1", "busy-box").
+				Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
 		},
 		{
 			name:      "should return expected init container when all annotations are specified",
 			expectNil: false,
-			expected: velerov1api.InitRestoreHook{
-				InitContainers: []corev1api.Container{
-					*builder.ForContainer("restore-init1", "busy-box").
-						Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
-				},
-			},
+			expected: builder.ForContainer("restore-init1", "busy-box").
+				Command([]string{"/usr/bin/data-populator /user-data full"}).Result(),
 			inputAnnotations: map[string]string{
 				podRestoreHookInitContainerImageAnnotationKey:   "busy-box",
 				podRestoreHookInitContainerNameAnnotationKey:    "restore-init",
@@ -1262,12 +1240,8 @@ func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 		{
 			name:      "should return expected init container when all annotations are specified with command as a JSON array",
 			expectNil: false,
-			expected: velerov1api.InitRestoreHook{
-				InitContainers: []corev1api.Container{
-					*builder.ForContainer("restore-init1", "busy-box").
-						Command([]string{"a", "b", "c"}).Result(),
-				},
-			},
+			expected: builder.ForContainer("restore-init1", "busy-box").
+				Command([]string{"a", "b", "c"}).Result(),
 			inputAnnotations: map[string]string{
 				podRestoreHookInitContainerImageAnnotationKey:   "busy-box",
 				podRestoreHookInitContainerNameAnnotationKey:    "restore-init",
@@ -1277,12 +1251,8 @@ func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 		{
 			name:      "should return expected init container when all annotations are specified with command as malformed a JSON array",
 			expectNil: false,
-			expected: velerov1api.InitRestoreHook{
-				InitContainers: []corev1api.Container{
-					*builder.ForContainer("restore-init1", "busy-box").
-						Command([]string{"[foobarbaz"}).Result(),
-				},
-			},
+			expected: builder.ForContainer("restore-init1", "busy-box").
+				Command([]string{"[foobarbaz"}).Result(),
 			inputAnnotations: map[string]string{
 				podRestoreHookInitContainerImageAnnotationKey:   "busy-box",
 				podRestoreHookInitContainerNameAnnotationKey:    "restore-init",
@@ -1293,15 +1263,14 @@ func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual := getInitRestoreHookFromAnnotation("test/pod1", tc.inputAnnotations, velerotest.NewLogger())
+			actualInitContainer := getInitContainerFromAnnotation("test/pod1", tc.inputAnnotations, velerotest.NewLogger())
 			if tc.expectNil {
-				assert.Nil(t, actual)
+				assert.Nil(t, actualInitContainer)
 				return
 			}
-			assert.NotEmpty(t, actual.InitContainers[0].Name)
-			assert.Equal(t, len(tc.expected.InitContainers), len(actual.InitContainers))
-			assert.Equal(t, tc.expected.InitContainers[0].Image, actual.InitContainers[0].Image)
-			assert.Equal(t, tc.expected.InitContainers[0].Command, actual.InitContainers[0].Command)
+			assert.NotEmpty(t, actualInitContainer.Name)
+			assert.Equal(t, tc.expected.Image, actualInitContainer.Image)
+			assert.Equal(t, tc.expected.Command, actualInitContainer.Command)
 		})
 	}
 }
@@ -1347,11 +1316,11 @@ func TestGetRestoreHooksFromSpec(t *testing.T) {
 						PostHooks: []velerov1api.RestoreResourceHook{
 							{
 								Init: &velerov1api.InitRestoreHook{
-									InitContainers: []corev1api.Container{
-										*builder.ForContainer("restore-init1", "busy-box").
-											Command([]string{"foobarbaz"}).Result(),
-										*builder.ForContainer("restore-init2", "busy-box").
-											Command([]string{"foobarbaz"}).Result(),
+									InitContainers: []runtime.RawExtension{
+										builder.ForContainer("restore-init1", "busy-box").
+											Command([]string{"foobarbaz"}).ResultRawExtension(),
+										builder.ForContainer("restore-init2", "busy-box").
+											Command([]string{"foobarbaz"}).ResultRawExtension(),
 									},
 								},
 							},
@@ -1363,17 +1332,17 @@ func TestGetRestoreHooksFromSpec(t *testing.T) {
 				{
 					Name: "h1",
 					Selector: ResourceHookSelector{
-						Namespaces: collections.NewIncludesExcludes().Includes([]string{"ns1", "ns2", "ns3"}...).Excludes([]string{"ns4", "ns5", "ns6"}...),
-						Resources:  collections.NewIncludesExcludes().Includes([]string{kuberesource.Pods.Resource}...),
+						Namespaces: collections.NewIncludesExcludes().Includes("ns1", "ns2", "ns3").Excludes("ns4", "ns5", "ns6"),
+						Resources:  collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource),
 					},
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init1", "busy-box").
-										Command([]string{"foobarbaz"}).Result(),
-									*builder.ForContainer("restore-init2", "busy-box").
-										Command([]string{"foobarbaz"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init1", "busy-box").
+										Command([]string{"foobarbaz"}).ResultRawExtension(),
+									builder.ForContainer("restore-init2", "busy-box").
+										Command([]string{"foobarbaz"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1539,9 +1508,9 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("should-not exist", "does-not-matter").
-										Command([]string{""}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("should-not exist", "does-not-matter").
+										Command([]string{""}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1556,6 +1525,9 @@ func TestHandleRestoreHooks(t *testing.T) {
 					Name:      "app1",
 					Namespace: "default",
 				},
+				Spec: corev1api.PodSpec{
+					InitContainers: []corev1api.Container{},
+				},
 			},
 			expectedError: nil,
 			expectedPod: &corev1api.Pod{
@@ -1582,11 +1554,11 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
-									*builder.ForContainer("restore-init-container-2", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
+									builder.ForContainer("restore-init-container-2", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1643,11 +1615,11 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
-									*builder.ForContainer("restore-init-container-2", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
+									builder.ForContainer("restore-init-container-2", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1680,11 +1652,11 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
-									*builder.ForContainer("restore-init-container-2", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
+									builder.ForContainer("restore-init-container-2", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1693,7 +1665,7 @@ func TestHandleRestoreHooks(t *testing.T) {
 			},
 		},
 		{
-			name: "should preserve restic-wait init container when it is the only existing init container",
+			name: "should preserve restore-wait init container when it is the only existing init container",
 			podInput: corev1api.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "app1",
@@ -1701,8 +1673,8 @@ func TestHandleRestoreHooks(t *testing.T) {
 				},
 				Spec: corev1api.PodSpec{
 					InitContainers: []corev1api.Container{
-						*builder.ForContainer("restic-wait", "bus-box").
-							Command([]string{"restic-restore"}).Result(),
+						*builder.ForContainer("restore-wait", "bus-box").
+							Command([]string{"pod-volume-restore"}).Result(),
 					},
 				},
 			},
@@ -1714,8 +1686,8 @@ func TestHandleRestoreHooks(t *testing.T) {
 				},
 				Spec: corev1api.PodSpec{
 					InitContainers: []corev1api.Container{
-						*builder.ForContainer("restic-wait", "bus-box").
-							Command([]string{"restic-restore"}).Result(),
+						*builder.ForContainer("restore-wait", "bus-box").
+							Command([]string{"pod-volume-restore"}).Result(),
 						*builder.ForContainer("restore-init-container-1", "nginx").
 							Command([]string{"a", "b", "c"}).Result(),
 						*builder.ForContainer("restore-init-container-2", "nginx").
@@ -1733,11 +1705,11 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
-									*builder.ForContainer("restore-init-container-2", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
+									builder.ForContainer("restore-init-container-2", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1747,7 +1719,7 @@ func TestHandleRestoreHooks(t *testing.T) {
 		},
 
 		{
-			name: "should preserve restic-wait init container when it exits with other init containers",
+			name: "should preserve restore-wait init container when it exits with other init containers",
 			podInput: corev1api.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "app1",
@@ -1755,8 +1727,8 @@ func TestHandleRestoreHooks(t *testing.T) {
 				},
 				Spec: corev1api.PodSpec{
 					InitContainers: []corev1api.Container{
-						*builder.ForContainer("restic-wait", "bus-box").
-							Command([]string{"restic-restore"}).Result(),
+						*builder.ForContainer("restore-wait", "bus-box").
+							Command([]string{"pod-volume-restore"}).Result(),
 						*builder.ForContainer("init-app-step1", "busy-box").
 							Command([]string{"init-step1"}).Result(),
 						*builder.ForContainer("init-app-step2", "busy-box").
@@ -1772,8 +1744,8 @@ func TestHandleRestoreHooks(t *testing.T) {
 				},
 				Spec: corev1api.PodSpec{
 					InitContainers: []corev1api.Container{
-						*builder.ForContainer("restic-wait", "bus-box").
-							Command([]string{"restic-restore"}).Result(),
+						*builder.ForContainer("restore-wait", "bus-box").
+							Command([]string{"pod-volume-restore"}).Result(),
 						*builder.ForContainer("restore-init-container-1", "nginx").
 							Command([]string{"a", "b", "c"}).Result(),
 						*builder.ForContainer("restore-init-container-2", "nginx").
@@ -1795,11 +1767,11 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
-									*builder.ForContainer("restore-init-container-2", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
+									builder.ForContainer("restore-init-container-2", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1868,9 +1840,9 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1911,9 +1883,9 @@ func TestHandleRestoreHooks(t *testing.T) {
 					RestoreHooks: []velerov1api.RestoreResourceHook{
 						{
 							Init: &velerov1api.InitRestoreHook{
-								InitContainers: []corev1api.Container{
-									*builder.ForContainer("restore-init-container-1", "nginx").
-										Command([]string{"a", "b", "c"}).Result(),
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										Command([]string{"a", "b", "c"}).ResultRawExtension(),
 								},
 							},
 						},
@@ -1922,6 +1894,37 @@ func TestHandleRestoreHooks(t *testing.T) {
 			},
 			namespaceMapping: map[string]string{"default": "new"},
 		},
+		{
+			name: "Invalid InitContainer in Restore hook should return nil as pod, and error.",
+			podInput: corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "app1",
+					Namespace: "new",
+				},
+				Spec: corev1api.PodSpec{},
+			},
+			expectedError: fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field"),
+			expectedPod:   nil,
+			restoreHooks: []ResourceRestoreHook{
+				{
+					Name: "hook1",
+					Selector: ResourceHookSelector{
+						Namespaces: collections.NewIncludesExcludes().Includes("new"),
+						Resources:  collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource),
+					},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Init: &velerov1api.InitRestoreHook{
+								InitContainers: []runtime.RawExtension{
+									builder.ForContainer("restore-init-container-1", "nginx").
+										ResultRawExtension(),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -1931,10 +1934,32 @@ func TestHandleRestoreHooks(t *testing.T) {
 			assert.NoError(t, err)
 			actual, err := handler.HandleRestoreHooks(velerotest.NewLogger(), kuberesource.Pods, &unstructured.Unstructured{Object: podMap}, tc.restoreHooks, tc.namespaceMapping)
 			assert.Equal(t, tc.expectedError, err)
-			actualPod := new(corev1api.Pod)
-			err = runtime.DefaultUnstructuredConverter.FromUnstructured(actual.UnstructuredContent(), actualPod)
-			assert.NoError(t, err)
-			assert.Equal(t, tc.expectedPod, actualPod)
+			if actual != nil {
+				actualPod := new(corev1api.Pod)
+				err = runtime.DefaultUnstructuredConverter.FromUnstructured(actual.UnstructuredContent(), actualPod)
+				assert.NoError(t, err)
+				assert.Equal(t, tc.expectedPod, actualPod)
+			}
 		})
 	}
 }
+
+func TestValidateContainer(t *testing.T) {
+	valid := `{"name": "test", "image": "busybox", "command": ["pwd"]}`
+	noName := `{"image": "busybox", "command": ["pwd"]}`
+	noImage := `{"name": "test", "command": ["pwd"]}`
+	noCommand := `{"name": "test", "image": "busybox"}`
+	expectedError := fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field")
+
+	// valid string should return nil as result.
+	assert.Equal(t, nil, ValidateContainer([]byte(valid)))
+
+	// noName string should return expected error as result.
+	assert.Equal(t, expectedError, ValidateContainer([]byte(noName)))
+
+	// noImage string should return expected error as result.
+	assert.Equal(t, expectedError, ValidateContainer([]byte(noImage)))
+
+	// noCommand string should return expected error as result.
+	assert.Equal(t, expectedError, ValidateContainer([]byte(noCommand)))
+}

internal/hook/wait_exec_hook_handler.go

@@ -119,7 +119,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 		)
 
 		if newPod.Status.Phase == v1.PodSucceeded || newPod.Status.Phase == v1.PodFailed {
-			err := fmt.Errorf("Pod entered phase %s before some post-restore exec hooks ran", newPod.Status.Phase)
+			err := fmt.Errorf("pod entered phase %s before some post-restore exec hooks ran", newPod.Status.Phase)
 			podLog.Warning(err)
 			cancel()
 			return
@@ -155,7 +155,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 				)
 				// Check the individual hook's wait timeout is not expired
 				if hook.Hook.WaitTimeout.Duration != 0 && time.Since(waitStart) > hook.Hook.WaitTimeout.Duration {
-					err := fmt.Errorf("Hook %s in container %s expired before executing", hook.HookName, hook.Hook.Container)
+					err := fmt.Errorf("hook %s in container %s expired before executing", hook.HookName, hook.Hook.Container)
 					hookLog.Error(err)
 					if hook.Hook.OnError == velerov1api.HookErrorModeFail {
 						errors = append(errors, err)
@@ -194,7 +194,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 			handler(newObj)
 		},
 		DeleteFunc: func(obj interface{}) {
-			err := fmt.Errorf("Pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod))
+			err := fmt.Errorf("pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod))
 			log.Error(err)
 			cancel()
 		},
@@ -212,7 +212,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 			if hook.executed {
 				continue
 			}
-			err := fmt.Errorf("Hook %s in container %s in pod %s not executed: %v", hook.HookName, hook.Hook.Container, kube.NamespaceAndName(pod), ctx.Err())
+			err := fmt.Errorf("hook %s in container %s in pod %s not executed: %v", hook.HookName, hook.Hook.Container, kube.NamespaceAndName(pod), ctx.Err())
 			hookLog := log.WithFields(
 				logrus.Fields{
 					"hookSource": hook.HookSource,

internal/hook/wait_exec_hook_handler_test.go

@@ -465,7 +465,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					},
 				}).
 				Result(),
-			expectedErrors: []error{errors.New("Hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
+			expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
 			byContainer: map[string][]PodExecRestoreHook{
 				"container1": {
 					{
@@ -496,7 +496,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 					},
 				}).
 				Result(),
-			expectedErrors: []error{errors.New("Hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
+			expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
 			byContainer: map[string][]PodExecRestoreHook{
 				"container1": {
 					{

internal/resourcepolicies/resource_policies.go

@@ -0,0 +1,154 @@
+package resourcepolicies
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
+)
+
+type VolumeActionType string
+
+const (
+	// currently only support configmap type of resource config
+	ConfigmapRefType string           = "configmap"
+	Skip             VolumeActionType = "skip"
+)
+
+// Action defined as one action for a specific way of backup
+type Action struct {
+	// Type defined specific type of action, currently only support 'skip'
+	Type VolumeActionType `yaml:"type"`
+	// Parameters defined map of parameters when executing a specific action
+	Parameters map[string]interface{} `yaml:"parameters,omitempty"`
+}
+
+// volumePolicy defined policy to conditions to match Volumes and related action to handle matched Volumes
+type volumePolicy struct {
+	// Conditions defined list of conditions to match Volumes
+	Conditions map[string]interface{} `yaml:"conditions"`
+	Action     Action                 `yaml:"action"`
+}
+
+// resourcePolicies currently defined slice of volume policies to handle backup
+type resourcePolicies struct {
+	Version        string         `yaml:"version"`
+	VolumePolicies []volumePolicy `yaml:"volumePolicies"`
+	// we may support other resource policies in the future, and they could be added separately
+	// OtherResourcePolicies []OtherResourcePolicy
+}
+
+type Policies struct {
+	version        string
+	volumePolicies []volPolicy
+	// OtherPolicies
+}
+
+func unmarshalResourcePolicies(yamlData *string) (*resourcePolicies, error) {
+	resPolicies := &resourcePolicies{}
+	if err := decodeStruct(strings.NewReader(*yamlData), resPolicies); err != nil {
+		return nil, fmt.Errorf("failed to decode yaml data into resource policies  %v", err)
+	} else {
+		return resPolicies, nil
+	}
+}
+
+func (policies *Policies) buildPolicy(resPolicies *resourcePolicies) error {
+	for _, vp := range resPolicies.VolumePolicies {
+		con, err := unmarshalVolConditions(vp.Conditions)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		volCap, err := parseCapacity(con.Capacity)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		var p volPolicy
+		p.action = vp.Action
+		p.conditions = append(p.conditions, &capacityCondition{capacity: *volCap})
+		p.conditions = append(p.conditions, &storageClassCondition{storageClass: con.StorageClass})
+		p.conditions = append(p.conditions, &nfsCondition{nfs: con.NFS})
+		p.conditions = append(p.conditions, &csiCondition{csi: con.CSI})
+		policies.volumePolicies = append(policies.volumePolicies, p)
+	}
+
+	// Other resource policies
+
+	policies.version = resPolicies.Version
+	return nil
+}
+
+func (p *Policies) match(res *structuredVolume) *Action {
+	for _, policy := range p.volumePolicies {
+		isAllMatch := false
+		for _, con := range policy.conditions {
+			if !con.match(res) {
+				isAllMatch = false
+				break
+			}
+			isAllMatch = true
+		}
+		if isAllMatch {
+			return &policy.action
+		}
+	}
+	return nil
+}
+
+func (p *Policies) GetMatchAction(res interface{}) (*Action, error) {
+	volume := &structuredVolume{}
+	switch obj := res.(type) {
+	case *v1.PersistentVolume:
+		volume.parsePV(obj)
+	case *v1.Volume:
+		volume.parsePodVolume(obj)
+	default:
+		return nil, errors.New("failed to convert object")
+	}
+	return p.match(volume), nil
+}
+
+func (p *Policies) Validate() error {
+	if p.version != currentSupportDataVersion {
+		return fmt.Errorf("incompatible version number %s with supported version %s", p.version, currentSupportDataVersion)
+	}
+
+	for _, policy := range p.volumePolicies {
+		if err := policy.action.validate(); err != nil {
+			return errors.WithStack(err)
+		}
+		for _, con := range policy.conditions {
+			if err := con.validate(); err != nil {
+				return errors.WithStack(err)
+			}
+		}
+	}
+	return nil
+}
+
+func GetResourcePoliciesFromConfig(cm *v1.ConfigMap) (*Policies, error) {
+	if cm == nil {
+		return nil, fmt.Errorf("could not parse config from nil configmap")
+	}
+	if len(cm.Data) != 1 {
+		return nil, fmt.Errorf("illegal resource policies %s/%s configmap", cm.Name, cm.Namespace)
+	}
+
+	var yamlData string
+	for _, v := range cm.Data {
+		yamlData = v
+	}
+
+	resPolicies, err := unmarshalResourcePolicies(&yamlData)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	policies := &Policies{}
+	if err := policies.buildPolicy(resPolicies); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return policies, nil
+}

internal/resourcepolicies/resource_policies_test.go

@@ -0,0 +1,381 @@
+package resourcepolicies
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestLoadResourcePolicies(t *testing.T) {
+	testCases := []struct {
+		name     string
+		yamlData string
+		wantErr  bool
+	}{
+		{
+			name: "unknown key in yaml",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		unknown: {}
+		storageClass:
+		- gp2
+		- ebs-sc
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+		{
+			name: "reduplicated key in yaml",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		capacity: "0,100Gi"
+		storageClass:
+		- gp2
+		- ebs-sc
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+		{
+			name: "error format of storageClass",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		storageClass: gp2
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+		{
+			name: "error format of csi",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		csi: gp2
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+		{
+			name: "error format of nfs",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		csi: {}
+		nfs: abc
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+		{
+			name: "supported formart volume policies",
+			yamlData: `version: v1
+	volumePolicies:
+	- conditions:
+		capacity: "0,100Gi"
+		csi:
+		  driver: aws.efs.csi.driver
+		nfs: {}
+		storageClass:
+		- gp2
+		- ebs-sc
+	  action:
+		type: skip`,
+			wantErr: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := unmarshalResourcePolicies(&tc.yamlData)
+
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("Expected error %v, but got error %v", tc.wantErr, err)
+			}
+		})
+	}
+}
+
+func TestGetResourceMatchedAction(t *testing.T) {
+	resPolicies := &resourcePolicies{
+		Version: "v1",
+		VolumePolicies: []volumePolicy{
+			{
+				Action: Action{Type: "skip"},
+				Conditions: map[string]interface{}{
+					"capacity":     "0,10Gi",
+					"storageClass": []string{"gp2", "ebs-sc"},
+					"csi": interface{}(
+						map[string]interface{}{
+							"driver": "aws.efs.csi.driver",
+						}),
+				},
+			},
+			{
+				Action: Action{Type: "volume-snapshot"},
+				Conditions: map[string]interface{}{
+					"capacity":     "10,100Gi",
+					"storageClass": []string{"gp2", "ebs-sc"},
+					"csi": interface{}(
+						map[string]interface{}{
+							"driver": "aws.efs.csi.driver",
+						}),
+				},
+			},
+			{
+				Action: Action{Type: "file-system-backup"},
+				Conditions: map[string]interface{}{
+					"storageClass": []string{"gp2", "ebs-sc"},
+					"csi": interface{}(
+						map[string]interface{}{
+							"driver": "aws.efs.csi.driver",
+						}),
+				},
+			},
+		},
+	}
+	testCases := []struct {
+		name           string
+		volume         *structuredVolume
+		expectedAction *Action
+	}{
+		{
+			name: "match policy",
+			volume: &structuredVolume{
+				capacity:     *resource.NewQuantity(5<<30, resource.BinarySI),
+				storageClass: "ebs-sc",
+				csi:          &csiVolumeSource{Driver: "aws.efs.csi.driver"},
+			},
+			expectedAction: &Action{Type: "skip"},
+		},
+		{
+			name: "both matches return the first policy",
+			volume: &structuredVolume{
+				capacity:     *resource.NewQuantity(50<<30, resource.BinarySI),
+				storageClass: "ebs-sc",
+				csi:          &csiVolumeSource{Driver: "aws.efs.csi.driver"},
+			},
+			expectedAction: &Action{Type: "volume-snapshot"},
+		},
+		{
+			name: "dismatch all policies",
+			volume: &structuredVolume{
+				capacity:     *resource.NewQuantity(50<<30, resource.BinarySI),
+				storageClass: "ebs-sc",
+				nfs:          &nFSVolumeSource{},
+			},
+			expectedAction: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			policies := &Policies{}
+			err := policies.buildPolicy(resPolicies)
+			if err != nil {
+				t.Errorf("Failed to build policy with error %v", err)
+			}
+
+			action := policies.match(tc.volume)
+			if action == nil {
+				if tc.expectedAction != nil {
+					t.Errorf("Expected action %v, but got result nil", tc.expectedAction.Type)
+				}
+			} else {
+				if tc.expectedAction != nil {
+					if action.Type != tc.expectedAction.Type {
+						t.Errorf("Expected action %v, but got result %v", tc.expectedAction.Type, action.Type)
+					}
+				} else {
+					t.Errorf("Expected action nil, but got result %v", action.Type)
+				}
+			}
+		})
+	}
+}
+
+func TestGetResourcePoliciesFromConfig(t *testing.T) {
+	// Create a test ConfigMap
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-configmap",
+			Namespace: "test-namespace",
+		},
+		Data: map[string]string{
+			"test-data": "version: v1\nvolumePolicies:\n- conditions:\n    capacity: '0,10Gi'\n  action:\n    type: skip",
+		},
+	}
+
+	// Call the function and check for errors
+	resPolicies, err := GetResourcePoliciesFromConfig(cm)
+	assert.Nil(t, err)
+
+	// Check that the returned resourcePolicies object contains the expected data
+	assert.Equal(t, "v1", resPolicies.version)
+	assert.Len(t, resPolicies.volumePolicies, 1)
+	policies := resourcePolicies{
+		Version: "v1",
+		VolumePolicies: []volumePolicy{
+			{
+				Conditions: map[string]interface{}{
+					"capacity": "0,10Gi",
+				},
+				Action: Action{
+					Type: Skip,
+				},
+			},
+		},
+	}
+	p := &Policies{}
+	err = p.buildPolicy(&policies)
+	if err != nil {
+		t.Fatalf("failed to build policy with error %v", err)
+	}
+	assert.Equal(t, p, resPolicies)
+}
+
+func TestGetMatchAction(t *testing.T) {
+	testCases := []struct {
+		name     string
+		yamlData string
+		vol      *v1.PersistentVolume
+		skip     bool
+	}{
+		{
+			name: "empty csi",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+   csi: {}
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					PersistentVolumeSource: v1.PersistentVolumeSource{
+						CSI: &v1.CSIPersistentVolumeSource{Driver: "aws.ebs.csi.driver"},
+					}},
+			},
+			skip: true,
+		},
+		{
+			name: "empty csi with pv no csi driver",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+   csi: {}
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					Capacity: v1.ResourceList{
+						v1.ResourceStorage: resource.MustParse("1Gi"),
+					}},
+			},
+			skip: false,
+		},
+		{
+			name: "csi not configured",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+    capacity: "0,100Gi"
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					Capacity: v1.ResourceList{
+						v1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+					PersistentVolumeSource: v1.PersistentVolumeSource{
+						CSI: &v1.CSIPersistentVolumeSource{Driver: "aws.ebs.csi.driver"},
+					}},
+			},
+			skip: true,
+		},
+		{
+			name: "empty nfs",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+    nfs: {}
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					PersistentVolumeSource: v1.PersistentVolumeSource{
+						NFS: &v1.NFSVolumeSource{Server: "192.168.1.20"},
+					}},
+			},
+			skip: true,
+		},
+		{
+			name: "nfs not configured",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+    capacity: "0,100Gi"
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					Capacity: v1.ResourceList{
+						v1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+					PersistentVolumeSource: v1.PersistentVolumeSource{
+						NFS: &v1.NFSVolumeSource{Server: "192.168.1.20"},
+					},
+				},
+			},
+			skip: true,
+		},
+		{
+			name: "empty nfs with pv no nfs volume source",
+			yamlData: `version: v1
+volumePolicies:
+- conditions:
+    capacity: "0,100Gi"
+    nfs: {}
+  action:
+    type: skip`,
+			vol: &v1.PersistentVolume{
+				Spec: v1.PersistentVolumeSpec{
+					Capacity: v1.ResourceList{
+						v1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+				},
+			},
+			skip: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			resPolicies, err := unmarshalResourcePolicies(&tc.yamlData)
+			if err != nil {
+				t.Fatalf("got error when get match action %v", err)
+			}
+			assert.Nil(t, err)
+			policies := &Policies{}
+			err = policies.buildPolicy(resPolicies)
+			assert.Nil(t, err)
+			action, err := policies.GetMatchAction(tc.vol)
+			assert.Nil(t, err)
+
+			if tc.skip {
+				if action.Type != Skip {
+					t.Fatalf("Expected action skip but is %v", action.Type)
+				}
+			} else if action != nil && action.Type == Skip {
+				t.Fatalf("Expected action not skip but is %v", action.Type)
+			}
+		})
+	}
+}

internal/resourcepolicies/volume_resources.go

@@ -0,0 +1,205 @@
+package resourcepolicies
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+	corev1api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+type volPolicy struct {
+	action     Action
+	conditions []volumeCondition
+}
+
+type volumeCondition interface {
+	match(v *structuredVolume) bool
+	validate() error
+}
+
+// capacity consist of the lower and upper boundary
+type capacity struct {
+	lower resource.Quantity
+	upper resource.Quantity
+}
+
+type structuredVolume struct {
+	capacity     resource.Quantity
+	storageClass string
+	nfs          *nFSVolumeSource
+	csi          *csiVolumeSource
+}
+
+func (s *structuredVolume) parsePV(pv *corev1api.PersistentVolume) {
+	s.capacity = *pv.Spec.Capacity.Storage()
+	s.storageClass = pv.Spec.StorageClassName
+	nfs := pv.Spec.NFS
+	if nfs != nil {
+		s.nfs = &nFSVolumeSource{Server: nfs.Server, Path: nfs.Path}
+	}
+
+	csi := pv.Spec.CSI
+	if csi != nil {
+		s.csi = &csiVolumeSource{Driver: csi.Driver}
+	}
+}
+
+func (s *structuredVolume) parsePodVolume(vol *corev1api.Volume) {
+	nfs := vol.NFS
+	if nfs != nil {
+		s.nfs = &nFSVolumeSource{Server: nfs.Server, Path: nfs.Path}
+	}
+
+	csi := vol.CSI
+	if csi != nil {
+		s.csi = &csiVolumeSource{Driver: csi.Driver}
+	}
+}
+
+type capacityCondition struct {
+	capacity capacity
+}
+
+func (c *capacityCondition) match(v *structuredVolume) bool {
+	return c.capacity.isInRange(v.capacity)
+}
+
+type storageClassCondition struct {
+	storageClass []string
+}
+
+func (s *storageClassCondition) match(v *structuredVolume) bool {
+	if len(s.storageClass) == 0 {
+		return true
+	}
+
+	if v.storageClass == "" {
+		return false
+	}
+
+	for _, sc := range s.storageClass {
+		if v.storageClass == sc {
+			return true
+		}
+	}
+
+	return false
+}
+
+type nfsCondition struct {
+	nfs *nFSVolumeSource
+}
+
+func (c *nfsCondition) match(v *structuredVolume) bool {
+	if c.nfs == nil {
+		return true
+	}
+	if v.nfs == nil {
+		return false
+	}
+
+	if c.nfs.Path == "" {
+		if c.nfs.Server == "" { // match nfs: {}
+			return v.nfs != nil
+		}
+		if c.nfs.Server != v.nfs.Server {
+			return false
+		}
+		return true
+	}
+	if c.nfs.Path != v.nfs.Path {
+		return false
+	}
+	if c.nfs.Server == "" {
+		return true
+	}
+	if c.nfs.Server != v.nfs.Server {
+		return false
+	}
+	return true
+
+}
+
+type csiCondition struct {
+	csi *csiVolumeSource
+}
+
+func (c *csiCondition) match(v *structuredVolume) bool {
+	if c.csi == nil {
+		return true
+	}
+
+	if c.csi.Driver == "" { // match csi: {}
+		return v.csi != nil
+	}
+
+	if v.csi == nil {
+		return false
+	}
+
+	return c.csi.Driver == v.csi.Driver
+}
+
+// parseCapacity parse string into capacity format
+func parseCapacity(cap string) (*capacity, error) {
+	if cap == "" {
+		cap = ","
+	}
+	capacities := strings.Split(cap, ",")
+	var quantities []resource.Quantity
+	if len(capacities) != 2 {
+		return nil, fmt.Errorf("wrong format of Capacity %v", cap)
+	} else {
+		for _, v := range capacities {
+			if strings.TrimSpace(v) == "" {
+				// case similar "10Gi,"
+				// if empty, the quantity will assigned with 0
+				quantities = append(quantities, *resource.NewQuantity(int64(0), resource.DecimalSI))
+			} else {
+				if quantity, err := resource.ParseQuantity(strings.TrimSpace(v)); err != nil {
+					return nil, fmt.Errorf("wrong format of Capacity %v with err %v", v, err)
+				} else {
+					quantities = append(quantities, quantity)
+				}
+			}
+		}
+	}
+	return &capacity{lower: quantities[0], upper: quantities[1]}, nil
+}
+
+// isInRange returns true if the quantity y is in range of capacity, or it returns false
+func (c *capacity) isInRange(y resource.Quantity) bool {
+	if c.lower.IsZero() && c.upper.Cmp(y) >= 0 {
+		// [0, a] y
+		return true
+	}
+	if c.upper.IsZero() && c.lower.Cmp(y) <= 0 {
+		// [b, 0] y
+		return true
+	}
+	if !c.lower.IsZero() && !c.upper.IsZero() {
+		// [a, b] y
+		return c.lower.Cmp(y) <= 0 && c.upper.Cmp(y) >= 0
+	}
+	return false
+}
+
+// unmarshalVolConditions parse map[string]interface{} into volumeConditions format
+// and validate key fields of the map.
+func unmarshalVolConditions(con map[string]interface{}) (*volumeConditions, error) {
+	volConditons := &volumeConditions{}
+	buffer := new(bytes.Buffer)
+	err := yaml.NewEncoder(buffer).Encode(con)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to encode volume conditions")
+	}
+
+	if err := decodeStruct(buffer, volConditons); err != nil {
+		return nil, errors.Wrap(err, "failed to decode volume conditions")
+	}
+	return volConditons, nil
+}

internal/resourcepolicies/volume_resources_test.go

@@ -0,0 +1,412 @@
+package resourcepolicies
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+func setStructuredVolume(capacity resource.Quantity, sc string, nfs *nFSVolumeSource, csi *csiVolumeSource) *structuredVolume {
+	return &structuredVolume{
+		capacity:     capacity,
+		storageClass: sc,
+		nfs:          nfs,
+		csi:          csi,
+	}
+}
+
+func TestParseCapacity(t *testing.T) {
+	var emptyCapacity capacity
+	tests := []struct {
+		input       string
+		expected    capacity
+		expectedErr error
+	}{
+		{"10Gi,20Gi", capacity{lower: *resource.NewQuantity(10<<30, resource.BinarySI), upper: *resource.NewQuantity(20<<30, resource.BinarySI)}, nil},
+		{"10Gi,", capacity{lower: *resource.NewQuantity(10<<30, resource.BinarySI), upper: *resource.NewQuantity(0, resource.DecimalSI)}, nil},
+		{"10Gi", emptyCapacity, fmt.Errorf("wrong format of Capacity 10Gi")},
+		{"", emptyCapacity, nil},
+	}
+
+	for _, test := range tests {
+		test := test // capture range variable
+		t.Run(test.input, func(t *testing.T) {
+			actual, actualErr := parseCapacity(test.input)
+			if test.expected != emptyCapacity {
+				assert.Equal(t, test.expected.lower.Cmp(actual.lower), 0)
+				assert.Equal(t, test.expected.upper.Cmp(actual.upper), 0)
+			}
+			assert.Equal(t, test.expectedErr, actualErr)
+		})
+	}
+}
+
+func TestCapacityIsInRange(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		capacity  *capacity
+		quantity  resource.Quantity
+		isInRange bool
+	}{
+		{&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(10<<30, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), true},
+		{&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(10<<30, resource.BinarySI)}, *resource.NewQuantity(15<<30, resource.BinarySI), false},
+		{&capacity{*resource.NewQuantity(20<<30, resource.BinarySI), *resource.NewQuantity(0, resource.DecimalSI)}, *resource.NewQuantity(25<<30, resource.BinarySI), true},
+		{&capacity{*resource.NewQuantity(20<<30, resource.BinarySI), *resource.NewQuantity(0, resource.DecimalSI)}, *resource.NewQuantity(15<<30, resource.BinarySI), false},
+		{&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(15<<30, resource.BinarySI), true},
+		{&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), false},
+		{&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(25<<30, resource.BinarySI), false},
+		{&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(0, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), true},
+	}
+
+	for _, test := range tests {
+		test := test // capture range variable
+		t.Run(fmt.Sprintf("%v with %v", test.capacity, test.quantity), func(t *testing.T) {
+			t.Parallel()
+
+			actual := test.capacity.isInRange(test.quantity)
+
+			assert.Equal(t, test.isInRange, actual)
+
+		})
+	}
+}
+
+func TestStorageClassConditionMatch(t *testing.T) {
+	tests := []struct {
+		name          string
+		condition     *storageClassCondition
+		volume        *structuredVolume
+		expectedMatch bool
+	}{
+		{
+			name:          "match single storage class",
+			condition:     &storageClassCondition{[]string{"gp2"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "gp2", nil, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "match multiple storage classes",
+			condition:     &storageClassCondition{[]string{"gp2", "ebs-sc"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "gp2", nil, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "mismatch storage class",
+			condition:     &storageClassCondition{[]string{"gp2"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "ebs-sc", nil, nil),
+			expectedMatch: false,
+		},
+		{
+			name:          "empty storage class",
+			condition:     &storageClassCondition{[]string{}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "ebs-sc", nil, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty volume storage class",
+			condition:     &storageClassCondition{[]string{"gp2"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, nil),
+			expectedMatch: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			match := tt.condition.match(tt.volume)
+			if match != tt.expectedMatch {
+				t.Errorf("expected %v, but got %v", tt.expectedMatch, match)
+			}
+		})
+	}
+}
+
+func TestNFSConditionMatch(t *testing.T) {
+
+	tests := []struct {
+		name          string
+		condition     *nfsCondition
+		volume        *structuredVolume
+		expectedMatch bool
+	}{
+		{
+			name:          "match nfs condition",
+			condition:     &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty nfs condition",
+			condition:     &nfsCondition{nil},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty nfs server and path condition",
+			condition:     &nfsCondition{&nFSVolumeSource{Server: "", Path: ""}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "server dismatch",
+			condition:     &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20", Path: ""}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: ""}, nil),
+			expectedMatch: false,
+		},
+		{
+			name:          "empty nfs server condition",
+			condition:     &nfsCondition{&nFSVolumeSource{Path: "/mnt/data"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20", Path: "/mnt/data"}, nil),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty nfs volume",
+			condition:     &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, nil),
+			expectedMatch: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			match := tt.condition.match(tt.volume)
+			if match != tt.expectedMatch {
+				t.Errorf("expected %v, but got %v", tt.expectedMatch, match)
+			}
+		})
+	}
+}
+
+func TestCSIConditionMatch(t *testing.T) {
+
+	tests := []struct {
+		name          string
+		condition     *csiCondition
+		volume        *structuredVolume
+		expectedMatch bool
+	}{
+		{
+			name:          "match csi condition",
+			condition:     &csiCondition{&csiVolumeSource{Driver: "test"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test"}),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty csi condition",
+			condition:     &csiCondition{nil},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test"}),
+			expectedMatch: true,
+		},
+		{
+			name:          "empty csi volume",
+			condition:     &csiCondition{&csiVolumeSource{Driver: "test"}},
+			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{}),
+			expectedMatch: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			match := tt.condition.match(tt.volume)
+			if match != tt.expectedMatch {
+				t.Errorf("expected %v, but got %v", tt.expectedMatch, match)
+			}
+		})
+	}
+}
+
+func TestUnmarshalVolumeConditions(t *testing.T) {
+	testCases := []struct {
+		name          string
+		input         map[string]interface{}
+		expectedError string
+	}{
+		{
+			name: "Valid input",
+			input: map[string]interface{}{
+				"capacity": "1Gi,10Gi",
+				"storageClass": []string{
+					"gp2",
+					"ebs-sc",
+				},
+				"csi": &csiVolumeSource{
+					Driver: "aws.efs.csi.driver",
+				},
+			},
+			expectedError: "",
+		},
+		{
+			name: "Invalid input: invalid capacity filed name",
+			input: map[string]interface{}{
+				"Capacity": "1Gi,10Gi",
+			},
+			expectedError: "field Capacity not found",
+		},
+		{
+			name: "Invalid input: invalid storage class format",
+			input: map[string]interface{}{
+				"storageClass": "ebs-sc",
+			},
+			expectedError: "str `ebs-sc` into []string",
+		},
+		{
+			name: "Invalid input: invalid csi format",
+			input: map[string]interface{}{
+				"csi": "csi.driver",
+			},
+			expectedError: "str `csi.driver` into resourcepolicies.csiVolumeSource",
+		},
+		{
+			name: "Invalid input: unknown field",
+			input: map[string]interface{}{
+				"unknown": "foo",
+			},
+			expectedError: "field unknown not found in type",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := unmarshalVolConditions(tc.input)
+			if tc.expectedError != "" {
+				if err == nil {
+					t.Errorf("Expected error '%s', but got nil", tc.expectedError)
+				} else if !strings.Contains(err.Error(), tc.expectedError) {
+					t.Errorf("Expected error '%s', but got '%v'", tc.expectedError, err)
+				}
+			}
+		})
+	}
+}
+
+func TestParsePodVolume(t *testing.T) {
+	// Mock data
+	nfsVolume := corev1api.Volume{}
+	nfsVolume.NFS = &corev1api.NFSVolumeSource{
+		Server: "nfs.example.com",
+		Path:   "/exports/data",
+	}
+	csiVolume := corev1api.Volume{}
+	csiVolume.CSI = &corev1api.CSIVolumeSource{
+		Driver: "csi.example.com",
+	}
+	emptyVolume := corev1api.Volume{}
+
+	// Test cases
+	testCases := []struct {
+		name        string
+		inputVolume *corev1api.Volume
+		expectedNFS *nFSVolumeSource
+		expectedCSI *csiVolumeSource
+	}{
+		{
+			name:        "NFS volume",
+			inputVolume: &nfsVolume,
+			expectedNFS: &nFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"},
+		},
+		{
+			name:        "CSI volume",
+			inputVolume: &csiVolume,
+			expectedCSI: &csiVolumeSource{Driver: "csi.example.com"},
+		},
+		{
+			name:        "Empty volume",
+			inputVolume: &emptyVolume,
+			expectedNFS: nil,
+			expectedCSI: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Call the function
+			structuredVolume := &structuredVolume{}
+			structuredVolume.parsePodVolume(tc.inputVolume)
+
+			// Check the results
+			if tc.expectedNFS != nil {
+				if structuredVolume.nfs == nil {
+					t.Errorf("Expected a non-nil NFS volume source")
+				} else if *tc.expectedNFS != *structuredVolume.nfs {
+					t.Errorf("NFS volume source does not match expected value")
+				}
+			}
+			if tc.expectedCSI != nil {
+				if structuredVolume.csi == nil {
+					t.Errorf("Expected a non-nil CSI volume source")
+				} else if *tc.expectedCSI != *structuredVolume.csi {
+					t.Errorf("CSI volume source does not match expected value")
+				}
+			}
+		})
+	}
+}
+
+func TestParsePV(t *testing.T) {
+	// Mock data
+	nfsVolume := corev1api.PersistentVolume{}
+	nfsVolume.Spec.Capacity = corev1api.ResourceList{corev1api.ResourceStorage: resource.MustParse("1Gi")}
+	nfsVolume.Spec.NFS = &corev1api.NFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"}
+	csiVolume := corev1api.PersistentVolume{}
+	csiVolume.Spec.Capacity = corev1api.ResourceList{corev1api.ResourceStorage: resource.MustParse("2Gi")}
+	csiVolume.Spec.CSI = &corev1api.CSIPersistentVolumeSource{Driver: "csi.example.com"}
+	emptyVolume := corev1api.PersistentVolume{}
+
+	// Test cases
+	testCases := []struct {
+		name        string
+		inputVolume *corev1api.PersistentVolume
+		expectedNFS *nFSVolumeSource
+		expectedCSI *csiVolumeSource
+	}{
+		{
+			name:        "NFS volume",
+			inputVolume: &nfsVolume,
+			expectedNFS: &nFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"},
+			expectedCSI: nil,
+		},
+		{
+			name:        "CSI volume",
+			inputVolume: &csiVolume,
+			expectedNFS: nil,
+			expectedCSI: &csiVolumeSource{Driver: "csi.example.com"},
+		},
+		{
+			name:        "Empty volume",
+			inputVolume: &emptyVolume,
+			expectedNFS: nil,
+			expectedCSI: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Call the function
+			structuredVolume := &structuredVolume{}
+			structuredVolume.parsePV(tc.inputVolume)
+			// Check the results
+			if structuredVolume.capacity != *tc.inputVolume.Spec.Capacity.Storage() {
+				t.Errorf("capacity does not match expected value")
+			}
+			if structuredVolume.storageClass != tc.inputVolume.Spec.StorageClassName {
+				t.Errorf("Storage class does not match expected value")
+			}
+			if tc.expectedNFS != nil {
+				if structuredVolume.nfs == nil {
+					t.Errorf("Expected a non-nil NFS volume source")
+				} else if *tc.expectedNFS != *structuredVolume.nfs {
+					t.Errorf("NFS volume source does not match expected value")
+				}
+			}
+			if tc.expectedCSI != nil {
+				if structuredVolume.csi == nil {
+					t.Errorf("Expected a non-nil CSI volume source")
+				} else if *tc.expectedCSI != *structuredVolume.csi {
+					t.Errorf("CSI volume source does not match expected value")
+				}
+			}
+		})
+	}
+}

internal/resourcepolicies/volume_resources_validator.go

@@ -0,0 +1,76 @@
+package resourcepolicies
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+const currentSupportDataVersion = "v1"
+
+type csiVolumeSource struct {
+	Driver string `yaml:"driver,omitempty"`
+}
+
+type nFSVolumeSource struct {
+	// Server is the hostname or IP address of the NFS server
+	Server string `yaml:"server,omitempty"`
+	// Path is the exported NFS share
+	Path string `yaml:"path,omitempty"`
+}
+
+// volumeConditions defined the current format of conditions we parsed
+type volumeConditions struct {
+	Capacity     string           `yaml:"capacity,omitempty"`
+	StorageClass []string         `yaml:"storageClass,omitempty"`
+	NFS          *nFSVolumeSource `yaml:"nfs,omitempty"`
+	CSI          *csiVolumeSource `yaml:"csi,omitempty"`
+}
+
+func (c *capacityCondition) validate() error {
+	// [0, a]
+	// [a, b]
+	// [b, 0]
+	// ==> low <= upper or upper is zero
+	if (c.capacity.upper.Cmp(c.capacity.lower) >= 0) ||
+		(!c.capacity.lower.IsZero() && c.capacity.upper.IsZero()) {
+		return nil
+	}
+	return errors.Errorf("illegal values for capacity %v", c.capacity)
+
+}
+
+func (s *storageClassCondition) validate() error {
+	// validate by yamlv3
+	return nil
+}
+
+func (c *nfsCondition) validate() error {
+	// validate by yamlv3
+	return nil
+}
+
+func (c *csiCondition) validate() error {
+	// validate by yamlv3
+	return nil
+}
+
+// decodeStruct restric validate the keys in decoded mappings to exist as fields in the struct being decoded into
+func decodeStruct(r io.Reader, s interface{}) error {
+	dec := yaml.NewDecoder(r)
+	dec.KnownFields(true)
+	return dec.Decode(s)
+}
+
+// validate check action format
+func (a *Action) validate() error {
+	// validate Type
+	if a.Type != Skip {
+		return fmt.Errorf("invalid action type %s", a.Type)
+	}
+
+	// TODO validate parameters
+	return nil
+}

internal/resourcepolicies/volume_resources_validator_test.go

@@ -0,0 +1,251 @@
+package resourcepolicies
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+func TestCapacityConditionValidate(t *testing.T) {
+	testCases := []struct {
+		name     string
+		capacity *capacity
+		wantErr  bool
+	}{
+		{
+			name:     "lower and upper are both zero",
+			capacity: &capacity{lower: *resource.NewQuantity(0, resource.DecimalSI), upper: *resource.NewQuantity(0, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower is zero and upper is greater than zero",
+			capacity: &capacity{lower: *resource.NewQuantity(0, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower is greater than upper",
+			capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(50, resource.DecimalSI)},
+			wantErr:  true,
+		},
+		{
+			name:     "lower and upper are equal",
+			capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower is greater than zero and upper is zero",
+			capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(0, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower and upper are both not zero and lower is less than upper",
+			capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(200, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower and upper are both not zero and lower is equal to upper",
+			capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)},
+			wantErr:  false,
+		},
+		{
+			name:     "lower and upper are both not zero and lower is greater than upper",
+			capacity: &capacity{lower: *resource.NewQuantity(200, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)},
+			wantErr:  true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := &capacityCondition{capacity: *tc.capacity}
+			err := c.validate()
+
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("Expected error %v, but got error %v", tc.wantErr, err)
+			}
+		})
+	}
+}
+
+func TestValidate(t *testing.T) {
+	testCases := []struct {
+		name    string
+		res     *resourcePolicies
+		wantErr bool
+	}{
+		{
+			name: "unknown key in yaml",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "0,10Gi",
+							"unknown":      "",
+							"storageClass": []string{"gp2", "ebs-sc"},
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "error format of capacity",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "10Gi",
+							"storageClass": []string{"gp2", "ebs-sc"},
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "error format of storageClass",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "0,10Gi",
+							"storageClass": "ebs-sc",
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "error format of csi",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "0,10Gi",
+							"storageClass": []string{"gp2", "ebs-sc"},
+							"csi":          "aws.efs.csi.driver",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "unsupported version",
+			res: &resourcePolicies{
+				Version: "v2",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity": "0,10Gi",
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "unsupported action",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "unsupported"},
+						Conditions: map[string]interface{}{
+							"capacity": "0,10Gi",
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "error format of nfs",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "0,10Gi",
+							"storageClass": []string{"gp2", "ebs-sc"},
+							"nfs":          "aws.efs.csi.driver",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "supported formart volume policies",
+			res: &resourcePolicies{
+				Version: "v1",
+				VolumePolicies: []volumePolicy{
+					{
+						Action: Action{Type: "skip"},
+						Conditions: map[string]interface{}{
+							"capacity":     "0,10Gi",
+							"storageClass": []string{"gp2", "ebs-sc"},
+							"csi": interface{}(
+								map[string]interface{}{
+									"driver": "aws.efs.csi.driver",
+								}),
+							"nfs": interface{}(
+								map[string]interface{}{
+									"server": "192.168.20.90",
+									"path":   "/mnt/data/",
+								}),
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			policies := &Policies{}
+			err1 := policies.buildPolicy(tc.res)
+			err2 := policies.Validate()
+
+			if tc.wantErr {
+				if err1 == nil && err2 == nil {
+					t.Fatalf("Expected error %v, but not get error", tc.wantErr)
+				}
+			} else {
+				if err1 != nil || err2 != nil {
+					t.Fatalf("Expected error %v, but got error %v %v", tc.wantErr, err1, err2)
+				}
+			}
+		})
+	}
+}

internal/restartabletest/restartable_delegate.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package clientmgmt
+package restartabletest
 
 import (
 	"reflect"
@@ -24,55 +24,83 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
-	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
+	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
 )
 
-type restartableDelegateTest struct {
-	function                string
-	inputs                  []interface{}
-	expectedErrorOutputs    []interface{}
-	expectedDelegateOutputs []interface{}
+type MockRestartableProcess struct {
+	mock.Mock
 }
 
-type mockable interface {
+func (rp *MockRestartableProcess) AddReinitializer(key process.KindAndName, r process.Reinitializer) {
+	rp.Called(key, r)
+}
+
+func (rp *MockRestartableProcess) Reset() error {
+	args := rp.Called()
+	return args.Error(0)
+}
+
+func (rp *MockRestartableProcess) ResetIfNeeded() error {
+	args := rp.Called()
+	return args.Error(0)
+}
+
+func (rp *MockRestartableProcess) GetByKindAndName(key process.KindAndName) (interface{}, error) {
+	args := rp.Called(key)
+	return args.Get(0), args.Error(1)
+}
+
+func (rp *MockRestartableProcess) Stop() {
+	rp.Called()
+}
+
+type RestartableDelegateTest struct {
+	Function                string
+	Inputs                  []interface{}
+	ExpectedErrorOutputs    []interface{}
+	ExpectedDelegateOutputs []interface{}
+}
+
+type Mockable interface {
 	Test(t mock.TestingT)
 	On(method string, args ...interface{}) *mock.Call
 	AssertExpectations(t mock.TestingT) bool
 }
 
-func runRestartableDelegateTests(
+func RunRestartableDelegateTests(
 	t *testing.T,
-	kind framework.PluginKind,
-	newRestartable func(key kindAndName, p RestartableProcess) interface{},
-	newMock func() mockable,
-	tests ...restartableDelegateTest,
+	kind common.PluginKind,
+	newRestartable func(key process.KindAndName, p process.RestartableProcess) interface{},
+	newMock func() Mockable,
+	tests ...RestartableDelegateTest,
 ) {
 	for _, tc := range tests {
-		t.Run(tc.function, func(t *testing.T) {
-			p := new(mockRestartableProcess)
+		t.Run(tc.Function, func(t *testing.T) {
+			p := new(MockRestartableProcess)
 			p.Test(t)
 			defer p.AssertExpectations(t)
 
 			// getDelegate error
-			p.On("resetIfNeeded").Return(errors.Errorf("reset error")).Once()
+			p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once()
 			name := "delegateName"
-			key := kindAndName{kind: kind, name: name}
+			key := process.KindAndName{Kind: kind, Name: name}
 			r := newRestartable(key, p)
 
 			// Get the method we're going to call using reflection
-			method := reflect.ValueOf(r).MethodByName(tc.function)
+			method := reflect.ValueOf(r).MethodByName(tc.Function)
 			require.NotEmpty(t, method)
 
 			// Convert the test case inputs ([]interface{}) to []reflect.Value
 			var inputValues []reflect.Value
-			for i := range tc.inputs {
-				inputValues = append(inputValues, reflect.ValueOf(tc.inputs[i]))
+			for i := range tc.Inputs {
+				inputValues = append(inputValues, reflect.ValueOf(tc.Inputs[i]))
 			}
 
 			// Invoke the method being tested
 			actual := method.Call(inputValues)
 
-			// This function asserts that the actual outputs match the expected outputs
+			// This Function asserts that the actual outputs match the expected outputs
 			checkOutputs := func(expected []interface{}, actual []reflect.Value) {
 				require.Equal(t, len(expected), len(actual))
 
@@ -90,7 +118,7 @@ func runRestartableDelegateTests(
 						continue
 					}
 
-					// If function returns nil as struct return type, we cannot just
+					// If Function returns nil as struct return type, we cannot just
 					// compare the interface to nil as its type will not be nil,
 					// only the value will be
 					if expected[i] == nil && reflect.ValueOf(a).Kind() == reflect.Ptr {
@@ -104,25 +132,25 @@ func runRestartableDelegateTests(
 			}
 
 			// Make sure we get what we expected when getDelegate returned an error
-			checkOutputs(tc.expectedErrorOutputs, actual)
+			checkOutputs(tc.ExpectedErrorOutputs, actual)
 
 			// Invoke delegate, make sure all returned values are passed through
-			p.On("resetIfNeeded").Return(nil)
+			p.On("ResetIfNeeded").Return(nil)
 
 			delegate := newMock()
 			delegate.Test(t)
 			defer delegate.AssertExpectations(t)
 
-			p.On("getByKindAndName", key).Return(delegate, nil)
+			p.On("GetByKindAndName", key).Return(delegate, nil)
 
 			// Set up the mocked method in the delegate
-			delegate.On(tc.function, tc.inputs...).Return(tc.expectedDelegateOutputs...)
+			delegate.On(tc.Function, tc.Inputs...).Return(tc.ExpectedDelegateOutputs...)
 
 			// Invoke the method being tested
 			actual = method.Call(inputValues)
 
 			// Make sure we get what we expected when invoking the delegate
-			checkOutputs(tc.expectedDelegateOutputs, actual)
+			checkOutputs(tc.ExpectedDelegateOutputs, actual)
 		})
 	}
 }

internal/storage/storagelocation.go

@@ -71,10 +71,7 @@ func IsReadyToValidate(bslValidationFrequency *metav1.Duration, lastValidationTi
 
 	// We want to validate BSL only if the set validation frequency/ interval has elapsed.
 	nextValidation := lastValidation.Add(validationFrequency) // next validation time: last validation time + validation frequency
-	if time.Now().UTC().Before(nextValidation) {              // ready only when NOW is equal to or after the next validation time
-		return false
-	}
-	return true
+	return !time.Now().UTC().Before(nextValidation)           // ready only when NOW is equal to or after the next validation time
 }
 
 // ListBackupStorageLocations verifies if there are any backup storage locations.

internal/storage/storagelocation_test.go

@@ -163,7 +163,7 @@ func TestListBackupStorageLocations(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			g := NewWithT(t)
 
-			client := fake.NewFakeClientWithScheme(scheme.Scheme, tt.backupLocations)
+			client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(tt.backupLocations).Build()
 			if tt.expectError {
 				_, err := ListBackupStorageLocations(context.Background(), client, "ns-1")
 				g.Expect(err).NotTo(BeNil())

internal/velero/images.go

@@ -44,8 +44,8 @@ func DefaultVeleroImage() string {
 	return fmt.Sprintf("%s/%s:%s", imageRegistry(), "velero", ImageTag())
 }
 
-// DefaultResticRestoreHelperImage returns the default container image to use for the restic restore helper
+// DefaultRestoreHelperImage returns the default container image to use for the restore helper
 // for this version of Velero.
-func DefaultResticRestoreHelperImage() string {
-	return fmt.Sprintf("%s/%s:%s", imageRegistry(), "velero-restic-restore-helper", ImageTag())
+func DefaultRestoreHelperImage() string {
+	return fmt.Sprintf("%s/%s:%s", imageRegistry(), "velero-restore-helper", ImageTag())
 }

internal/velero/images_test.go

@@ -135,6 +135,6 @@ func TestDefaultVeleroImage(t *testing.T) {
 	testDefaultImage(t, DefaultVeleroImage, "velero")
 }
 
-func TestDefaultResticRestoreHelperImage(t *testing.T) {
-	testDefaultImage(t, DefaultResticRestoreHelperImage, "velero-restic-restore-helper")
+func TestDefaultRestoreHelperImage(t *testing.T) {
+	testDefaultImage(t, DefaultRestoreHelperImage, "velero-restore-helper")
 }