Merge pull request #4721 from ywk253100/220307_golang_1.8

Bump up golang to 1.17.8

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,10 +10,13 @@ about: Tell us about a problem you are experiencing
 
 **What did you expect to happen:**
 
+**The following information will help us better understand what's going on**:
 
-**The output of the following commands will help us better understand what's going on**:
-(Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.)
+_If you are using velero v1.7.0+:_  
+Please use `velero debug  --backup <backupname> --restore <restorename>` to generate the support bundle, and attach to this issue, more options please refer to `velero debug --help` 
 
+_If you are using earlier versions:_  
+Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.)
 - `kubectl logs deployment/velero -n velero`
 - `velero backup describe <backupname>` or `kubectl get backup/<backupname> -n velero -o yaml`
 - `velero backup logs <backupname>`

.github/pull_request_template.md

@@ -9,5 +9,5 @@ Fixes #(issue)
 # Please indicate you've done the following:
 
 - [ ] [Accepted the DCO](https://velero.io/docs/v1.5/code-standards/#dco-sign-off). Commits without the DCO will delay acceptance.
-- [ ] [Created a changelog file](https://velero.io/docs/v1.5/code-standards/#adding-a-changelog) or added `/kind changelog-not-required`.
+- [ ] [Created a changelog file](https://velero.io/docs/v1.5/code-standards/#adding-a-changelog) or added `/kind changelog-not-required` as a comment on this pull request.
 - [ ] Updated the corresponding documentation in `site/content/docs/main`.

.github/workflows/crds-verify-kind.yaml

@@ -57,7 +57,6 @@ jobs:
       matrix:
         # Latest k8s versions. There's no series-based tag, nor is there a latest tag.
         k8s:
-          - 1.15.12
           - 1.16.15
           - 1.17.17
           - 1.18.15

.github/workflows/e2e-test-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
@@ -69,12 +69,13 @@ jobs:
           - 1.20.2
           - 1.21.1
           - 1.22.0
+          - 1.23.0
       fail-fast: false
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       - name: Check out the code
         uses: actions/checkout@v2

.github/workflows/pr-ci-check.yml

@@ -8,7 +8,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       - name: Check out the code
         uses: actions/checkout@v2

.github/workflows/push.yml

@@ -2,7 +2,9 @@ name: Main CI
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - 'main'
+      - 'release-**'
     tags:
       - '*'
 
@@ -16,7 +18,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.16
+        go-version: 1.17
       id: go
 
     - name: Check out code into the Go module directory

.goreleaser.yml

@@ -27,11 +27,9 @@ builds:
       - arm64
       - ppc64le
     ignore:
-      # don't build arm/arm64 for darwin or windows
+      # don't build arm for darwin and arm/arm64 for windows
       - goos: darwin
         goarch: arm
-      - goos: darwin
-        goarch: arm64
       - goos: darwin
         goarch: ppc64le
       - goos: windows

Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.16 as builder-env
+FROM --platform=$BUILDPLATFORM golang:1.17.8 as builder-env
 
 ARG GOPROXY
 ARG PKG
@@ -50,7 +50,8 @@ RUN mkdir -p /output/usr/bin && \
     go build -o /output/${BIN} \
     -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN}
 
-FROM gcr.io/distroless/base-debian10:nonroot
+# The digest for tag 'nonroot' at the time of the release
+FROM gcr.io/distroless/base-debian10@sha256:6dc8ca7c3bbdb1a00fd8f1229b1b8c88986a5818b830e3a42d4946982dbbf18b
 
 LABEL maintainer="Nolan Brubaker <brubakern@vmware.com>"
 

Makefile

@@ -84,7 +84,7 @@ endef
 # The version of restic binary to be downloaded
 RESTIC_VERSION ?= 0.12.1
 
-CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 windows-amd64 linux-ppc64le
+CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le
 BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH))
 BUILDX_OUTPUT_TYPE ?= docker
 
@@ -338,9 +338,9 @@ changelog:
 #		PUBLISH=false \
 #		make release
 #
-# To run the release, which will publish a *DRAFT* GitHub release in github.com/vmware-tanzu/velero 
+# To run the release, which will publish a *DRAFT* GitHub release in github.com/vmware-tanzu/velero
 # (you still need to review/publish the GitHub release manually):
-#		GITHUB_TOKEN=your-github-token \ 
+#		GITHUB_TOKEN=your-github-token \
 #		RELEASE_NOTES_FILE=changelogs/CHANGELOG-1.2.md \
 #		PUBLISH=true \
 #		make release
@@ -359,7 +359,7 @@ serve-docs: build-image-hugo
 	-it -p 1313:1313 \
 	$(HUGO_IMAGE) \
 	hugo server --bind=0.0.0.0 --enableGitInfo=false
-# gen-docs generates a new versioned docs directory under site/content/docs. 
+# gen-docs generates a new versioned docs directory under site/content/docs.
 # Please read the documentation in the script for instructions on how to use it.
 gen-docs:
 	@hack/release-tools/gen-docs.sh

PROJECT

@@ -1,7 +0,0 @@
-domain: io
-repo: github.com/vmware-tanzu/velero
-resources:
-- group: velero
-  kind: BackupStorageLocation
-  version: v1
-version: "2"

README.md

@@ -34,6 +34,18 @@ If you are ready to jump in and test, add code, or help with documentation, foll
 
 See [the list of releases][6] to find out about feature changes.
 
+### Velero compatibility matrix
+
+The following is a list of the supported Kubernetes versions for each Velero version.
+
+| Velero version | Kubernetes versions|
+|----------------|--------------------|
+| 1.8            | 1.16-latest        |
+| 1.6.3-1.7.1    | 1.12-latest        |
+| 1.60-1.6.2     | 1.12-1.21          |
+| 1.5            | 1.12-1.21          |
+| 1.4            | 1.10-1.21          |
+
 [1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg
 [2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI"
 [4]: https://github.com/vmware-tanzu/velero/issues

Tiltfile

@@ -50,7 +50,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip(
 
 tilt_helper_dockerfile_header = """
 # Tilt image
-FROM golang:1.16.6 as tilt-helper
+FROM golang:1.17 as tilt-helper
 
 # Support live reloading with Tilt
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \
@@ -103,7 +103,7 @@ local_resource(
 
 local_resource(
     "restic_binary",
-    cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=' + local_goos + ' GOARCH=amd64 RESTIC_VERSION=0.12.0 OUTPUT_DIR=_tiltbuild/restic ./hack/download-restic.sh',
+    cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=linux GOARCH=amd64 RESTIC_VERSION=0.12.0 OUTPUT_DIR=_tiltbuild/restic ./hack/download-restic.sh',
 )
 
 # Note: we need a distro with a bash shell to exec into the Velero container

changelogs/CHANGELOG-1.8.md

@@ -0,0 +1,114 @@
+## v1.8.1
+### 2022-03-04
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.8.1
+
+### Container Image
+`velero/velero:v1.8.1`
+
+### Documentation
+https://velero.io/docs/v1.8
+
+### Upgrading
+https://velero.io/docs/v1.8/upgrade-to-1.8/
+
+### All changes
+* Bypass the remap CRD version plugin when v1beta1 CRD is not supported (#4706, @reasonerjt)
+* Support regional pv for GKE (#4691, @jxun)
+* Bump up golang to 1.17.8 (#4721, @ywk253100)
+
+## v1.8.0
+### 2022-01-14
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.8.0
+
+### Container Image
+`velero/velero:v1.8.0`
+
+### Documentation
+https://velero.io/docs/v1.8
+
+### Upgrading
+https://velero.io/docs/v1.8/upgrade-to-1.8/
+
+### Highlights
+
+#### The plugins to support handling volumes created by the AWS CSI driver
+The new versions of plugins for AWS, Azure and GCP will be released with the support of snapshotting and restoring the persistent volumes provisioned by CSI driver via the APIs of the cloud providers.  With this enhancement, users can backup and restore the persistent volumes on these could providers without using the CSI plugin, which will remain beta and the feature flag `EnableCSI` will be disabled by default.
+
+For the version of the plugins and the CSI drivers they support respectively please see the table:
+
+| Plugin | Version | CSI Driver |
+| --- | ----------- | ---------- |
+| velero-plugin-for-aws | v1.4.0 | ebs.csi.aws.com |
+| velero-plugin-for-microsoft-azure | v1.4.0 | disk.csi.azure.com |
+| velero-plugin-for-gcp | v1.4.0 | pd.csi.storage.gke.io |
+
+#### Break change
+Starting v1.8 velero will only support v1 CRD, therefore, it will only run on Kubernetes v1.16+
+
+### All changes
+
+* E2E SSR test add retry mechanism and logs  (#4594, @mqiu)
+* Update doc for v1.8 (#4517, @reasonerjt)
+* E2E test on ssr object with controller namespace mix-ups (#4521, @mqiu)
+* Check whether the volume is provisioned by CSI driver or not by the annotation as well (#4513, @ywk253100)
+* Initialize the labels field of `velero backup-location create` option to avoid #4484 (#4491, @ywk253100)
+* Fix e2e 2500 namespaces scale test timeout problem (#4480, @mqiu)
+* Add backup deletion e2e test  (#4401, @danfengliu)
+* Return the error when getting backup store in backup deletion controller (#4465, @reasonerjt)
+* Ignore the provided port is already allocated error when restoring the LoadBalancer service (#4462, @ywk253100)
+* Revert #4423 migrate backup sync controller to kubebuilder. (#4457, @jxun)
+* Add rbac and annotation test cases (#4455, @mqiu)
+* remove --crds-version in velero install command. (#4446, @jxun)
+* Upgrade e2e test vsphere plugin (#4440, @mqiu)
+* Fix e2e test failures for the inappropriate optimaze of velero install (#4438, @mqiu)
+* Limit backup namespaces on test resource filtering cases (#4437, @mqiu)
+* Bump up Go to 1.17 (#4431, @reasonerjt)
+* Added `<backup name>`-itemsnapshots.json.gz to the backup format.  This file exists
+  when item snapshots are taken and contains an array of volume.Itemsnapshots
+  containing the information about the snapshots.  This will not be used unless
+  upload progress monitoring and item snapshots are enabled and an ItemSnapshot
+  plugin is used to take snapshots.
+
+Also added DownloadTargetKindBackupItemSnapshots for retrieving the signed URL to download only the `<backup name>`-itemsnapshots.json.gz part of a backup for use by
+`velero backup describe`. (#4429, @dsmithuchida)
+* Migrate backup sync controller from code-generator to kubebuilder. (#4423, @jxun)
+* Added UploadProgressFeature flag to enable Upload Progress Monitoring and Item
+  Snapshotters. (#4416, @dsmithuchida)
+* Added BackupWithResolvers and RestoreWithResolvers calls.  Will eventually replace Backup and Restore methods.
+  Adds ItemSnapshotters to Backup and Restore workflows. (#4410, @dsu)
+* Build for darwin-arm64 (#4409, @epk)
+* Add resource filtering test cases (#4404, @mqiu)
+* Fix the issue that the backup cannot be deleted after the application uninstalled (#4398, @ywk253100)
+* Add restoreactionitem plugin to handle admission webhook configurations (#4397, @reasonerjt)
+* Keep the annotation "pv.kubernetes.io/provisioned-by" when restoring PVs (#4391, @ywk253100)
+* Adjust structure of e2e test codes (#4386, @mqiu)
+* feat: migrate velero controller from kubebuilder v2 to v3
+  From Velero v1.8, apiextesions.k8s.io/v1beta1 is no longer supported,
+  which means only CRD of apiextensions.k8s.io/v1 is supported,
+  and the supported Kubernetes version is updated to v1.16 and later. (#4382, @jxun)
+* Delete backups and Restic repos associated with deleted BSL(s) (#4377, @codegold79)
+* Add the key for GKE zone for AZ collection (#4376, @reasonerjt)
+* Fix statefulsets volumeClaimTemplates storageClassName when use Changing PV/PVC Storage Classes (#4375, @Box-Cube)
+* Fix snapshot e2e test issue of jsonpath (#4372, @danfengliu)
+* Modify the timestamp in the name of a backup generated from schedule to use UTC. (#4353, @jxun)
+* Read Availability zone from nodeAffinity requirements  (#4350, @reasonerjt)
+* Use factory.Namespace() to replace hardcoded velero namespace (#4346, @half-life666)
+* Return the error if velero failed to detect S3 region for restic repo (#4343, @reasonerjt)
+* Add init log option for velero controller-runtime manager. (#4341, @jxun)
+* Ignore the `provided port is already allocated` error when restoring the `NodePort` service (#4336, @ywk253100)
+* Fixed an issue with the `backup-location create` command where the BSL Credential field would be set to an invalid empty SecretKeySelector when no credential details were provided. (#4322, @zubron)
+* fix buggy pager func (#4306, @alaypatel07)
+* Don't create a backup immediately after creating a schedule (#4281, @ywk253100)
+* Fix CVE-2020-29652 and CVE-2020-26160 (#4274, @ywk253100)
+* Refine tag-release.sh to align with change in release process (#4185, @reasonerjt)
+* Fix plugins incompatible issue in upgrade test (#4141, @danfengliu)
+* Verify group before treating resource as cohabitating (#4126, @sseago)
+* Added ItemSnapshotter plugin definition and plugin framework - addresses #3533.
+  Part of the Upload Progress enhancement (#3533) (#4077, @dsmithuchida)
+* Add upgrade test in E2E test (#4058, @danfengliu)
+* Handle namespace mapping for PVs without snapshots on restore (#3708, @sseago)
+

changelogs/unreleased/4058-danfengliu

@@ -1 +0,0 @@
-Add upgrade test in E2E test

changelogs/unreleased/4126-sseago

@@ -1 +0,0 @@
-Verify group before treating resource as cohabitating

changelogs/unreleased/4141-danfengliu

@@ -1 +0,0 @@
-Fix plugins incompatible issue in upgrade test

changelogs/unreleased/4185-reasonerjt

@@ -1 +0,0 @@
-Refine tag-release.sh to align with change in release process

changelogs/unreleased/4274-ywk253100

@@ -1 +0,0 @@
-Fix CVE-2020-29652 and CVE-2020-26160

changelogs/unreleased/4281-ywk253100

@@ -1 +0,0 @@
-Don't create a backup immediately after creating a schedule

config/crd/v1/bases/velero.io_backups.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: backups.velero.io
 spec:

config/crd/v1/bases/velero.io_backupstoragelocations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: backupstoragelocations.velero.io
 spec:

config/crd/v1/bases/velero.io_deletebackuprequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: deletebackuprequests.velero.io
 spec:

config/crd/v1/bases/velero.io_downloadrequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: downloadrequests.velero.io
 spec:
@@ -46,6 +46,7 @@ spec:
                     - BackupLog
                     - BackupContents
                     - BackupVolumeSnapshots
+                    - BackupItemSnapshots
                     - BackupResourceList
                     - RestoreLog
                     - RestoreResults

config/crd/v1/bases/velero.io_podvolumebackups.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: podvolumebackups.velero.io
 spec:

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: podvolumerestores.velero.io
 spec:

config/crd/v1/bases/velero.io_resticrepositories.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: resticrepositories.velero.io
 spec:

config/crd/v1/bases/velero.io_restores.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: restores.velero.io
 spec:
@@ -892,6 +892,7 @@ spec:
                                                   by services.
                                                 type: string
                                               protocol:
+                                                default: TCP
                                                 description: Protocol for port. Must
                                                   be UDP, TCP, or SCTP. Defaults to
                                                   "TCP".

config/crd/v1/bases/velero.io_schedules.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: schedules.velero.io
 spec:

config/crd/v1/bases/velero.io_serverstatusrequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: serverstatusrequests.velero.io
 spec:

config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: volumesnapshotlocations.velero.io
 spec:

config/crd/v1beta1/bases/velero.io_backups.yaml

@@ -1,439 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: backups.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: Backup
-    listKind: BackupList
-    plural: backups
-    singular: backup
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      description: Backup is a Velero resource that represents the capture of Kubernetes
-        cluster state at a point in time (API objects and associated volume state).
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: BackupSpec defines the specification for a Velero backup.
-          properties:
-            defaultVolumesToRestic:
-              description: DefaultVolumesToRestic specifies whether restic should
-                be used to take a backup of all pod volumes by default.
-              type: boolean
-            excludedNamespaces:
-              description: ExcludedNamespaces contains a list of namespaces that are
-                not included in the backup.
-              items:
-                type: string
-              nullable: true
-              type: array
-            excludedResources:
-              description: ExcludedResources is a slice of resource names that are
-                not included in the backup.
-              items:
-                type: string
-              nullable: true
-              type: array
-            hooks:
-              description: Hooks represent custom behaviors that should be executed
-                at different phases of the backup.
-              properties:
-                resources:
-                  description: Resources are hooks that should be executed when backing
-                    up individual instances of a resource.
-                  items:
-                    description: BackupResourceHookSpec defines one or more BackupResourceHooks
-                      that should be executed based on the rules defined for namespaces,
-                      resources, and label selector.
-                    properties:
-                      excludedNamespaces:
-                        description: ExcludedNamespaces specifies the namespaces to
-                          which this hook spec does not apply.
-                        items:
-                          type: string
-                        nullable: true
-                        type: array
-                      excludedResources:
-                        description: ExcludedResources specifies the resources to
-                          which this hook spec does not apply.
-                        items:
-                          type: string
-                        nullable: true
-                        type: array
-                      includedNamespaces:
-                        description: IncludedNamespaces specifies the namespaces to
-                          which this hook spec applies. If empty, it applies to all
-                          namespaces.
-                        items:
-                          type: string
-                        nullable: true
-                        type: array
-                      includedResources:
-                        description: IncludedResources specifies the resources to
-                          which this hook spec applies. If empty, it applies to all
-                          resources.
-                        items:
-                          type: string
-                        nullable: true
-                        type: array
-                      labelSelector:
-                        description: LabelSelector, if specified, filters the resources
-                          to which this hook spec applies.
-                        nullable: true
-                        properties:
-                          matchExpressions:
-                            description: matchExpressions is a list of label selector
-                              requirements. The requirements are ANDed.
-                            items:
-                              description: A label selector requirement is a selector
-                                that contains values, a key, and an operator that
-                                relates the key and values.
-                              properties:
-                                key:
-                                  description: key is the label key that the selector
-                                    applies to.
-                                  type: string
-                                operator:
-                                  description: operator represents a key's relationship
-                                    to a set of values. Valid operators are In, NotIn,
-                                    Exists and DoesNotExist.
-                                  type: string
-                                values:
-                                  description: values is an array of string values.
-                                    If the operator is In or NotIn, the values array
-                                    must be non-empty. If the operator is Exists or
-                                    DoesNotExist, the values array must be empty.
-                                    This array is replaced during a strategic merge
-                                    patch.
-                                  items:
-                                    type: string
-                                  type: array
-                              required:
-                              - key
-                              - operator
-                              type: object
-                            type: array
-                          matchLabels:
-                            additionalProperties:
-                              type: string
-                            description: matchLabels is a map of {key,value} pairs.
-                              A single {key,value} in the matchLabels map is equivalent
-                              to an element of matchExpressions, whose key field is
-                              "key", the operator is "In", and the values array contains
-                              only "value". The requirements are ANDed.
-                            type: object
-                        type: object
-                      name:
-                        description: Name is the name of this hook.
-                        type: string
-                      post:
-                        description: PostHooks is a list of BackupResourceHooks to
-                          execute after storing the item in the backup. These are
-                          executed after all "additional items" from item actions
-                          are processed.
-                        items:
-                          description: BackupResourceHook defines a hook for a resource.
-                          properties:
-                            exec:
-                              description: Exec defines an exec hook.
-                              properties:
-                                command:
-                                  description: Command is the command and arguments
-                                    to execute.
-                                  items:
-                                    type: string
-                                  minItems: 1
-                                  type: array
-                                container:
-                                  description: Container is the container in the pod
-                                    where the command should be executed. If not specified,
-                                    the pod's first container is used.
-                                  type: string
-                                onError:
-                                  description: OnError specifies how Velero should
-                                    behave if it encounters an error executing this
-                                    hook.
-                                  enum:
-                                  - Continue
-                                  - Fail
-                                  type: string
-                                timeout:
-                                  description: Timeout defines the maximum amount
-                                    of time Velero should wait for the hook to complete
-                                    before considering the execution a failure.
-                                  type: string
-                              required:
-                              - command
-                              type: object
-                          required:
-                          - exec
-                          type: object
-                        type: array
-                      pre:
-                        description: PreHooks is a list of BackupResourceHooks to
-                          execute prior to storing the item in the backup. These are
-                          executed before any "additional items" from item actions
-                          are processed.
-                        items:
-                          description: BackupResourceHook defines a hook for a resource.
-                          properties:
-                            exec:
-                              description: Exec defines an exec hook.
-                              properties:
-                                command:
-                                  description: Command is the command and arguments
-                                    to execute.
-                                  items:
-                                    type: string
-                                  minItems: 1
-                                  type: array
-                                container:
-                                  description: Container is the container in the pod
-                                    where the command should be executed. If not specified,
-                                    the pod's first container is used.
-                                  type: string
-                                onError:
-                                  description: OnError specifies how Velero should
-                                    behave if it encounters an error executing this
-                                    hook.
-                                  enum:
-                                  - Continue
-                                  - Fail
-                                  type: string
-                                timeout:
-                                  description: Timeout defines the maximum amount
-                                    of time Velero should wait for the hook to complete
-                                    before considering the execution a failure.
-                                  type: string
-                              required:
-                              - command
-                              type: object
-                          required:
-                          - exec
-                          type: object
-                        type: array
-                    required:
-                    - name
-                    type: object
-                  nullable: true
-                  type: array
-              type: object
-            includeClusterResources:
-              description: IncludeClusterResources specifies whether cluster-scoped
-                resources should be included for consideration in the backup.
-              nullable: true
-              type: boolean
-            includedNamespaces:
-              description: IncludedNamespaces is a slice of namespace names to include
-                objects from. If empty, all namespaces are included.
-              items:
-                type: string
-              nullable: true
-              type: array
-            includedResources:
-              description: IncludedResources is a slice of resource names to include
-                in the backup. If empty, all resources are included.
-              items:
-                type: string
-              nullable: true
-              type: array
-            labelSelector:
-              description: LabelSelector is a metav1.LabelSelector to filter with
-                when adding individual objects to the backup. If empty or nil, all
-                objects are included. Optional.
-              nullable: true
-              properties:
-                matchExpressions:
-                  description: matchExpressions is a list of label selector requirements.
-                    The requirements are ANDed.
-                  items:
-                    description: A label selector requirement is a selector that contains
-                      values, a key, and an operator that relates the key and values.
-                    properties:
-                      key:
-                        description: key is the label key that the selector applies
-                          to.
-                        type: string
-                      operator:
-                        description: operator represents a key's relationship to a
-                          set of values. Valid operators are In, NotIn, Exists and
-                          DoesNotExist.
-                        type: string
-                      values:
-                        description: values is an array of string values. If the operator
-                          is In or NotIn, the values array must be non-empty. If the
-                          operator is Exists or DoesNotExist, the values array must
-                          be empty. This array is replaced during a strategic merge
-                          patch.
-                        items:
-                          type: string
-                        type: array
-                    required:
-                    - key
-                    - operator
-                    type: object
-                  type: array
-                matchLabels:
-                  additionalProperties:
-                    type: string
-                  description: matchLabels is a map of {key,value} pairs. A single
-                    {key,value} in the matchLabels map is equivalent to an element
-                    of matchExpressions, whose key field is "key", the operator is
-                    "In", and the values array contains only "value". The requirements
-                    are ANDed.
-                  type: object
-              type: object
-            metadata:
-              properties:
-                labels:
-                  additionalProperties:
-                    type: string
-                  type: object
-              type: object
-            orderedResources:
-              additionalProperties:
-                type: string
-              description: OrderedResources specifies the backup order of resources
-                of specific Kind. The map key is the Kind name and value is a list
-                of resource names separated by commas. Each resource name has format
-                "namespace/resourcename".  For cluster resources, simply use "resourcename".
-              nullable: true
-              type: object
-            snapshotVolumes:
-              description: SnapshotVolumes specifies whether to take cloud snapshots
-                of any PV's referenced in the set of objects included in the Backup.
-              nullable: true
-              type: boolean
-            storageLocation:
-              description: StorageLocation is a string containing the name of a BackupStorageLocation
-                where the backup should be stored.
-              type: string
-            ttl:
-              description: TTL is a time.Duration-parseable string describing how
-                long the Backup should be retained for.
-              type: string
-            volumeSnapshotLocations:
-              description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations
-                associated with this backup.
-              items:
-                type: string
-              type: array
-          type: object
-        status:
-          description: BackupStatus captures the current status of a Velero backup.
-          properties:
-            completionTimestamp:
-              description: CompletionTimestamp records the time a backup was completed.
-                Completion time is recorded even on failed backups. Completion time
-                is recorded before uploading the backup object. The server's time
-                is used for CompletionTimestamps
-              format: date-time
-              nullable: true
-              type: string
-            errors:
-              description: Errors is a count of all error messages that were generated
-                during execution of the backup.  The actual errors are in the backup's
-                log file in object storage.
-              type: integer
-            expiration:
-              description: Expiration is when this Backup is eligible for garbage-collection.
-              format: date-time
-              nullable: true
-              type: string
-            formatVersion:
-              description: FormatVersion is the backup format version, including major,
-                minor, and patch version.
-              type: string
-            phase:
-              description: Phase is the current state of the Backup.
-              enum:
-              - New
-              - FailedValidation
-              - InProgress
-              - Completed
-              - PartiallyFailed
-              - Failed
-              - Deleting
-              type: string
-            progress:
-              description: Progress contains information about the backup's execution
-                progress. Note that this information is best-effort only -- if Velero
-                fails to update it during a backup for any reason, it may be inaccurate/stale.
-              nullable: true
-              properties:
-                itemsBackedUp:
-                  description: ItemsBackedUp is the number of items that have actually
-                    been written to the backup tarball so far.
-                  type: integer
-                totalItems:
-                  description: TotalItems is the total number of items to be backed
-                    up. This number may change throughout the execution of the backup
-                    due to plugins that return additional related items to back up,
-                    the velero.io/exclude-from-backup label, and various other filters
-                    that happen as items are processed.
-                  type: integer
-              type: object
-            startTimestamp:
-              description: StartTimestamp records the time a backup was started. Separate
-                from CreationTimestamp, since that value changes on restores. The
-                server's time is used for StartTimestamps
-              format: date-time
-              nullable: true
-              type: string
-            validationErrors:
-              description: ValidationErrors is a slice of all validation errors (if
-                applicable).
-              items:
-                type: string
-              nullable: true
-              type: array
-            version:
-              description: 'Version is the backup format major version. Deprecated:
-                Please see FormatVersion'
-              type: integer
-            volumeSnapshotsAttempted:
-              description: VolumeSnapshotsAttempted is the total number of attempted
-                volume snapshots for this backup.
-              type: integer
-            volumeSnapshotsCompleted:
-              description: VolumeSnapshotsCompleted is the total number of successfully
-                completed volume snapshots for this backup.
-              type: integer
-            warnings:
-              description: Warnings is a count of all warning messages that were generated
-                during execution of the backup. The actual warnings are in the backup's
-                log file in object storage.
-              type: integer
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_backupstoragelocations.yaml

@@ -1,179 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: backupstoragelocations.velero.io
-spec:
-  additionalPrinterColumns:
-  - JSONPath: .status.phase
-    description: Backup Storage Location status such as Available/Unavailable
-    name: Phase
-    type: string
-  - JSONPath: .status.lastValidationTime
-    description: LastValidationTime is the last time the backup store location was
-      validated
-    name: Last Validated
-    type: date
-  - JSONPath: .metadata.creationTimestamp
-    name: Age
-    type: date
-  - JSONPath: .spec.default
-    description: Default backup storage location
-    name: Default
-    type: boolean
-  group: velero.io
-  names:
-    kind: BackupStorageLocation
-    listKind: BackupStorageLocationList
-    plural: backupstoragelocations
-    shortNames:
-    - bsl
-    singular: backupstoragelocation
-  preserveUnknownFields: false
-  scope: Namespaced
-  subresources:
-    status: {}
-  validation:
-    openAPIV3Schema:
-      description: BackupStorageLocation is a location where Velero stores backup
-        objects
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: BackupStorageLocationSpec defines the desired state of a Velero
-            BackupStorageLocation
-          properties:
-            accessMode:
-              description: AccessMode defines the permissions for the backup storage
-                location.
-              enum:
-              - ReadOnly
-              - ReadWrite
-              type: string
-            backupSyncPeriod:
-              description: BackupSyncPeriod defines how frequently to sync backup
-                API objects from object storage. A value of 0 disables sync.
-              nullable: true
-              type: string
-            config:
-              additionalProperties:
-                type: string
-              description: Config is for provider-specific configuration fields.
-              type: object
-            credential:
-              description: Credential contains the credential information intended
-                to be used with this location
-              properties:
-                key:
-                  description: The key of the secret to select from.  Must be a valid
-                    secret key.
-                  type: string
-                name:
-                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                    TODO: Add other useful fields. apiVersion, kind, uid?'
-                  type: string
-                optional:
-                  description: Specify whether the Secret or its key must be defined
-                  type: boolean
-              required:
-              - key
-              type: object
-            default:
-              description: Default indicates this location is the default backup storage
-                location.
-              type: boolean
-            objectStorage:
-              description: ObjectStorageLocation specifies the settings necessary
-                to connect to a provider's object storage.
-              properties:
-                bucket:
-                  description: Bucket is the bucket to use for object storage.
-                  type: string
-                caCert:
-                  description: CACert defines a CA bundle to use when verifying TLS
-                    connections to the provider.
-                  format: byte
-                  type: string
-                prefix:
-                  description: Prefix is the path inside a bucket to use for Velero
-                    storage. Optional.
-                  type: string
-              required:
-              - bucket
-              type: object
-            provider:
-              description: Provider is the provider of the backup storage.
-              type: string
-            validationFrequency:
-              description: ValidationFrequency defines how frequently to validate
-                the corresponding object storage. A value of 0 disables validation.
-              nullable: true
-              type: string
-          required:
-          - objectStorage
-          - provider
-          type: object
-        status:
-          description: BackupStorageLocationStatus defines the observed state of BackupStorageLocation
-          properties:
-            accessMode:
-              description: "AccessMode is an unused field. \n Deprecated: there is
-                now an AccessMode field on the Spec and this field will be removed
-                entirely as of v2.0."
-              enum:
-              - ReadOnly
-              - ReadWrite
-              type: string
-            lastSyncedRevision:
-              description: "LastSyncedRevision is the value of the `metadata/revision`
-                file in the backup storage location the last time the BSL's contents
-                were synced into the cluster. \n Deprecated: this field is no longer
-                updated or used for detecting changes to the location's contents and
-                will be removed entirely in v2.0."
-              type: string
-            lastSyncedTime:
-              description: LastSyncedTime is the last time the contents of the location
-                were synced into the cluster.
-              format: date-time
-              nullable: true
-              type: string
-            lastValidationTime:
-              description: LastValidationTime is the last time the backup store location
-                was validated the cluster.
-              format: date-time
-              nullable: true
-              type: string
-            phase:
-              description: Phase is the current state of the BackupStorageLocation.
-              enum:
-              - Available
-              - Unavailable
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_deletebackuprequests.yaml

@@ -1,73 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: deletebackuprequests.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: DeleteBackupRequest
-    listKind: DeleteBackupRequestList
-    plural: deletebackuprequests
-    singular: deletebackuprequest
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      description: DeleteBackupRequest is a request to delete one or more backups.
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: DeleteBackupRequestSpec is the specification for which backups
-            to delete.
-          properties:
-            backupName:
-              type: string
-          required:
-          - backupName
-          type: object
-        status:
-          description: DeleteBackupRequestStatus is the current status of a DeleteBackupRequest.
-          properties:
-            errors:
-              description: Errors contains any errors that were encountered during
-                the deletion process.
-              items:
-                type: string
-              nullable: true
-              type: array
-            phase:
-              description: Phase is the current state of the DeleteBackupRequest.
-              enum:
-              - New
-              - InProgress
-              - Processed
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_downloadrequests.yaml

@@ -1,96 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: downloadrequests.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: DownloadRequest
-    listKind: DownloadRequestList
-    plural: downloadrequests
-    singular: downloadrequest
-  preserveUnknownFields: false
-  scope: Namespaced
-  subresources:
-    status: {}
-  validation:
-    openAPIV3Schema:
-      description: DownloadRequest is a request to download an artifact from backup
-        object storage, such as a backup log file.
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: DownloadRequestSpec is the specification for a download request.
-          properties:
-            target:
-              description: Target is what to download (e.g. logs for a backup).
-              properties:
-                kind:
-                  description: Kind is the type of file to download.
-                  enum:
-                  - BackupLog
-                  - BackupContents
-                  - BackupVolumeSnapshots
-                  - BackupResourceList
-                  - RestoreLog
-                  - RestoreResults
-                  type: string
-                name:
-                  description: Name is the name of the kubernetes resource with which
-                    the file is associated.
-                  type: string
-              required:
-              - kind
-              - name
-              type: object
-          required:
-          - target
-          type: object
-        status:
-          description: DownloadRequestStatus is the current status of a DownloadRequest.
-          properties:
-            downloadURL:
-              description: DownloadURL contains the pre-signed URL for the target
-                file.
-              type: string
-            expiration:
-              description: Expiration is when this DownloadRequest expires and can
-                be deleted by the system.
-              format: date-time
-              nullable: true
-              type: string
-            phase:
-              description: Phase is the current state of the DownloadRequest.
-              enum:
-              - New
-              - Processed
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_podvolumebackups.yaml

@@ -1,162 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: podvolumebackups.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: PodVolumeBackup
-    listKind: PodVolumeBackupList
-    plural: podvolumebackups
-    singular: podvolumebackup
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: PodVolumeBackupSpec is the specification for a PodVolumeBackup.
-          properties:
-            backupStorageLocation:
-              description: BackupStorageLocation is the name of the backup storage
-                location where the restic repository is stored.
-              type: string
-            node:
-              description: Node is the name of the node that the Pod is running on.
-              type: string
-            pod:
-              description: Pod is a reference to the pod containing the volume to
-                be backed up.
-              properties:
-                apiVersion:
-                  description: API version of the referent.
-                  type: string
-                fieldPath:
-                  description: 'If referring to a piece of an object instead of an
-                    entire object, this string should contain a valid JSON/Go field
-                    access statement, such as desiredState.manifest.containers[2].
-                    For example, if the object reference is to a container within
-                    a pod, this would take on a value like: "spec.containers{name}"
-                    (where "name" refers to the name of the container that triggered
-                    the event) or if no container name is specified "spec.containers[2]"
-                    (container with index 2 in this pod). This syntax is chosen only
-                    to have some well-defined way of referencing a part of an object.
-                    TODO: this design is not final and this field is subject to change
-                    in the future.'
-                  type: string
-                kind:
-                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                  type: string
-                name:
-                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
-                  type: string
-                namespace:
-                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
-                  type: string
-                resourceVersion:
-                  description: 'Specific resourceVersion to which this reference is
-                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
-                  type: string
-                uid:
-                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
-                  type: string
-              type: object
-            repoIdentifier:
-              description: RepoIdentifier is the restic repository identifier.
-              type: string
-            tags:
-              additionalProperties:
-                type: string
-              description: Tags are a map of key-value pairs that should be applied
-                to the volume backup as tags.
-              type: object
-            volume:
-              description: Volume is the name of the volume within the Pod to be backed
-                up.
-              type: string
-          required:
-          - backupStorageLocation
-          - node
-          - pod
-          - repoIdentifier
-          - volume
-          type: object
-        status:
-          description: PodVolumeBackupStatus is the current status of a PodVolumeBackup.
-          properties:
-            completionTimestamp:
-              description: CompletionTimestamp records the time a backup was completed.
-                Completion time is recorded even on failed backups. Completion time
-                is recorded before uploading the backup object. The server's time
-                is used for CompletionTimestamps
-              format: date-time
-              nullable: true
-              type: string
-            message:
-              description: Message is a message about the pod volume backup's status.
-              type: string
-            path:
-              description: Path is the full path within the controller pod being backed
-                up.
-              type: string
-            phase:
-              description: Phase is the current state of the PodVolumeBackup.
-              enum:
-              - New
-              - InProgress
-              - Completed
-              - Failed
-              type: string
-            progress:
-              description: Progress holds the total number of bytes of the volume
-                and the current number of backed up bytes. This can be used to display
-                progress information about the backup operation.
-              properties:
-                bytesDone:
-                  format: int64
-                  type: integer
-                totalBytes:
-                  format: int64
-                  type: integer
-              type: object
-            snapshotID:
-              description: SnapshotID is the identifier for the snapshot of the pod
-                volume.
-              type: string
-            startTimestamp:
-              description: StartTimestamp records the time a backup was started. Separate
-                from CreationTimestamp, since that value changes on restores. The
-                server's time is used for StartTimestamps
-              format: date-time
-              nullable: true
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_podvolumerestores.yaml

@@ -1,145 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: podvolumerestores.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: PodVolumeRestore
-    listKind: PodVolumeRestoreList
-    plural: podvolumerestores
-    singular: podvolumerestore
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore.
-          properties:
-            backupStorageLocation:
-              description: BackupStorageLocation is the name of the backup storage
-                location where the restic repository is stored.
-              type: string
-            pod:
-              description: Pod is a reference to the pod containing the volume to
-                be restored.
-              properties:
-                apiVersion:
-                  description: API version of the referent.
-                  type: string
-                fieldPath:
-                  description: 'If referring to a piece of an object instead of an
-                    entire object, this string should contain a valid JSON/Go field
-                    access statement, such as desiredState.manifest.containers[2].
-                    For example, if the object reference is to a container within
-                    a pod, this would take on a value like: "spec.containers{name}"
-                    (where "name" refers to the name of the container that triggered
-                    the event) or if no container name is specified "spec.containers[2]"
-                    (container with index 2 in this pod). This syntax is chosen only
-                    to have some well-defined way of referencing a part of an object.
-                    TODO: this design is not final and this field is subject to change
-                    in the future.'
-                  type: string
-                kind:
-                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                  type: string
-                name:
-                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
-                  type: string
-                namespace:
-                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
-                  type: string
-                resourceVersion:
-                  description: 'Specific resourceVersion to which this reference is
-                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
-                  type: string
-                uid:
-                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
-                  type: string
-              type: object
-            repoIdentifier:
-              description: RepoIdentifier is the restic repository identifier.
-              type: string
-            snapshotID:
-              description: SnapshotID is the ID of the volume snapshot to be restored.
-              type: string
-            volume:
-              description: Volume is the name of the volume within the Pod to be restored.
-              type: string
-          required:
-          - backupStorageLocation
-          - pod
-          - repoIdentifier
-          - snapshotID
-          - volume
-          type: object
-        status:
-          description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore.
-          properties:
-            completionTimestamp:
-              description: CompletionTimestamp records the time a restore was completed.
-                Completion time is recorded even on failed restores. The server's
-                time is used for CompletionTimestamps
-              format: date-time
-              nullable: true
-              type: string
-            message:
-              description: Message is a message about the pod volume restore's status.
-              type: string
-            phase:
-              description: Phase is the current state of the PodVolumeRestore.
-              enum:
-              - New
-              - InProgress
-              - Completed
-              - Failed
-              type: string
-            progress:
-              description: Progress holds the total number of bytes of the snapshot
-                and the current number of restored bytes. This can be used to display
-                progress information about the restore operation.
-              properties:
-                bytesDone:
-                  format: int64
-                  type: integer
-                totalBytes:
-                  format: int64
-                  type: integer
-              type: object
-            startTimestamp:
-              description: StartTimestamp records the time a restore was started.
-                The server's time is used for StartTimestamps
-              format: date-time
-              nullable: true
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_resticrepositories.yaml

@@ -1,89 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: resticrepositories.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: ResticRepository
-    listKind: ResticRepositoryList
-    plural: resticrepositories
-    singular: resticrepository
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: ResticRepositorySpec is the specification for a ResticRepository.
-          properties:
-            backupStorageLocation:
-              description: BackupStorageLocation is the name of the BackupStorageLocation
-                that should contain this repository.
-              type: string
-            maintenanceFrequency:
-              description: MaintenanceFrequency is how often maintenance should be
-                run.
-              type: string
-            resticIdentifier:
-              description: ResticIdentifier is the full restic-compatible string for
-                identifying this repository.
-              type: string
-            volumeNamespace:
-              description: VolumeNamespace is the namespace this restic repository
-                contains pod volume backups for.
-              type: string
-          required:
-          - backupStorageLocation
-          - maintenanceFrequency
-          - resticIdentifier
-          - volumeNamespace
-          type: object
-        status:
-          description: ResticRepositoryStatus is the current status of a ResticRepository.
-          properties:
-            lastMaintenanceTime:
-              description: LastMaintenanceTime is the last time maintenance was run.
-              format: date-time
-              nullable: true
-              type: string
-            message:
-              description: Message is a message about the current status of the ResticRepository.
-              type: string
-            phase:
-              description: Phase is the current state of the ResticRepository.
-              enum:
-              - New
-              - Ready
-              - NotReady
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_schedules.yaml

@@ -1,401 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: schedules.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: Schedule
-    listKind: ScheduleList
-    plural: schedules
-    singular: schedule
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      description: Schedule is a Velero resource that represents a pre-scheduled or
-        periodic Backup that should be run.
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: ScheduleSpec defines the specification for a Velero schedule
-          properties:
-            schedule:
-              description: Schedule is a Cron expression defining when to run the
-                Backup.
-              type: string
-            template:
-              description: Template is the definition of the Backup to be run on the
-                provided schedule
-              properties:
-                defaultVolumesToRestic:
-                  description: DefaultVolumesToRestic specifies whether restic should
-                    be used to take a backup of all pod volumes by default.
-                  type: boolean
-                excludedNamespaces:
-                  description: ExcludedNamespaces contains a list of namespaces that
-                    are not included in the backup.
-                  items:
-                    type: string
-                  nullable: true
-                  type: array
-                excludedResources:
-                  description: ExcludedResources is a slice of resource names that
-                    are not included in the backup.
-                  items:
-                    type: string
-                  nullable: true
-                  type: array
-                hooks:
-                  description: Hooks represent custom behaviors that should be executed
-                    at different phases of the backup.
-                  properties:
-                    resources:
-                      description: Resources are hooks that should be executed when
-                        backing up individual instances of a resource.
-                      items:
-                        description: BackupResourceHookSpec defines one or more BackupResourceHooks
-                          that should be executed based on the rules defined for namespaces,
-                          resources, and label selector.
-                        properties:
-                          excludedNamespaces:
-                            description: ExcludedNamespaces specifies the namespaces
-                              to which this hook spec does not apply.
-                            items:
-                              type: string
-                            nullable: true
-                            type: array
-                          excludedResources:
-                            description: ExcludedResources specifies the resources
-                              to which this hook spec does not apply.
-                            items:
-                              type: string
-                            nullable: true
-                            type: array
-                          includedNamespaces:
-                            description: IncludedNamespaces specifies the namespaces
-                              to which this hook spec applies. If empty, it applies
-                              to all namespaces.
-                            items:
-                              type: string
-                            nullable: true
-                            type: array
-                          includedResources:
-                            description: IncludedResources specifies the resources
-                              to which this hook spec applies. If empty, it applies
-                              to all resources.
-                            items:
-                              type: string
-                            nullable: true
-                            type: array
-                          labelSelector:
-                            description: LabelSelector, if specified, filters the
-                              resources to which this hook spec applies.
-                            nullable: true
-                            properties:
-                              matchExpressions:
-                                description: matchExpressions is a list of label selector
-                                  requirements. The requirements are ANDed.
-                                items:
-                                  description: A label selector requirement is a selector
-                                    that contains values, a key, and an operator that
-                                    relates the key and values.
-                                  properties:
-                                    key:
-                                      description: key is the label key that the selector
-                                        applies to.
-                                      type: string
-                                    operator:
-                                      description: operator represents a key's relationship
-                                        to a set of values. Valid operators are In,
-                                        NotIn, Exists and DoesNotExist.
-                                      type: string
-                                    values:
-                                      description: values is an array of string values.
-                                        If the operator is In or NotIn, the values
-                                        array must be non-empty. If the operator is
-                                        Exists or DoesNotExist, the values array must
-                                        be empty. This array is replaced during a
-                                        strategic merge patch.
-                                      items:
-                                        type: string
-                                      type: array
-                                  required:
-                                  - key
-                                  - operator
-                                  type: object
-                                type: array
-                              matchLabels:
-                                additionalProperties:
-                                  type: string
-                                description: matchLabels is a map of {key,value} pairs.
-                                  A single {key,value} in the matchLabels map is equivalent
-                                  to an element of matchExpressions, whose key field
-                                  is "key", the operator is "In", and the values array
-                                  contains only "value". The requirements are ANDed.
-                                type: object
-                            type: object
-                          name:
-                            description: Name is the name of this hook.
-                            type: string
-                          post:
-                            description: PostHooks is a list of BackupResourceHooks
-                              to execute after storing the item in the backup. These
-                              are executed after all "additional items" from item
-                              actions are processed.
-                            items:
-                              description: BackupResourceHook defines a hook for a
-                                resource.
-                              properties:
-                                exec:
-                                  description: Exec defines an exec hook.
-                                  properties:
-                                    command:
-                                      description: Command is the command and arguments
-                                        to execute.
-                                      items:
-                                        type: string
-                                      minItems: 1
-                                      type: array
-                                    container:
-                                      description: Container is the container in the
-                                        pod where the command should be executed.
-                                        If not specified, the pod's first container
-                                        is used.
-                                      type: string
-                                    onError:
-                                      description: OnError specifies how Velero should
-                                        behave if it encounters an error executing
-                                        this hook.
-                                      enum:
-                                      - Continue
-                                      - Fail
-                                      type: string
-                                    timeout:
-                                      description: Timeout defines the maximum amount
-                                        of time Velero should wait for the hook to
-                                        complete before considering the execution
-                                        a failure.
-                                      type: string
-                                  required:
-                                  - command
-                                  type: object
-                              required:
-                              - exec
-                              type: object
-                            type: array
-                          pre:
-                            description: PreHooks is a list of BackupResourceHooks
-                              to execute prior to storing the item in the backup.
-                              These are executed before any "additional items" from
-                              item actions are processed.
-                            items:
-                              description: BackupResourceHook defines a hook for a
-                                resource.
-                              properties:
-                                exec:
-                                  description: Exec defines an exec hook.
-                                  properties:
-                                    command:
-                                      description: Command is the command and arguments
-                                        to execute.
-                                      items:
-                                        type: string
-                                      minItems: 1
-                                      type: array
-                                    container:
-                                      description: Container is the container in the
-                                        pod where the command should be executed.
-                                        If not specified, the pod's first container
-                                        is used.
-                                      type: string
-                                    onError:
-                                      description: OnError specifies how Velero should
-                                        behave if it encounters an error executing
-                                        this hook.
-                                      enum:
-                                      - Continue
-                                      - Fail
-                                      type: string
-                                    timeout:
-                                      description: Timeout defines the maximum amount
-                                        of time Velero should wait for the hook to
-                                        complete before considering the execution
-                                        a failure.
-                                      type: string
-                                  required:
-                                  - command
-                                  type: object
-                              required:
-                              - exec
-                              type: object
-                            type: array
-                        required:
-                        - name
-                        type: object
-                      nullable: true
-                      type: array
-                  type: object
-                includeClusterResources:
-                  description: IncludeClusterResources specifies whether cluster-scoped
-                    resources should be included for consideration in the backup.
-                  nullable: true
-                  type: boolean
-                includedNamespaces:
-                  description: IncludedNamespaces is a slice of namespace names to
-                    include objects from. If empty, all namespaces are included.
-                  items:
-                    type: string
-                  nullable: true
-                  type: array
-                includedResources:
-                  description: IncludedResources is a slice of resource names to include
-                    in the backup. If empty, all resources are included.
-                  items:
-                    type: string
-                  nullable: true
-                  type: array
-                labelSelector:
-                  description: LabelSelector is a metav1.LabelSelector to filter with
-                    when adding individual objects to the backup. If empty or nil,
-                    all objects are included. Optional.
-                  nullable: true
-                  properties:
-                    matchExpressions:
-                      description: matchExpressions is a list of label selector requirements.
-                        The requirements are ANDed.
-                      items:
-                        description: A label selector requirement is a selector that
-                          contains values, a key, and an operator that relates the
-                          key and values.
-                        properties:
-                          key:
-                            description: key is the label key that the selector applies
-                              to.
-                            type: string
-                          operator:
-                            description: operator represents a key's relationship
-                              to a set of values. Valid operators are In, NotIn, Exists
-                              and DoesNotExist.
-                            type: string
-                          values:
-                            description: values is an array of string values. If the
-                              operator is In or NotIn, the values array must be non-empty.
-                              If the operator is Exists or DoesNotExist, the values
-                              array must be empty. This array is replaced during a
-                              strategic merge patch.
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - key
-                        - operator
-                        type: object
-                      type: array
-                    matchLabels:
-                      additionalProperties:
-                        type: string
-                      description: matchLabels is a map of {key,value} pairs. A single
-                        {key,value} in the matchLabels map is equivalent to an element
-                        of matchExpressions, whose key field is "key", the operator
-                        is "In", and the values array contains only "value". The requirements
-                        are ANDed.
-                      type: object
-                  type: object
-                metadata:
-                  properties:
-                    labels:
-                      additionalProperties:
-                        type: string
-                      type: object
-                  type: object
-                orderedResources:
-                  additionalProperties:
-                    type: string
-                  description: OrderedResources specifies the backup order of resources
-                    of specific Kind. The map key is the Kind name and value is a
-                    list of resource names separated by commas. Each resource name
-                    has format "namespace/resourcename".  For cluster resources, simply
-                    use "resourcename".
-                  nullable: true
-                  type: object
-                snapshotVolumes:
-                  description: SnapshotVolumes specifies whether to take cloud snapshots
-                    of any PV's referenced in the set of objects included in the Backup.
-                  nullable: true
-                  type: boolean
-                storageLocation:
-                  description: StorageLocation is a string containing the name of
-                    a BackupStorageLocation where the backup should be stored.
-                  type: string
-                ttl:
-                  description: TTL is a time.Duration-parseable string describing
-                    how long the Backup should be retained for.
-                  type: string
-                volumeSnapshotLocations:
-                  description: VolumeSnapshotLocations is a list containing names
-                    of VolumeSnapshotLocations associated with this backup.
-                  items:
-                    type: string
-                  type: array
-              type: object
-            useOwnerReferencesInBackup:
-              description: UseOwnerReferencesBackup specifies whether to use OwnerReferences
-                on backups created by this Schedule.
-              nullable: true
-              type: boolean
-          required:
-          - schedule
-          - template
-          type: object
-        status:
-          description: ScheduleStatus captures the current state of a Velero schedule
-          properties:
-            lastBackup:
-              description: LastBackup is the last time a Backup was run for this Schedule
-                schedule
-              format: date-time
-              nullable: true
-              type: string
-            phase:
-              description: Phase is the current phase of the Schedule
-              enum:
-              - New
-              - Enabled
-              - FailedValidation
-              type: string
-            validationErrors:
-              description: ValidationErrors is a slice of all validation errors (if
-                applicable)
-              items:
-                type: string
-              type: array
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_serverstatusrequests.yaml

@@ -1,89 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: serverstatusrequests.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: ServerStatusRequest
-    listKind: ServerStatusRequestList
-    plural: serverstatusrequests
-    shortNames:
-    - ssr
-    singular: serverstatusrequest
-  preserveUnknownFields: false
-  scope: Namespaced
-  subresources:
-    status: {}
-  validation:
-    openAPIV3Schema:
-      description: ServerStatusRequest is a request to access current status information
-        about the Velero server.
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: ServerStatusRequestSpec is the specification for a ServerStatusRequest.
-          type: object
-        status:
-          description: ServerStatusRequestStatus is the current status of a ServerStatusRequest.
-          properties:
-            phase:
-              description: Phase is the current lifecycle phase of the ServerStatusRequest.
-              enum:
-              - New
-              - Processed
-              type: string
-            plugins:
-              description: Plugins list information about the plugins running on the
-                Velero server
-              items:
-                description: PluginInfo contains attributes of a Velero plugin
-                properties:
-                  kind:
-                    type: string
-                  name:
-                    type: string
-                required:
-                - kind
-                - name
-                type: object
-              nullable: true
-              type: array
-            processedTimestamp:
-              description: ProcessedTimestamp is when the ServerStatusRequest was
-                processed by the ServerStatusRequestController.
-              format: date-time
-              nullable: true
-              type: string
-            serverVersion:
-              description: ServerVersion is the Velero server version.
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/bases/velero.io_volumesnapshotlocations.yaml

@@ -1,74 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
-  creationTimestamp: null
-  name: volumesnapshotlocations.velero.io
-spec:
-  group: velero.io
-  names:
-    kind: VolumeSnapshotLocation
-    listKind: VolumeSnapshotLocationList
-    plural: volumesnapshotlocations
-    singular: volumesnapshotlocation
-  preserveUnknownFields: false
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      description: VolumeSnapshotLocation is a location where Velero stores volume
-        snapshots.
-      properties:
-        apiVersion:
-          description: 'APIVersion defines the versioned schema of this representation
-            of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          type: string
-        kind:
-          description: 'Kind is a string value representing the REST resource this
-            object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-          type: string
-        metadata:
-          type: object
-        spec:
-          description: VolumeSnapshotLocationSpec defines the specification for a
-            Velero VolumeSnapshotLocation.
-          properties:
-            config:
-              additionalProperties:
-                type: string
-              description: Config is for provider-specific configuration fields.
-              type: object
-            provider:
-              description: Provider is the provider of the volume storage.
-              type: string
-          required:
-          - provider
-          type: object
-        status:
-          description: VolumeSnapshotLocationStatus describes the current status of
-            a Velero VolumeSnapshotLocation.
-          properties:
-            phase:
-              description: VolumeSnapshotLocationPhase is the lifecycle phase of a
-                Velero VolumeSnapshotLocation.
-              enum:
-              - Available
-              - Unavailable
-              type: string
-          type: object
-      type: object
-  version: v1
-  versions:
-  - name: v1
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/v1beta1/crds/doc.go

@@ -1,4 +0,0 @@
-// Package crds embeds the controller-tools generated CRD manifests
-package crds
-
-//go:generate go run ../../../../hack/crd-gen/v1beta1/main.go

design/2082-bsl-delete-associated-resources_design.md

@@ -0,0 +1,40 @@
+# Delete Backup and Restic Repo Resources when BSL is Deleted
+
+## Abstract
+
+Issue #2082 requested that with the command `velero backup-location delete <bsl name>` (implemented in Velero 1.6 with #3073), the following will be deleted:
+
+- associated Velero backups (to be clear, these are custom Kubernetes resources called "backups" that are stored in the API server)
+- associated Restic repositories (custom Kubernetes resources called "resticrepositories")
+
+This design doc explains how the request will be implemented.
+
+## Background
+
+When a BSL resource is deleted from its Velero namespace, the associated custom Kubernetes resources, backups and Restic repositories, can no longer be used.
+It makes sense to clean those resources up when a BSL is deleted.
+
+## Goals
+
+Update the `velero backup-location delete <bsl name>` command to delete associated backup and Restic repository resources in the same Velero namespace.
+
+## Non Goals
+
+[It was suggested](https://github.com/vmware-tanzu/velero/issues/2082#issuecomment-827951311) to fix bug #2697 alongside this issue.
+However, I think that should be fixed separately because although it is similar (restore objects are not being deleted), it is also quite different.
+One is adding a command feature update (this issue) and the other is a bug fix and each affect different parts of the code base.
+
+## High-Level Design
+
+Update the `velero backup-location delete <bsl name>` command to do the following:
+
+- find in the same Velero namespace from which the BSL was deleted the associated backup resources and Restic repositories, called "backups.velero.io" and "resticrepositories.velero.io" respectively
+- delete the resources found
+
+The above logic will be added to [where BSLs are deleted](https://github.com/vmware-tanzu/velero/blob/main/pkg/cmd/cli/backuplocation/delete.go).
+
+## Alternative Considered
+
+I had considered deleting the backup files (the ones in json format and tarballs) in the BSL itself.
+However, a standard use case is to back up a cluster and then restore into a new cluster.
+Deleting the backup storage location in either location is not expected to remove all of the backups in the backup storage location and should not be done.

design/Implemented/restore-with-EnableAPIGroupVersions-feature.md

@@ -1,6 +1,6 @@
 # Restore API Group Version by Priority Level When EnableAPIGroupVersions Feature is Set
 
-Status: Draft
+Status: Accepted
 
 ## Abstract
 

go.mod

@@ -1,25 +1,28 @@
 module github.com/vmware-tanzu/velero
 
-go 1.16
+go 1.17
 
 require (
+	cloud.google.com/go/storage v1.10.0
+	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
+	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/Azure/go-autorest/autorest v0.11.21
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
 	github.com/Azure/go-autorest/autorest/to v0.3.0
-	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
 	github.com/aws/aws-sdk-go v1.28.2
+	github.com/bombsimon/logrusr v1.1.0
 	github.com/evanphx/json-patch v4.11.0+incompatible
 	github.com/fatih/color v1.13.0
 	github.com/gobwas/glob v0.2.3
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.5.2
-	github.com/google/uuid v1.1.2
+	github.com/google/uuid v1.2.0
 	github.com/hashicorp/go-hclog v0.12.0
 	github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
 	github.com/joho/godotenv v1.3.0
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
-	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.16.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
@@ -32,6 +35,7 @@ require (
 	github.com/vmware-tanzu/crash-diagnostics v0.3.7
 	golang.org/x/mod v0.4.2
 	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
+	google.golang.org/api v0.56.0
 	google.golang.org/grpc v1.40.0
 	k8s.io/api v0.22.2
 	k8s.io/apiextensions-apiserver v0.22.2
@@ -44,4 +48,79 @@ require (
 	sigs.k8s.io/controller-runtime v0.10.2
 )
 
+require (
+	cloud.google.com/go v0.93.3 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.14 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-logr/zapr v0.4.0 // indirect
+	github.com/gobuffalo/flect v0.2.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mattn/go-colorable v0.1.9 // indirect
+	github.com/mattn/go-ieproxy v0.0.1 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/oklog/run v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/vladimirvivien/gexe v0.1.1 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.19.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/component-base v0.22.2 // indirect
+	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
 replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2

go.sum

@@ -43,10 +43,15 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
+github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM=
+github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
@@ -128,6 +133,8 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bombsimon/logrusr v1.1.0 h1:Y03FI4Z/Shyrc9jF26vuaUbnPxC5NMJnTtJA/3Lihq8=
+github.com/bombsimon/logrusr v1.1.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -318,9 +325,11 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
 github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -340,10 +349,12 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
@@ -427,11 +438,14 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -455,6 +469,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
+github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -502,7 +518,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -517,8 +532,9 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -663,6 +679,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
@@ -767,6 +784,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -851,6 +869,7 @@ golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -874,6 +893,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1024,6 +1044,7 @@ google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtuk
 google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
 google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
 google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
+google.golang.org/api v0.56.0 h1:08F9XVYTLOGeSQb3xI9C0gXMuQanhdGed0cWFhDozbI=
 google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1135,8 +1156,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

hack/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.16
+FROM golang:1.17.8
 
 ARG GOPROXY
 
@@ -24,17 +24,19 @@ ENV GOPROXY=${GOPROXY}
 RUN mkdir -p /go/src/k8s.io
 WORKDIR /go/src/k8s.io
 RUN git config --global advice.detachedHead false
-RUN git clone -b v0.18.4 https://github.com/kubernetes/code-generator
+RUN git clone -b v0.22.2 https://github.com/kubernetes/code-generator
 
-RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz && \
-    tar -zxvf kubebuilder_2.3.1_linux_amd64.tar.gz && \
-    mv kubebuilder_2.3.1_linux_amd64 /usr/local/kubebuilder && \
-    chmod +x /usr/local/kubebuilder && \
-    export PATH=$PATH:/usr/local/kubebuilder/bin && \
-    rm kubebuilder_2.3.1_linux_amd64.tar.gz
+# kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test.
+RUN curl -sSLo envtest-bins.tar.gz https://go.kubebuilder.io/test-tools/1.22.1/linux/amd64 && \
+    mkdir /usr/local/kubebuilder && \
+    tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz
+
+RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_amd64 && \
+    mv kubebuilder_linux_amd64 /usr/local/kubebuilder/bin/kubebuilder && \
+    chmod +x /usr/local/kubebuilder/bin/kubebuilder
 
 # get controller-tools
-RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0
+RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0
 
 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit
 # is not important)

hack/crd-gen/v1beta1/main.go

@@ -1,136 +0,0 @@
-/*
-Copyright the Velero contributors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// This code embeds the CRD manifests in config/crd/v1beta1/bases in
-// config/crd/v1beta1/crds/crds.go.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"os"
-	"text/template"
-)
-
-// This is relative to config/crd/crds
-const goHeaderFile = "../../../../hack/boilerplate.go.txt"
-
-const tpl = `{{.GoHeader}}
-// Code generated by crds_generate.go; DO NOT EDIT.
-
-package crds
-
-import (
-	"bytes"
-	"compress/gzip"
-	"io/ioutil"
-
-	apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
-	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	"k8s.io/client-go/kubernetes/scheme"
-)
-
-var rawCRDs = [][]byte{
-{{- range .RawCRDs }}
-	[]byte({{ . }}),
-{{- end }}
-}
-
-var CRDs = crds()
-
-func crds() []*apiextv1beta1.CustomResourceDefinition {
-	apiextinstall.Install(scheme.Scheme)
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-	var objs []*apiextv1beta1.CustomResourceDefinition
-	for _, crd := range rawCRDs {
-		gzr, err := gzip.NewReader(bytes.NewReader(crd))
-		if err != nil {
-			panic(err)
-		}
-		bytes, err := ioutil.ReadAll(gzr)
-		if err != nil {
-			panic(err)
-		}
-		gzr.Close()
-
-		obj, _, err := decode(bytes, nil, nil)
-		if err != nil {
-			panic(err)
-		}
-		objs = append(objs, obj.(*apiextv1beta1.CustomResourceDefinition))
-	}
-	return objs
-}
-`
-
-type templateData struct {
-	GoHeader string
-	RawCRDs  []string
-}
-
-func main() {
-	headerBytes, err := ioutil.ReadFile(goHeaderFile)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	data := templateData{
-		GoHeader: string(headerBytes),
-	}
-
-	// This is relative to config/crd/crds
-	manifests, err := ioutil.ReadDir("../bases")
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	for _, crd := range manifests {
-		file, err := os.Open("../bases/" + crd.Name())
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		// gzip compress manifest
-		var buf bytes.Buffer
-		gzw := gzip.NewWriter(&buf)
-		if _, err := io.Copy(gzw, file); err != nil {
-			log.Fatalln(err)
-		}
-		file.Close()
-		gzw.Close()
-
-		data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes()))
-	}
-
-	t, err := template.New("crd").Parse(tpl)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	out, err := os.Create("crds.go")
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if err := t.Execute(out, data); err != nil {
-		log.Fatalln(err)
-	}
-}

hack/docker-push.sh

@@ -56,26 +56,24 @@ elif [[ "$triggeredBy" == "tags" ]]; then
     TAG=$(echo $GITHUB_REF | cut -d / -f 3)
 fi
 
-if [[ "$BRANCH" == "main" ]]; then
-    VERSION="$BRANCH"
-elif [[ ! -z "$TAG" ]]; then
+TAG_LATEST=false
+if [[ ! -z "$TAG" ]]; then
+    echo "We're building tag $TAG"
+    VERSION="$TAG"
     # Explicitly checkout tags when building from a git tag.
     # This is not needed when building from main
     git fetch --tags
     # Calculate the latest release if there's a tag.
     highest_release
-    VERSION="$TAG"
+    if [[ "$TAG" == "$HIGHEST" ]]; then
+      TAG_LATEST=true
+    fi
 else
-    echo "We're not on main and we're not building a tag, exit early."
-    exit 0
-fi
-
-# Assume we're not tagging `latest` by default, and never on main.
-TAG_LATEST=false
-if [[ "$BRANCH" == "main" ]]; then
-    echo "Building main, not tagging latest."
-elif [[ "$TAG" == "$HIGHEST" ]]; then
-    TAG_LATEST=true
+    echo "We're on branch $BRANCH"
+    VERSION="$BRANCH"
+    if [[ "$VERSION" == release-* ]]; then
+      VERSION=${VERSION}-dev
+    fi
 fi
 
 if [[ -z "$BUILDX_PLATFORMS" ]]; then
@@ -87,6 +85,7 @@ echo "Highest tag found: $HIGHEST"
 echo "BRANCH: $BRANCH"
 echo "TAG: $TAG"
 echo "TAG_LATEST: $TAG_LATEST"
+echo "VERSION: $VERSION"
 echo "BUILDX_PLATFORMS: $BUILDX_PLATFORMS"
 
 echo "Building and pushing container images."

hack/update-generated-crd-code.sh

@@ -44,24 +44,21 @@ ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \
   --output-base ../../.. \
   $@
 
-# Generate both apiextensions.k8s.io/v1beta1 and apiextensions.k8s.io/v1
-for version in v1beta1 v1
-do
-  # Generate manifests e.g. CRD, RBAC etc.
-  controller-gen \
-    crd:crdVersions=$version,preserveUnknownFields=false,trivialVersions=true \
-    paths=./pkg/apis/velero/v1/... \
-    paths=./pkg/controller/... \
-    output:crd:artifacts:config=config/crd/$version/bases
+# Generate apiextensions.k8s.io/v1
+# Generate manifests e.g. CRD, RBAC etc.
+controller-gen \
+  crd:crdVersions=v1\
+  paths=./pkg/apis/velero/v1/... \
+  paths=./pkg/controller/... \
+  output:crd:artifacts:config=config/crd/v1/bases
 
-  # this is a super hacky workaround for https://github.com/kubernetes/kubernetes/issues/91395
-  # which a result of fixing the validation on CRD objects. The validation ensures the fields that are list map keys, are either marked
-  # as required or have default values to ensure merging of list map items work as expected.
-  # With "containerPort" and "protocol" being considered as x-kubernetes-list-map-keys in the container ports, and "protocol" was not
-  # a required field, the CRD would fail validation with errors similar to the one reported in https://github.com/kubernetes/kubernetes/issues/91395.
-  # once controller-gen (above) is able to generate CRDs with `protocol` as a required field, this hack can be removed.
-  kubectl patch -f config/crd/$version/bases/velero.io_restores.yaml -p "$(cat hack/restore-crd-patch-$version.json)" --type=json --local=true -o yaml > /tmp/velero.io_restores-yaml.patched
-  mv /tmp/velero.io_restores-yaml.patched config/crd/$version/bases/velero.io_restores.yaml
+# this is a super hacky workaround for https://github.com/kubernetes/kubernetes/issues/91395
+# which a result of fixing the validation on CRD objects. The validation ensures the fields that are list map keys, are either marked
+# as required or have default values to ensure merging of list map items work as expected.
+# With "containerPort" and "protocol" being considered as x-kubernetes-list-map-keys in the container ports, and "protocol" was not
+# a required field, the CRD would fail validation with errors similar to the one reported in https://github.com/kubernetes/kubernetes/issues/91395.
+# once controller-gen (above) is able to generate CRDs with `protocol` as a required field, this hack can be removed.
+kubectl patch -f config/crd/v1/bases/velero.io_restores.yaml -p "$(cat hack/restore-crd-patch-v1.json)" --type=json --local=true -o yaml > /tmp/velero.io_restores-yaml.patched
+mv /tmp/velero.io_restores-yaml.patched config/crd/v1/bases/velero.io_restores.yaml
 
-  go generate ./config/crd/$version/crds
-done
+go generate ./config/crd/v1/crds

hack/verify-generated-crd-code.sh

@@ -19,14 +19,11 @@ HACK_DIR=$(dirname "${BASH_SOURCE}")
 ${HACK_DIR}/update-generated-crd-code.sh --verify-only
 
 # ensure no changes to generated CRDs
-for version in v1beta1 v1
-do
-  if ! git diff --exit-code config/crd/$version/crds/crds.go >/dev/null; then
-    # revert changes to state before running CRD generation to stay consistent
-    # with code-generator `--verify-only` option which discards generated changes
-    git checkout config/crd
+if ! git diff --exit-code config/crd/v1/crds/crds.go >/dev/null; then
+  # revert changes to state before running CRD generation to stay consistent
+  # with code-generator `--verify-only` option which discards generated changes
+  git checkout config/crd
 
-    echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)."
-    exit 1
-  fi
-done
+  echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)."
+  exit 1
+fi

internal/delete/delete_item_action_handler.go

@@ -19,6 +19,8 @@ package delete
 import (
 	"io"
 
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/labels"
@@ -29,7 +31,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/archive"
 	"github.com/vmware-tanzu/velero/pkg/discovery"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 )
 
@@ -41,14 +42,13 @@ type Context struct {
 	Filesystem      filesystem.Interface
 	Log             logrus.FieldLogger
 	DiscoveryHelper discovery.Helper
-
-	resolvedActions []resolvedAction
+	resolvedActions []framework.DeleteItemResolvedAction
 }
 
 func InvokeDeleteActions(ctx *Context) error {
 	var err error
-	ctx.resolvedActions, err = resolveActions(ctx.Actions, ctx.DiscoveryHelper)
-
+	resolver := framework.NewDeleteItemActionResolver(ctx.Actions)
+	ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper)
 	// No actions installed and no error means we don't have to continue;
 	// just do the backup deletion without worrying about plugins.
 	if len(ctx.resolvedActions) == 0 && err == nil {
@@ -74,15 +74,8 @@ func InvokeDeleteActions(ctx *Context) error {
 
 	processdResources := sets.NewString()
 
-	ctx.Log.Debugf("Trying to reconcile resource names with Kube API server.")
-	// Transform resource names based on what's canonical in the API server.
 	for resource := range backupResources {
-		gvr, _, err := ctx.DiscoveryHelper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
-		if err != nil {
-			return errors.Wrapf(err, "failed to resolve resource into complete group/version/resource: %v", resource)
-		}
-
-		groupResource := gvr.GroupResource()
+		groupResource := schema.ParseGroupResource(resource)
 
 		// We've already seen this group/resource, so don't process it again.
 		if processdResources.Has(groupResource.String()) {
@@ -92,8 +85,6 @@ func InvokeDeleteActions(ctx *Context) error {
 		// Get a list of all items that exist for this resource
 		resourceList := backupResources[groupResource.String()]
 		if resourceList == nil {
-			// After canonicalization from the API server, the resources may not exist in the tarball
-			// Skip them if that's the case.
 			continue
 		}
 
@@ -119,10 +110,10 @@ func InvokeDeleteActions(ctx *Context) error {
 				itemLog.Infof("invoking DeleteItemAction plugins")
 
 				for _, action := range actions {
-					if !action.selector.Matches(labels.Set(obj.GetLabels())) {
+					if !action.Selector.Matches(labels.Set(obj.GetLabels())) {
 						continue
 					}
-					err = action.Execute(&velero.DeleteItemActionExecuteInput{
+					err = action.DeleteItemAction.Execute(&velero.DeleteItemActionExecuteInput{
 						Item:   obj,
 						Backup: ctx.Backup,
 					})
@@ -139,65 +130,12 @@ func InvokeDeleteActions(ctx *Context) error {
 }
 
 // getApplicableActions takes resolved DeleteItemActions and filters them for a given group/resource and namespace.
-func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []resolvedAction {
-	var actions []resolvedAction
-
+func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.DeleteItemResolvedAction {
+	var actions []framework.DeleteItemResolvedAction
 	for _, action := range ctx.resolvedActions {
-		if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) {
-			continue
+		if action.ShouldUse(groupResource, namespace, nil, ctx.Log) {
+			actions = append(actions, action)
 		}
-
-		if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) {
-			continue
-		}
-
-		if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() {
-			continue
-		}
-
-		actions = append(actions, action)
 	}
-
 	return actions
 }
-
-// resolvedActions are DeleteItemActions decorated with resource/namespace include/exclude collections, as well as label selectors for easy comparison.
-type resolvedAction struct {
-	velero.DeleteItemAction
-
-	resourceIncludesExcludes  *collections.IncludesExcludes
-	namespaceIncludesExcludes *collections.IncludesExcludes
-	selector                  labels.Selector
-}
-
-// resolveActions resolves the AppliesTo ResourceSelectors of DeleteItemActions plugins against the Kubernetes discovery API for fully-qualified names.
-func resolveActions(actions []velero.DeleteItemAction, helper discovery.Helper) ([]resolvedAction, error) {
-	var resolved []resolvedAction
-
-	for _, action := range actions {
-		resourceSelector, err := action.AppliesTo()
-		if err != nil {
-			return nil, err
-		}
-
-		resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources)
-		namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...)
-
-		selector := labels.Everything()
-		if resourceSelector.LabelSelector != "" {
-			if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil {
-				return nil, err
-			}
-		}
-
-		res := resolvedAction{
-			DeleteItemAction:          action,
-			resourceIncludesExcludes:  resources,
-			namespaceIncludesExcludes: namespaces,
-			selector:                  selector,
-		}
-		resolved = append(resolved, res)
-	}
-
-	return resolved, nil
-}

pkg/apis/velero/v1/constants.go

@@ -46,4 +46,8 @@ const (
 
 	// APIGroupVersionsFeatureFlag is the feature flag string that defines whether or not to handle multiple API Group Versions
 	APIGroupVersionsFeatureFlag = "EnableAPIGroupVersions"
+
+	// UploadProgressFeatureFlag is the feature flag string that defines whether or not upload progress monitoring is enabled
+	// and whether or not ItemSnapshotters should be invoked
+	UploadProgressFeatureFlag = "EnableUploadProgress"
 )

pkg/apis/velero/v1/download_request_types.go

@@ -25,13 +25,14 @@ type DownloadRequestSpec struct {
 }
 
 // DownloadTargetKind represents what type of file to download.
-// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupResourceList;RestoreLog;RestoreResults
+// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemSnapshots;BackupResourceList;RestoreLog;RestoreResults
 type DownloadTargetKind string
 
 const (
 	DownloadTargetKindBackupLog             DownloadTargetKind = "BackupLog"
 	DownloadTargetKindBackupContents        DownloadTargetKind = "BackupContents"
 	DownloadTargetKindBackupVolumeSnapshots DownloadTargetKind = "BackupVolumeSnapshots"
+	DownloadTargetKindBackupItemSnapshots   DownloadTargetKind = "BackupItemSnapshots"
 	DownloadTargetKindBackupResourceList    DownloadTargetKind = "BackupResourceList"
 	DownloadTargetKindRestoreLog            DownloadTargetKind = "RestoreLog"
 	DownloadTargetKindRestoreResults        DownloadTargetKind = "RestoreResults"

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*

pkg/backup/backup.go

@@ -33,7 +33,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
@@ -44,9 +43,11 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/discovery"
 	velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/podexec"
 	"github.com/vmware-tanzu/velero/pkg/restic"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 )
 
@@ -62,6 +63,9 @@ type Backupper interface {
 	// Backup takes a backup using the specification in the velerov1api.Backup and writes backup and log data
 	// to the given writers.
 	Backup(logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error
+	BackupWithResolvers(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer,
+		backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver,
+		volumeSnapshotterGetter VolumeSnapshotterGetter) error
 }
 
 // kubernetesBackupper implements Backupper.
@@ -76,14 +80,6 @@ type kubernetesBackupper struct {
 	clientPageSize         int
 }
 
-type resolvedAction struct {
-	velero.BackupItemAction
-
-	resourceIncludesExcludes  *collections.IncludesExcludes
-	namespaceIncludesExcludes *collections.IncludesExcludes
-	selector                  labels.Selector
-}
-
 func (i *itemKey) String() string {
 	return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name)
 }
@@ -121,38 +117,6 @@ func NewKubernetesBackupper(
 	}, nil
 }
 
-func resolveActions(actions []velero.BackupItemAction, helper discovery.Helper) ([]resolvedAction, error) {
-	var resolved []resolvedAction
-
-	for _, action := range actions {
-		resourceSelector, err := action.AppliesTo()
-		if err != nil {
-			return nil, err
-		}
-
-		resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources)
-		namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...)
-
-		selector := labels.Everything()
-		if resourceSelector.LabelSelector != "" {
-			if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil {
-				return nil, err
-			}
-		}
-
-		res := resolvedAction{
-			BackupItemAction:          action,
-			resourceIncludesExcludes:  resources,
-			namespaceIncludesExcludes: namespaces,
-			selector:                  selector,
-		}
-
-		resolved = append(resolved, res)
-	}
-
-	return resolved, nil
-}
-
 // getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
 // include and exclude from the backup.
 func getNamespaceIncludesExcludes(backup *velerov1api.Backup) *collections.IncludesExcludes {
@@ -205,7 +169,20 @@ type VolumeSnapshotterGetter interface {
 // a complete backup failure is returned. Errors that constitute partial failures (i.e. failures to
 // back up individual resources that don't prevent the backup from continuing to be processed) are logged
 // to the backup log.
-func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error {
+func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer,
+	actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error {
+	backupItemActions := framework.NewBackupItemActionResolver(actions)
+	itemSnapshotters := framework.NewItemSnapshotterResolver(nil)
+	return kb.BackupWithResolvers(log, backupRequest, backupFile, backupItemActions, itemSnapshotters,
+		volumeSnapshotterGetter)
+}
+
+func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
+	backupRequest *Request,
+	backupFile io.Writer,
+	backupItemActionResolver framework.BackupItemActionResolver,
+	itemSnapshotterResolver framework.ItemSnapshotterResolver,
+	volumeSnapshotterGetter VolumeSnapshotterGetter) error {
 	gzippedData := gzip.NewWriter(backupFile)
 	defer gzippedData.Close()
 
@@ -224,7 +201,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 	backupRequest.ResourceIncludesExcludes = collections.GetResourceIncludesExcludes(kb.discoveryHelper, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources)
 	log.Infof("Including resources: %s", backupRequest.ResourceIncludesExcludes.IncludesString())
 	log.Infof("Excluding resources: %s", backupRequest.ResourceIncludesExcludes.ExcludesString())
-	log.Infof("Backing up all pod volumes using restic: %t", *backupRequest.Backup.Spec.DefaultVolumesToRestic)
+	log.Infof("Backing up all pod volumes using restic: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToRestic))
 
 	var err error
 	backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper)
@@ -232,7 +209,12 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 		return err
 	}
 
-	backupRequest.ResolvedActions, err = resolveActions(actions, kb.discoveryHelper)
+	backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper)
+	if err != nil {
+		return err
+	}
+
+	backupRequest.ResolvedItemSnapshotters, err = itemSnapshotterResolver.ResolveActions(kb.discoveryHelper)
 	if err != nil {
 		return err
 	}

pkg/backup/item_backupper.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 the Velero contributors.
+Copyright the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -29,10 +30,10 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/vmware-tanzu/velero/internal/hook"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -304,26 +305,9 @@ func (ib *itemBackupper) executeActions(
 	metadata metav1.Object,
 ) (runtime.Unstructured, error) {
 	for _, action := range ib.backupRequest.ResolvedActions {
-		if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) {
-			log.Debug("Skipping action because it does not apply to this resource")
+		if !action.ShouldUse(groupResource, namespace, metadata, log) {
 			continue
 		}
-
-		if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) {
-			log.Debug("Skipping action because it does not apply to this namespace")
-			continue
-		}
-
-		if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() {
-			log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces")
-			continue
-		}
-
-		if !action.selector.Matches(labels.Set(metadata.GetLabels())) {
-			log.Debug("Skipping action because label selector does not match")
-			continue
-		}
-
 		log.Info("Executing custom action")
 
 		updatedItem, additionalItemIdentifiers, err := action.Execute(obj, ib.backupRequest.Backup)
@@ -395,7 +379,13 @@ func (ib *itemBackupper) volumeSnapshotter(snapshotLocation *velerov1api.VolumeS
 // on PVs
 const (
 	zoneLabelDeprecated = "failure-domain.beta.kubernetes.io/zone"
-	zoneLabel           = "topology.kubernetes.io/zone"
+	// this is reused for nodeAffinity requirements
+	zoneLabel = "topology.kubernetes.io/zone"
+
+	awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone"
+	azureCsiZoneKey  = "topology.disk.csi.azure.com/zone"
+	gkeCsiZoneKey    = "topology.gke.io/zone"
+	gkeZoneSeparator = "__"
 )
 
 // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided
@@ -432,7 +422,14 @@ func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.Fie
 		log.Infof("label %q is not present on PersistentVolume, checking deprecated label...", zoneLabel)
 		pvFailureDomainZone, labelFound = pv.Labels[zoneLabelDeprecated]
 		if !labelFound {
+			var k string
 			log.Infof("label %q is not present on PersistentVolume", zoneLabelDeprecated)
+			k, pvFailureDomainZone = zoneFromPVNodeAffinity(pv, awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, zoneLabel, zoneLabelDeprecated)
+			if pvFailureDomainZone != "" {
+				log.Infof("zone info from nodeAffinity requirements: %s, key: %s", pvFailureDomainZone, k)
+			} else {
+				log.Infof("zone info not available in nodeAffinity requirements")
+			}
 		}
 	}
 
@@ -535,3 +532,36 @@ func resourceVersion(obj runtime.Unstructured) string {
 	gvk := obj.GetObjectKind().GroupVersionKind()
 	return gvk.Version
 }
+
+// zoneFromPVNodeAffinity iterates the node affinity requirement of a PV to
+// get its availability zone, it returns the key merely for logging.
+func zoneFromPVNodeAffinity(res *corev1api.PersistentVolume, topologyKeys ...string) (string, string) {
+	nodeAffinity := res.Spec.NodeAffinity
+	if nodeAffinity == nil {
+		return "", ""
+	}
+	keySet := sets.NewString(topologyKeys...)
+	providerGke := false
+	zones := make([]string, 0)
+	for _, term := range nodeAffinity.Required.NodeSelectorTerms {
+		if term.MatchExpressions == nil {
+			continue
+		}
+		for _, exp := range term.MatchExpressions {
+			if keySet.Has(exp.Key) && exp.Operator == "In" && len(exp.Values) > 0 {
+				if exp.Key == gkeCsiZoneKey {
+					providerGke = true
+					zones = append(zones, exp.Values[0])
+				} else {
+					return exp.Key, exp.Values[0]
+				}
+			}
+		}
+	}
+
+	if providerGke {
+		return gkeCsiZoneKey, strings.Join(zones, gkeZoneSeparator)
+	}
+
+	return "", ""
+}

pkg/backup/item_backupper_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -45,3 +46,127 @@ func Test_resourceKey(t *testing.T) {
 		})
 	}
 }
+
+func Test_zoneFromPVNodeAffinity(t *testing.T) {
+	keys := []string{
+		awsEbsCsiZoneKey,
+		azureCsiZoneKey,
+		gkeCsiZoneKey,
+		zoneLabel,
+		zoneLabelDeprecated,
+	}
+	tests := []struct {
+		name      string
+		pv        *corev1api.PersistentVolume
+		wantKey   string
+		wantValue string
+	}{
+		{
+			name: "AWS CSI Volume",
+			pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone",
+						"In", "us-east-2c").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.ebs.csi.aws.com/zone",
+			wantValue: "us-east-2c",
+		},
+		{
+			name: "Azure CSI Volume",
+			pv: builder.ForPersistentVolume("azurecsi").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone",
+						"In", "us-central").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.disk.csi.azure.com/zone",
+			wantValue: "us-central",
+		},
+		{
+			name: "GCP CSI Volume",
+			pv: builder.ForPersistentVolume("gcpcsi").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "us-west1-a").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.gke.io/zone",
+			wantValue: "us-west1-a",
+		},
+		{
+			name: "AWS CSI Volume with multiple zone value, returns the first",
+			pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone",
+						"In", "us-east-2c", "us-west").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.ebs.csi.aws.com/zone",
+			wantValue: "us-east-2c",
+		},
+		{
+			name: "Volume with no matching key",
+			pv: builder.ForPersistentVolume("no-matching-pv").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("some-key",
+						"In", "us-west").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "",
+			wantValue: "",
+		},
+		{
+			name: "Volume with multiple valid keys, returns the first match", // it should never happen
+			pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone",
+						"In", "us-central").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone",
+						"In", "us-east-2c", "us-west").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone",
+						"In", "unknown").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.disk.csi.azure.com/zone",
+			wantValue: "us-central",
+		},
+		{
+			/* an valid example of node affinity in a GKE's regional PV
+			nodeAffinity:
+			  required:
+			    nodeSelectorTerms:
+			    - matchExpressions:
+			      - key: topology.gke.io/zone
+			        operator: In
+			        values:
+			        - us-central1-a
+			    - matchExpressions:
+			      - key: topology.gke.io/zone
+			        operator: In
+			        values:
+			        - us-central1-c
+			*/
+			name: "Volume with multiple valid keys, and provider is gke, returns all valid entries's first zone value",
+			pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "us-central1-c").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "us-east-2c", "us-east-2b").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "europe-north1-a").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.gke.io/zone",
+			wantValue: "us-central1-c__us-east-2c__europe-north1-a",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			k, v := zoneFromPVNodeAffinity(tt.pv, keys...)
+			assert.Equal(t, tt.wantKey, k)
+			assert.Equal(t, tt.wantValue, v)
+		})
+	}
+}

pkg/backup/item_collector.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
@@ -295,7 +295,6 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
 		if selector := r.backupRequest.Spec.LabelSelector; selector != nil {
 			labelSelector = metav1.FormatLabelSelector(selector)
 		}
-		listOptions := metav1.ListOptions{LabelSelector: labelSelector}
 
 		log.Info("Listing items")
 		unstructuredItems := make([]unstructured.Unstructured, 0)
@@ -303,50 +302,42 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
 		if r.pageSize > 0 {
 			// If limit is positive, use a pager to split list over multiple requests
 			// Use Velero's dynamic list function instead of the default
-			listFunc := pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
-				list, err := resourceClient.List(listOptions)
-				if err != nil {
-					return nil, err
-				}
-				return list, nil
-			})
-			listPager := pager.New(listFunc)
+			listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
+				return resourceClient.List(opts)
+			}))
 			// Use the page size defined in the server config
 			// TODO allow configuration of page buffer size
 			listPager.PageSize = int64(r.pageSize)
 			// Add each item to temporary slice
-			var items []unstructured.Unstructured
-			err := listPager.EachListItem(context.Background(), listOptions, func(object runtime.Object) error {
-				item, isUnstructured := object.(*unstructured.Unstructured)
-				if !isUnstructured {
-					// We should never hit this
-					log.Error("Got type other than Unstructured from pager func")
-					return nil
-				}
-				items = append(items, *item)
-				return nil
-			})
-			if statusError, isStatusError := err.(*apierrors.StatusError); isStatusError && statusError.Status().Reason == metav1.StatusReasonExpired {
-				log.WithError(errors.WithStack(err)).Error("Error paging item list. Falling back on unpaginated list")
-				unstructuredList, err := resourceClient.List(listOptions)
-				if err != nil {
-					log.WithError(errors.WithStack(err)).Error("Error listing items")
-					continue
-				}
-				items = unstructuredList.Items
-			} else if err != nil {
-				log.WithError(errors.WithStack(err)).Error("Error paging item list")
+			list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
+			if err != nil {
+				log.WithError(errors.WithStack(err)).Error("Error listing resources")
+				continue
+			}
+			if !paginated {
+				log.Infof("list for groupResource %s was not paginated", gr)
+			}
+			err = meta.EachListItem(list, func(object runtime.Object) error {
+				u, ok := object.(*unstructured.Unstructured)
+				if !ok {
+					log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry in the list")
+					return fmt.Errorf("expected *unstructured.Unstructured but got %T", u)
+				}
+				unstructuredItems = append(unstructuredItems, *u)
+				return nil
+			})
+			if err != nil {
+				log.WithError(errors.WithStack(err)).Error("unable to understand paginated list")
 				continue
 			}
-			unstructuredItems = append(unstructuredItems, items...)
 		} else {
 			// If limit is not positive, do not use paging. Instead, request all items at once
 			unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector})
-			unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
 			if err != nil {
 				log.WithError(errors.WithStack(err)).Error("Error listing items")
 				continue
 			}
+			unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
 		}
 
 		log.Infof("Retrieved %d items", len(unstructuredItems))

pkg/backup/remap_crd_version_action.go

@@ -30,6 +30,8 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
+
 	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 )
@@ -37,13 +39,14 @@ import (
 // RemapCRDVersionAction inspects CustomResourceDefinition and decides if it is a v1
 // CRD that needs to be backed up as v1beta1.
 type RemapCRDVersionAction struct {
-	logger        logrus.FieldLogger
-	betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface
+	logger          logrus.FieldLogger
+	betaCRDClient   apiextv1beta1client.CustomResourceDefinitionInterface
+	discoveryHelper velerodiscovery.Helper
 }
 
 // NewRemapCRDVersionAction instantiates a new RemapCRDVersionAction plugin.
-func NewRemapCRDVersionAction(logger logrus.FieldLogger, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface) *RemapCRDVersionAction {
-	return &RemapCRDVersionAction{logger: logger, betaCRDClient: betaCRDClient}
+func NewRemapCRDVersionAction(logger logrus.FieldLogger, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface, discoveryHelper velerodiscovery.Helper) *RemapCRDVersionAction {
+	return &RemapCRDVersionAction{logger: logger, betaCRDClient: betaCRDClient, discoveryHelper: discoveryHelper}
 }
 
 // AppliesTo selects the resources the plugin should run against. In this case, CustomResourceDefinitions.
@@ -68,7 +71,26 @@ func (a *RemapCRDVersionAction) Execute(item runtime.Unstructured, backup *v1.Ba
 		return item, nil, nil
 	}
 
-	// We've got a v1 CRD, so proceed.
+	// This plugin will exit if the CRD was installed via v1beta1 but the cluster does not support v1beta1 CRD
+	supportv1b1 := false
+CheckVersion:
+	for _, g := range a.discoveryHelper.APIGroups() {
+		if g.Name == apiextv1.GroupName {
+			for _, v := range g.Versions {
+				if v.Version == apiextv1beta1.SchemeGroupVersion.Version {
+					supportv1b1 = true
+					break CheckVersion
+				}
+			}
+
+		}
+	}
+	if !supportv1b1 {
+		a.logger.Info("Exiting RemapCRDVersionAction, the cluster does not support v1beta1 CRD")
+		return item, nil, nil
+	}
+
+	// We've got a v1 CRD and the cluster supports v1beta1 CRD, so proceed.
 	var crd apiextv1.CustomResourceDefinition
 
 	// Do not use runtime.DefaultUnstructuredConverter.FromUnstructured here because it has a bug when converting integers/whole

pkg/backup/remap_crd_version_action_test.go

@@ -32,6 +32,8 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
+
 	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
@@ -48,8 +50,7 @@ func TestRemapCRDVersionAction(t *testing.T) {
 	c := b.Result()
 	_, err := betaClient.Create(context.TODO(), c, metav1.CreateOptions{})
 	require.NoError(t, err)
-
-	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient)
+	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper())
 
 	t.Run("Test a v1 CRD without any Schema information", func(t *testing.T) {
 		b := builder.ForV1CustomResourceDefinition("test.velero.io")
@@ -109,6 +110,33 @@ func TestRemapCRDVersionAction(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"])
 	})
+
+	t.Run("When the cluster only supports v1 CRD, v1 CRD will be returned even the input has Spec.PreserveUnknownFields set to true (issue 4080)", func(t *testing.T) {
+		a.discoveryHelper = &velerotest.FakeDiscoveryHelper{
+			APIGroupsList: []metav1.APIGroup{
+				{
+					Name: apiextv1.GroupName,
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: apiextv1.SchemeGroupVersion.Version,
+						},
+					},
+				},
+			},
+		}
+		b := builder.ForV1CustomResourceDefinition("test.velero.io")
+		b.PreserveUnknownFields(true)
+		c := b.Result()
+		obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c)
+		require.NoError(t, err)
+
+		item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup)
+		require.NoError(t, err)
+		assert.Equal(t, "apiextensions.k8s.io/v1", item.UnstructuredContent()["apiVersion"])
+		// set it back to the default one
+		a.discoveryHelper = fakeDiscoveryHelper()
+	})
+
 }
 
 // TestRemapCRDVersionActionData tests the RemapCRDVersionAction plugin against actual CRD to confirm that the v1beta1 version is returned when the v1 version is passed in to the plugin.
@@ -116,8 +144,7 @@ func TestRemapCRDVersionActionData(t *testing.T) {
 	backup := &v1.Backup{}
 	clientset := apiextfakes.NewSimpleClientset()
 	betaClient := clientset.ApiextensionsV1beta1().CustomResourceDefinitions()
-
-	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient)
+	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper())
 
 	tests := []struct {
 		crd                     string
@@ -192,3 +219,21 @@ func TestRemapCRDVersionActionData(t *testing.T) {
 	}
 
 }
+
+func fakeDiscoveryHelper() velerodiscovery.Helper {
+	return &velerotest.FakeDiscoveryHelper{
+		APIGroupsList: []metav1.APIGroup{
+			{
+				Name: apiextv1.GroupName,
+				Versions: []metav1.GroupVersionForDiscovery{
+					{
+						Version: apiextv1beta1.SchemeGroupVersion.Version,
+					},
+					{
+						Version: apiextv1.SchemeGroupVersion.Version,
+					},
+				},
+			},
+		},
+	}
+}

pkg/backup/request.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/vmware-tanzu/velero/internal/hook"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
@@ -42,11 +43,11 @@ type Request struct {
 	NamespaceIncludesExcludes *collections.IncludesExcludes
 	ResourceIncludesExcludes  *collections.IncludesExcludes
 	ResourceHooks             []hook.ResourceHook
-	ResolvedActions           []resolvedAction
-
-	VolumeSnapshots  []*volume.Snapshot
-	PodVolumeBackups []*velerov1api.PodVolumeBackup
-	BackedUpItems    map[itemKey]struct{}
+	ResolvedActions           []framework.BackupItemResolvedAction
+	ResolvedItemSnapshotters  []framework.ItemSnapshotterResolvedAction
+	VolumeSnapshots           []*volume.Snapshot
+	PodVolumeBackups          []*velerov1api.PodVolumeBackup
+	BackedUpItems             map[itemKey]struct{}
 }
 
 // BackupResourceList returns the list of backed up resources grouped by the API

pkg/builder/node_selector_builder.go

@@ -0,0 +1,64 @@
+package builder
+
+import corev1api "k8s.io/api/core/v1"
+
+// NodeSelectorBuilder builds NodeSelector objects
+type NodeSelectorBuilder struct {
+	object *corev1api.NodeSelector
+}
+
+// ForNodeSelector returns the NodeSelectorBuilder instance with given terms
+func ForNodeSelector(term ...corev1api.NodeSelectorTerm) *NodeSelectorBuilder {
+	return &NodeSelectorBuilder{
+		object: &corev1api.NodeSelector{
+			NodeSelectorTerms: term,
+		},
+	}
+}
+
+// Result returns the built NodeSelector
+func (b *NodeSelectorBuilder) Result() *corev1api.NodeSelector {
+	return b.object
+}
+
+// NodeSelectorTermBuilder builds NodeSelectorTerm objects.
+type NodeSelectorTermBuilder struct {
+	object *corev1api.NodeSelectorTerm
+}
+
+// NewNodeSelectorTermBuilder initializes an instance of NodeSelectorTermBuilder
+func NewNodeSelectorTermBuilder() *NodeSelectorTermBuilder {
+	return &NodeSelectorTermBuilder{
+		object: &corev1api.NodeSelectorTerm{
+			MatchExpressions: make([]corev1api.NodeSelectorRequirement, 0),
+			MatchFields:      make([]corev1api.NodeSelectorRequirement, 0),
+		},
+	}
+}
+
+// WithMatchExpression appends the MatchExpression to the NodeSelectorTerm
+func (ntb *NodeSelectorTermBuilder) WithMatchExpression(key string, op string, values ...string) *NodeSelectorTermBuilder {
+	req := corev1api.NodeSelectorRequirement{
+		Key:      key,
+		Operator: corev1api.NodeSelectorOperator(op),
+		Values:   values,
+	}
+	ntb.object.MatchExpressions = append(ntb.object.MatchExpressions, req)
+	return ntb
+}
+
+// WithMatchField appends the MatchField to the NodeSelectorTerm
+func (ntb *NodeSelectorTermBuilder) WithMatchField(key string, op string, values ...string) *NodeSelectorTermBuilder {
+	req := corev1api.NodeSelectorRequirement{
+		Key:      key,
+		Operator: corev1api.NodeSelectorOperator(op),
+		Values:   values,
+	}
+	ntb.object.MatchFields = append(ntb.object.MatchFields, req)
+	return ntb
+}
+
+// Result returns the built NodeSelectorTerm
+func (ntb *NodeSelectorTermBuilder) Result() *corev1api.NodeSelectorTerm {
+	return ntb.object
+}

pkg/builder/persistent_volume_builder.go

@@ -94,3 +94,11 @@ func (b *PersistentVolumeBuilder) StorageClass(name string) *PersistentVolumeBui
 	b.object.Spec.StorageClassName = name
 	return b
 }
+
+// NodeAffinityRequired sets the PersistentVolume's NodeAffinity Requirement.
+func (b *PersistentVolumeBuilder) NodeAffinityRequired(req *corev1api.NodeSelector) *PersistentVolumeBuilder {
+	b.object.Spec.NodeAffinity = &corev1api.VolumeNodeAffinity{
+		Required: req,
+	}
+	return b
+}

pkg/builder/service_builder.go

@@ -0,0 +1,57 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ServiceBuilder builds Service objects.
+type ServiceBuilder struct {
+	object *corev1api.Service
+}
+
+// ForService is the constructor for a ServiceBuilder.
+func ForService(ns, name string) *ServiceBuilder {
+	return &ServiceBuilder{
+		object: &corev1api.Service{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: corev1api.SchemeGroupVersion.String(),
+				Kind:       "Service",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: ns,
+				Name:      name,
+			},
+		},
+	}
+}
+
+// Result returns the built Service.
+func (s *ServiceBuilder) Result() *corev1api.Service {
+	return s.object
+}
+
+// ObjectMeta applies functional options to the Service's ObjectMeta.
+func (s *ServiceBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ServiceBuilder {
+	for _, opt := range opts {
+		opt(s.object)
+	}
+
+	return s
+}

pkg/builder/statefulset_builder.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2021 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// StatefulSetBuilder builds StatefulSet objects.
+type StatefulSetBuilder struct {
+	object *appsv1.StatefulSet
+}
+
+// ForStatefulSet is the constructor for a StatefulSetBuilder.
+func ForStatefulSet(ns, name string) *StatefulSetBuilder {
+	return &StatefulSetBuilder{
+		object: &appsv1.StatefulSet{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: appsv1.SchemeGroupVersion.String(),
+				Kind:       "StatefulSet",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: ns,
+				Name:      name,
+			},
+			Spec: appsv1.StatefulSetSpec{
+				VolumeClaimTemplates: []corev1.PersistentVolumeClaim{},
+			},
+		},
+	}
+}
+
+// Result returns the built StatefulSet.
+func (b *StatefulSetBuilder) Result() *appsv1.StatefulSet {
+	return b.object
+}
+
+// StorageClass sets the StatefulSet's VolumeClaimTemplates storage class name.
+func (b *StatefulSetBuilder) StorageClass(names ...string) *StatefulSetBuilder {
+	for _, name := range names {
+		nameTmp := name
+		b.object.Spec.VolumeClaimTemplates = append(b.object.Spec.VolumeClaimTemplates,
+			corev1.PersistentVolumeClaim{Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &nameTmp}})
+	}
+	return b
+}

pkg/builder/storage_class_builder.go

@@ -23,7 +23,8 @@ import (
 
 // StorageClassBuilder builds StorageClass objects.
 type StorageClassBuilder struct {
-	object *storagev1api.StorageClass
+	object      *storagev1api.StorageClass
+	objectSlice []*storagev1api.StorageClass
 }
 
 // ForStorageClass is the constructor for a StorageClassBuilder.
@@ -54,3 +55,29 @@ func (b *StorageClassBuilder) ObjectMeta(opts ...ObjectMetaOpt) *StorageClassBui
 
 	return b
 }
+
+// ForStorageClassSlice is the constructor for a storageClassSlice in StorageClassBuilder.
+func ForStorageClassSlice(names ...string) *StorageClassBuilder {
+	var storageClassSlice []*storagev1api.StorageClass
+	for _, name := range names {
+		storageClass := &storagev1api.StorageClass{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: storagev1api.SchemeGroupVersion.String(),
+				Kind:       "StorageClass",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+		}
+		storageClassSlice = append(storageClassSlice, storageClass)
+	}
+
+	return &StorageClassBuilder{
+		objectSlice: storageClassSlice,
+	}
+}
+
+// SliceResult returns the built StorageClass slice.
+func (b *StorageClassBuilder) SliceResult() []*storagev1api.StorageClass {
+	return b.objectSlice
+}

pkg/cmd/cli/backup/create.go

@@ -322,7 +322,7 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 			return nil, err
 		}
 		if o.Name == "" {
-			o.Name = schedule.TimestampedName(time.Now())
+			o.Name = schedule.TimestampedName(time.Now().UTC())
 		}
 		backupBuilder = builder.ForBackup(namespace, o.Name).
 			FromSchedule(schedule)

pkg/cmd/cli/backuplocation/create.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 the Velero contributors.
+Copyright the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -78,6 +78,7 @@ func NewCreateOptions() *CreateOptions {
 	return &CreateOptions{
 		Credential: flag.NewMap(),
 		Config:     flag.NewMap(),
+		Labels:     flag.NewMap(),
 		AccessMode: flag.NewEnum(
 			string(velerov1api.BackupStorageLocationAccessModeReadWrite),
 			string(velerov1api.BackupStorageLocationAccessModeReadWrite),
@@ -133,39 +134,22 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error {
 	return nil
 }
 
-func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
-	var backupSyncPeriod, validationFrequency *metav1.Duration
-
+func (o *CreateOptions) BuildBackupStorageLocation(namespace string, setBackupSyncPeriod, setValidationFrequency bool) (*velerov1api.BackupStorageLocation, error) {
 	var caCertData []byte
 	if o.CACertFile != "" {
 		realPath, err := filepath.Abs(o.CACertFile)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		caCertData, err = ioutil.ReadFile(realPath)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	if c.Flags().Changed("backup-sync-period") {
-		backupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod}
-	}
-
-	if c.Flags().Changed("validation-frequency") {
-		validationFrequency = &metav1.Duration{Duration: o.ValidationFrequency}
-	}
-
-	var secretName, secretKey string
-	for k, v := range o.Credential.Data() {
-		secretName = k
-		secretKey = v
-		break
-	}
-
 	backupStorageLocation := &velerov1api.BackupStorageLocation{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: f.Namespace(),
+			Namespace: namespace,
 			Name:      o.Name,
 			Labels:    o.Labels.Data(),
 		},
@@ -178,15 +162,37 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 					CACert: caCertData,
 				},
 			},
-			Config:              o.Config.Data(),
-			Credential:          builder.ForSecretKeySelector(secretName, secretKey).Result(),
-			Default:             o.DefaultBackupStorageLocation,
-			AccessMode:          velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
-			BackupSyncPeriod:    backupSyncPeriod,
-			ValidationFrequency: validationFrequency,
+			Config:     o.Config.Data(),
+			Default:    o.DefaultBackupStorageLocation,
+			AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
 		},
 	}
 
+	if setBackupSyncPeriod {
+		backupStorageLocation.Spec.BackupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod}
+	}
+
+	if setValidationFrequency {
+		backupStorageLocation.Spec.ValidationFrequency = &metav1.Duration{Duration: o.ValidationFrequency}
+	}
+
+	for secretName, secretKey := range o.Credential.Data() {
+		backupStorageLocation.Spec.Credential = builder.ForSecretKeySelector(secretName, secretKey).Result()
+		break
+	}
+
+	return backupStorageLocation, nil
+}
+
+func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
+	setBackupSyncPeriod := c.Flags().Changed("backup-sync-period")
+	setValidationFrequency := c.Flags().Changed("validation-frequency")
+
+	backupStorageLocation, err := o.BuildBackupStorageLocation(f.Namespace(), setBackupSyncPeriod, setValidationFrequency)
+	if err != nil {
+		return err
+	}
+
 	if printed, err := output.PrintWithFormat(c, backupStorageLocation); printed || err != nil {
 		return err
 	}

pkg/cmd/cli/backuplocation/create_test.go

@@ -0,0 +1,89 @@
+/*
+Copyright the Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backuplocation
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestBuildBackupStorageLocationSetsNamespace(t *testing.T) {
+	o := NewCreateOptions()
+
+	bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false)
+	assert.NoError(t, err)
+	assert.Equal(t, "velero-test-ns", bsl.Namespace)
+}
+
+func TestBuildBackupStorageLocationSetsSyncPeriod(t *testing.T) {
+	o := NewCreateOptions()
+	o.BackupSyncPeriod = 2 * time.Minute
+
+	bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false)
+	assert.NoError(t, err)
+	assert.Nil(t, bsl.Spec.BackupSyncPeriod)
+
+	bsl, err = o.BuildBackupStorageLocation("velero-test-ns", true, false)
+	assert.NoError(t, err)
+	assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.BackupSyncPeriod)
+}
+
+func TestBuildBackupStorageLocationSetsValidationFrequency(t *testing.T) {
+	o := NewCreateOptions()
+	o.ValidationFrequency = 2 * time.Minute
+
+	bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false)
+	assert.NoError(t, err)
+	assert.Nil(t, bsl.Spec.ValidationFrequency)
+
+	bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true)
+	assert.NoError(t, err)
+	assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.ValidationFrequency)
+}
+
+func TestBuildBackupStorageLocationSetsCredential(t *testing.T) {
+	o := NewCreateOptions()
+
+	bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false)
+	assert.NoError(t, err)
+	assert.Nil(t, bsl.Spec.Credential)
+
+	setErr := o.Credential.Set("my-secret=key-from-secret")
+	assert.NoError(t, setErr)
+
+	bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true)
+	assert.NoError(t, err)
+	assert.Equal(t, &v1.SecretKeySelector{
+		LocalObjectReference: v1.LocalObjectReference{Name: "my-secret"},
+		Key:                  "key-from-secret",
+	}, bsl.Spec.Credential)
+}
+
+func TestBuildBackupStorageLocationSetsLabels(t *testing.T) {
+	o := NewCreateOptions()
+
+	err := o.Labels.Set("key=value")
+	assert.NoError(t, err)
+
+	bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]string{"key": "value"}, bsl.Labels)
+}

pkg/cmd/cli/backuplocation/delete.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 the Velero contributors.
+Copyright The Velero Contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
@@ -34,6 +33,8 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli"
 )
 
+const bslLabelKey = "velero.io/storage-location"
+
 // NewDeleteCommand creates and returns a new cobra command for deleting backup-locations.
 func NewDeleteCommand(f client.Factory, use string) *cobra.Command {
 	o := cli.NewDeleteOptions("backup-location")
@@ -120,7 +121,65 @@ func Run(f client.Factory, o *cli.DeleteOptions) error {
 			continue
 		}
 		fmt.Printf("Backup storage location %q deleted successfully.\n", location.Name)
+
+		// Delete backups associated with the deleted BSL.
+		backupList, err := findAssociatedBackups(kbClient, location.Name, f.Namespace())
+		if err != nil {
+			errs = append(errs, fmt.Errorf("find backups associated with BSL %q: %w", location.Name, err))
+		} else if deleteErrs := deleteBackups(kbClient, backupList); deleteErrs != nil {
+			errs = append(errs, deleteErrs...)
+		}
+
+		// Delete Restic repositories associated with the deleted BSL.
+		resticRepoList, err := findAssociatedResticRepos(kbClient, location.Name, f.Namespace())
+		if err != nil {
+			errs = append(errs, fmt.Errorf("find Restic repositories associated with BSL %q: %w", location.Name, err))
+		} else if deleteErrs := deleteResticRepos(kbClient, resticRepoList); deleteErrs != nil {
+			errs = append(errs, deleteErrs...)
+		}
 	}
 
 	return kubeerrs.NewAggregate(errs)
 }
+
+func findAssociatedBackups(client kbclient.Client, bslName, ns string) (velerov1api.BackupList, error) {
+	var backups velerov1api.BackupList
+	err := client.List(context.Background(), &backups, &kbclient.ListOptions{
+		Namespace: ns,
+		Raw:       &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName},
+	})
+	return backups, err
+}
+
+func findAssociatedResticRepos(client kbclient.Client, bslName, ns string) (velerov1api.ResticRepositoryList, error) {
+	var repos velerov1api.ResticRepositoryList
+	err := client.List(context.Background(), &repos, &kbclient.ListOptions{
+		Namespace: ns,
+		Raw:       &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName},
+	})
+	return repos, err
+}
+
+func deleteBackups(client kbclient.Client, backups velerov1api.BackupList) []error {
+	var errs []error
+	for _, backup := range backups.Items {
+		if err := client.Delete(context.Background(), &backup, &kbclient.DeleteOptions{}); err != nil {
+			errs = append(errs, errors.WithStack(fmt.Errorf("delete backup %q associated with deleted BSL: %w", backup.Name, err)))
+			continue
+		}
+		fmt.Printf("Backup associated with deleted BSL(s) %q deleted successfully.\n", backup.Name)
+	}
+	return errs
+}
+
+func deleteResticRepos(client kbclient.Client, repos velerov1api.ResticRepositoryList) []error {
+	var errs []error
+	for _, repo := range repos.Items {
+		if err := client.Delete(context.Background(), &repo, &kbclient.DeleteOptions{}); err != nil {
+			errs = append(errs, errors.WithStack(fmt.Errorf("delete Restic repository %q associated with deleted BSL: %w", repo.Name, err)))
+			continue
+		}
+		fmt.Printf("Restic repository associated with deleted BSL(s) %q deleted successfully.\n", repo.Name)
+	}
+	return errs
+}

pkg/cmd/cli/bug/bug.go

@@ -55,10 +55,15 @@ about: Tell us about a problem you are experiencing
 
 **What did you expect to happen:**
 
+**The following information will help us better understand what's going on**:
 
-**The output of the following commands will help us better understand what's going on**:
-(Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.)
+_If you are using velero v1.7.0+:_  
+Please use ` + "`velero debug  --backup <backupname> --restore <restorename>` " +
+		`to generate the support bundle, and attach to this issue, more options please refer to ` +
+		"`velero debug --help` " + `
 
+_If you are using earlier versions:_  
+Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.)
 - ` + "`kubectl logs deployment/velero -n velero`" + `
 - ` + "`velero backup describe <backupname>` or `kubectl get backup/<backupname> -n velero -o yaml`" + `
 - ` + "`velero backup logs <backupname>`" + `

pkg/cmd/cli/install/install.go

@@ -25,11 +25,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"github.com/vmware-tanzu/velero/internal/velero"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -37,7 +35,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
-	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
 	"github.com/vmware-tanzu/velero/pkg/install"
 	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
 )
@@ -72,7 +69,6 @@ type InstallOptions struct {
 	Plugins                           flag.StringArray
 	NoDefaultBackupLocation           bool
 	CRDsOnly                          bool
-	CRDsVersion                       string
 	CACertFile                        string
 	Features                          string
 	DefaultVolumesToRestic            bool
@@ -107,7 +103,6 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.DurationVar(&o.DefaultResticMaintenanceFrequency, "default-restic-prune-frequency", o.DefaultResticMaintenanceFrequency, "How often 'restic prune' is run for restic repositories by default. Optional.")
 	flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment")
 	flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.")
-	flags.StringVar(&o.CRDsVersion, "crds-version", o.CRDsVersion, "The version to generate CustomResourceDefinition resources if Velero can't discover the Kubernetes preferred CRD API version. Optional.")
 	flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.")
 	flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the restic daemonset, if restic is enabled")
 	flags.BoolVar(&o.DefaultVolumesToRestic, "default-volumes-to-restic", o.DefaultVolumesToRestic, "Bool flag to configure Velero server to use restic by default to backup all pod volumes on all backups. Optional.")
@@ -134,7 +129,6 @@ func NewInstallOptions() *InstallOptions {
 		UseVolumeSnapshots:      true,
 		NoDefaultBackupLocation: false,
 		CRDsOnly:                false,
-		CRDsVersion:             "v1",
 		DefaultVolumesToRestic:  false,
 	}
 }
@@ -193,7 +187,6 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 		NoDefaultBackupLocation:           o.NoDefaultBackupLocation,
 		CACertData:                        caCertData,
 		Features:                          strings.Split(o.Features, ","),
-		CRDsVersion:                       o.CRDsVersion,
 		DefaultVolumesToRestic:            o.DefaultVolumesToRestic,
 	}, nil
 }
@@ -254,30 +247,9 @@ This is useful as a starting point for more customized installations.
 
 // Run executes a command in the context of the provided arguments.
 func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error {
-	// Find the kube-apiserver group apiextensions.k8s.io preferred API version
-	clientset, err := f.KubeClient()
-	if err == nil {
-		// kubeconfig available
-		discoveryHelper, err := velerodiscovery.NewHelper(clientset.Discovery(), &logrus.Logger{})
-		if err == nil {
-			// kubernetes apiserver available
-			gvr, _, err := discoveryHelper.ResourceFor(
-				schema.GroupVersionResource{
-					Group:    "apiextensions.k8s.io",
-					Resource: "customresourcedefinitions",
-				})
-			if err != nil {
-				return err
-			}
-
-			// Update the group apiextensions.k8s.io preferred API version
-			o.CRDsVersion = gvr.Version
-		}
-	}
-
 	var resources *unstructured.UnstructuredList
 	if o.CRDsOnly {
-		resources = install.AllCRDs(o.CRDsVersion)
+		resources = install.AllCRDs()
 	} else {
 		vo, err := o.AsVeleroOptions()
 		if err != nil {
@@ -348,11 +320,6 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact
 		return err
 	}
 
-	// Check the CRD version is valid.
-	if o.CRDsVersion != "v1beta1" && o.CRDsVersion != "v1" {
-		return errors.Errorf("CRD version must be v1beta1 or v1")
-	}
-
 	// If we're only installing CRDs, we can skip the rest of the validation.
 	if o.CRDsOnly {
 		return nil

pkg/cmd/cli/restic/server.go

@@ -32,6 +32,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	v1 "k8s.io/api/core/v1"
+	storagev1api "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -151,6 +152,7 @@ func newResticServer(logger logrus.FieldLogger, factory client.Factory, metricAd
 
 	velerov1api.AddToScheme(scheme)
 	v1.AddToScheme(scheme)
+	storagev1api.AddToScheme(scheme)
 	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
 		Scheme: scheme,
 	})

pkg/cmd/server/plugin/plugin.go

@@ -55,6 +55,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 				RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction).
 				RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)).
 				RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction).
+				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction).
 				Serve()
 		},
 	}
@@ -109,7 +110,16 @@ func newRemapCRDVersionAction(f client.Factory) veleroplugin.HandlerInitializer
 			return nil, err
 		}
 
-		return backup.NewRemapCRDVersionAction(logger, client.ApiextensionsV1beta1().CustomResourceDefinitions()), nil
+		clientset, err := f.KubeClient()
+		if err != nil {
+			return nil, err
+		}
+		discoveryHelper, err := velerodiscovery.NewHelper(clientset.Discovery(), logger)
+		if err != nil {
+			return nil, err
+		}
+
+		return backup.NewRemapCRDVersionAction(logger, client.ApiextensionsV1beta1().CustomResourceDefinitions(), discoveryHelper), nil
 	}
 }
 
@@ -202,3 +212,7 @@ func newChangePVCNodeSelectorItemAction(f client.Factory) veleroplugin.HandlerIn
 func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) {
 	return restore.NewAPIServiceAction(logger), nil
 }
+
+func newAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) (interface{}, error) {
+	return restore.NewAdmissionWebhookConfigurationAction(logger), nil
+}

pkg/cmd/server/server.go

@@ -27,6 +27,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bombsimon/logrusr"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
@@ -302,6 +303,8 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 	velerov1api.AddToScheme(scheme)
 	corev1api.AddToScheme(scheme)
 
+	ctrl.SetLogger(logrusr.NewLogger(logger))
+
 	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
 		Scheme:    scheme,
 		Namespace: f.Namespace(),

pkg/controller/backup_controller.go

@@ -53,6 +53,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	"github.com/vmware-tanzu/velero/pkg/persistence"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	"github.com/vmware-tanzu/velero/pkg/util/encode"
@@ -569,6 +570,10 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 	if err != nil {
 		return err
 	}
+	itemSnapshotters, err := pluginManager.GetItemSnapshotters()
+	if err != nil {
+		return err
+	}
 
 	backupLog.Info("Setting up backup store to check for backup existence")
 	backupStore, err := c.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog)
@@ -586,8 +591,12 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 		return errors.Errorf("backup already exists in object storage")
 	}
 
+	backupItemActionsResolver := framework.NewBackupItemActionResolver(actions)
+	itemSnapshottersResolver := framework.NewItemSnapshotterResolver(itemSnapshotters)
+
 	var fatalErrs []error
-	if err := c.backupper.Backup(backupLog, backup, backupFile, actions, pluginManager); err != nil {
+	if err := c.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver,
+		itemSnapshottersResolver, pluginManager); err != nil {
 		fatalErrs = append(fatalErrs, err)
 	}
 

pkg/controller/backup_controller_test.go

@@ -47,6 +47,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/persistence"
 	persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
@@ -63,6 +64,13 @@ func (b *fakeBackupper) Backup(logger logrus.FieldLogger, backup *pkgbackup.Requ
 	return args.Error(0)
 }
 
+func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *pkgbackup.Request, backupFile io.Writer,
+	backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver,
+	volumeSnapshotterGetter pkgbackup.VolumeSnapshotterGetter) error {
+	args := b.Called(logger, backup, backupFile, backupItemActionResolver, itemSnapshotterResolver, volumeSnapshotterGetter)
+	return args.Error(0)
+}
+
 func defaultBackup() *builder.BackupBuilder {
 	return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1")
 }
@@ -825,7 +833,9 @@ func TestProcessBackupCompletions(t *testing.T) {
 
 			pluginManager.On("GetBackupItemActions").Return(nil, nil)
 			pluginManager.On("CleanupClients").Return(nil)
+			pluginManager.On("GetItemSnapshotters").Return(nil, nil)
 			backupper.On("Backup", mock.Anything, mock.Anything, mock.Anything, []velero.BackupItemAction(nil), pluginManager).Return(nil)
+			backupper.On("BackupWithResolvers", mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolver{}, framework.ItemSnapshotterResolver{}, pluginManager).Return(nil)
 			backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError)
 
 			// Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store.

pkg/controller/backup_deletion_controller.go

@@ -291,7 +291,7 @@ func (c *backupDeletionController) processRequest(req *velerov1api.DeleteBackupR
 
 	backupStore, err := c.backupStoreGetter.Get(location, pluginManager, log)
 	if err != nil {
-		errs = append(errs, err.Error())
+		return errors.Wrap(err, "error getting the backup store")
 	}
 
 	actions, err := pluginManager.GetDeleteItemActions()

pkg/controller/backup_deletion_controller_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"strings"
 	"testing"
 	"time"
 
@@ -36,19 +37,20 @@ import (
 	core "k8s.io/client-go/testing"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks"
+	"github.com/vmware-tanzu/velero/pkg/volume"
+
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
-	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
 	informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
 	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
-	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	"github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
-	"github.com/vmware-tanzu/velero/pkg/volume"
 )
 
 func TestBackupDeletionControllerProcessQueueItem(t *testing.T) {
@@ -183,6 +185,29 @@ func setupBackupDeletionControllerTest(t *testing.T, objects ...runtime.Object)
 }
 
 func TestBackupDeletionControllerProcessRequest(t *testing.T) {
+	t.Run("failed to get backup store", func(t *testing.T) {
+		backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result()
+		location := &velerov1api.BackupStorageLocation{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: backup.Namespace,
+				Name:      backup.Spec.StorageLocation,
+			},
+			Spec: velerov1api.BackupStorageLocationSpec{
+				Provider: "objStoreProvider",
+				StorageType: velerov1api.StorageType{
+					ObjectStorage: &velerov1api.ObjectStorageLocation{
+						Bucket: "bucket",
+					},
+				},
+			},
+		}
+		td := setupBackupDeletionControllerTest(t, location, backup)
+		td.controller.backupStoreGetter = &fakeErrorBackupStoreGetter{}
+		err := td.controller.processRequest(td.req)
+		assert.NotNil(t, err)
+		assert.True(t, strings.HasPrefix(err.Error(), "error getting the backup store"))
+	})
+
 	t.Run("missing spec.backupName", func(t *testing.T) {
 		td := setupBackupDeletionControllerTest(t)
 		td.req.Spec.BackupName = ""

pkg/controller/pod_volume_backup_controller.go

@@ -208,7 +208,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 		return c.fail(req, errors.Wrap(err, "error getting pod").Error(), log)
 	}
 
-	volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister, c.pvLister)
+	volumeDir, err := kube.GetVolumeDirectory(log, pod, req.Spec.Volume, c.pvcLister, c.pvLister, c.kbClient)
 	if err != nil {
 		log.WithError(err).Error("Error getting volume directory name")
 		return c.fail(req, errors.Wrap(err, "error getting volume directory name").Error(), log)

pkg/controller/pod_volume_restore_controller.go

@@ -297,7 +297,7 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe
 		return c.failRestore(req, errors.Wrap(err, "error getting pod").Error(), log)
 	}
 
-	volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister, c.pvLister)
+	volumeDir, err := kube.GetVolumeDirectory(log, pod, req.Spec.Volume, c.pvcLister, c.pvLister, c.kbClient)
 	if err != nil {
 		log.WithError(err).Error("Error getting volume directory name")
 		return c.failRestore(req, errors.Wrap(err, "error getting volume directory name").Error(), log)

pkg/controller/restore_controller.go

@@ -47,6 +47,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	"github.com/vmware-tanzu/velero/pkg/persistence"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	pkgrestore "github.com/vmware-tanzu/velero/pkg/restore"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
@@ -443,6 +444,13 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu
 	if err != nil {
 		return errors.Wrap(err, "error getting restore item actions")
 	}
+	actionsResolver := framework.NewRestoreItemActionResolver(actions)
+
+	itemSnapshotters, err := pluginManager.GetItemSnapshotters()
+	if err != nil {
+		return errors.Wrap(err, "error getting item snapshotters")
+	}
+	snapshotItemResolver := framework.NewItemSnapshotterResolver(itemSnapshotters)
 
 	backupFile, err := downloadToTempFile(restore.Spec.BackupName, info.backupStore, restoreLog)
 	if err != nil {
@@ -476,7 +484,8 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu
 		VolumeSnapshots:  volumeSnapshots,
 		BackupReader:     backupFile,
 	}
-	restoreWarnings, restoreErrors := c.restorer.Restore(restoreReq, actions, c.snapshotLocationLister, pluginManager)
+	restoreWarnings, restoreErrors := c.restorer.RestoreWithResolvers(restoreReq, actionsResolver, snapshotItemResolver,
+		c.snapshotLocationLister, pluginManager)
 	restoreLog.Info("restore completed")
 
 	// re-instantiate the backup store because credentials could have changed since the original

pkg/controller/restore_controller_test.go

@@ -44,8 +44,10 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1"
 	pkgrestore "github.com/vmware-tanzu/velero/pkg/restore"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -505,7 +507,8 @@ func TestProcessQueueItem(t *testing.T) {
 			if test.expectedRestorerCall != nil {
 				backupStore.On("GetBackupContents", test.backup.Name).Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil)
 
-				restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
+				restorer.On("RestoreWithResolvers", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
+					mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
 
 				backupStore.On("PutRestoreLog", test.backup.Name, test.restore.Name, mock.Anything).Return(test.putRestoreLogErr)
 
@@ -545,6 +548,7 @@ func TestProcessQueueItem(t *testing.T) {
 
 			if test.restore != nil {
 				pluginManager.On("GetRestoreItemActions").Return(nil, nil)
+				pluginManager.On("GetItemSnapshotters").Return([]isv1.ItemSnapshotter{}, nil)
 				pluginManager.On("CleanupClients")
 			}
 
@@ -858,3 +862,17 @@ func (r *fakeRestorer) Restore(
 
 	return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result)
 }
+
+func (r *fakeRestorer) RestoreWithResolvers(req pkgrestore.Request,
+	resolver framework.RestoreItemActionResolver,
+	itemSnapshotterResolver framework.ItemSnapshotterResolver,
+	snapshotLocationLister listers.VolumeSnapshotLocationLister,
+	volumeSnapshotterGetter pkgrestore.VolumeSnapshotterGetter,
+) (pkgrestore.Result, pkgrestore.Result) {
+	res := r.Called(req.Log, req.Restore, req.Backup, req.BackupReader, resolver, itemSnapshotterResolver,
+		snapshotLocationLister, volumeSnapshotterGetter)
+
+	r.calledWithArg = *req.Restore
+
+	return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result)
+}

pkg/controller/suite_test.go

@@ -18,6 +18,7 @@ package controller
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"testing"
 	"time"
@@ -130,6 +131,13 @@ func (t *testEnvironment) stop() error {
 	return env.Stop()
 }
 
+type fakeErrorBackupStoreGetter struct {
+}
+
+func (f *fakeErrorBackupStoreGetter) Get(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
+	return nil, fmt.Errorf("some error")
+}
+
 type fakeSingleObjectBackupStoreGetter struct {
 	store persistence.BackupStore
 }

pkg/generated/clientset/versioned/fake/clientset_generated.go

@@ -74,7 +74,10 @@ func (c *Clientset) Tracker() testing.ObjectTracker {
 	return c.tracker
 }
 
-var _ clientset.Interface = &Clientset{}
+var (
+	_ clientset.Interface = &Clientset{}
+	_ testing.FakeClient  = &Clientset{}
+)
 
 // VeleroV1 retrieves the VeleroV1Client
 func (c *Clientset) VeleroV1() velerov1.VeleroV1Interface {

pkg/generated/clientset/versioned/fake/register.go

@@ -29,7 +29,7 @@ import (
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-var parameterCodec = runtime.NewParameterCodec(scheme)
+
 var localSchemeBuilder = runtime.SchemeBuilder{
 	velerov1.AddToScheme,
 }

pkg/generated/listers/velero/v1/backup.go

@@ -26,8 +26,10 @@ import (
 )
 
 // BackupLister helps list Backups.
+// All objects returned here must be treated as read-only.
 type BackupLister interface {
 	// List lists all Backups in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Backup, err error)
 	// Backups returns an object that can list and get Backups.
 	Backups(namespace string) BackupNamespaceLister
@@ -58,10 +60,13 @@ func (s *backupLister) Backups(namespace string) BackupNamespaceLister {
 }
 
 // BackupNamespaceLister helps list and get Backups.
+// All objects returned here must be treated as read-only.
 type BackupNamespaceLister interface {
 	// List lists all Backups in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Backup, err error)
 	// Get retrieves the Backup from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.Backup, error)
 	BackupNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/backupstoragelocation.go

@@ -26,8 +26,10 @@ import (
 )
 
 // BackupStorageLocationLister helps list BackupStorageLocations.
+// All objects returned here must be treated as read-only.
 type BackupStorageLocationLister interface {
 	// List lists all BackupStorageLocations in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error)
 	// BackupStorageLocations returns an object that can list and get BackupStorageLocations.
 	BackupStorageLocations(namespace string) BackupStorageLocationNamespaceLister
@@ -58,10 +60,13 @@ func (s *backupStorageLocationLister) BackupStorageLocations(namespace string) B
 }
 
 // BackupStorageLocationNamespaceLister helps list and get BackupStorageLocations.
+// All objects returned here must be treated as read-only.
 type BackupStorageLocationNamespaceLister interface {
 	// List lists all BackupStorageLocations in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error)
 	// Get retrieves the BackupStorageLocation from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.BackupStorageLocation, error)
 	BackupStorageLocationNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/deletebackuprequest.go

@@ -26,8 +26,10 @@ import (
 )
 
 // DeleteBackupRequestLister helps list DeleteBackupRequests.
+// All objects returned here must be treated as read-only.
 type DeleteBackupRequestLister interface {
 	// List lists all DeleteBackupRequests in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.DeleteBackupRequest, err error)
 	// DeleteBackupRequests returns an object that can list and get DeleteBackupRequests.
 	DeleteBackupRequests(namespace string) DeleteBackupRequestNamespaceLister
@@ -58,10 +60,13 @@ func (s *deleteBackupRequestLister) DeleteBackupRequests(namespace string) Delet
 }
 
 // DeleteBackupRequestNamespaceLister helps list and get DeleteBackupRequests.
+// All objects returned here must be treated as read-only.
 type DeleteBackupRequestNamespaceLister interface {
 	// List lists all DeleteBackupRequests in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.DeleteBackupRequest, err error)
 	// Get retrieves the DeleteBackupRequest from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.DeleteBackupRequest, error)
 	DeleteBackupRequestNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/downloadrequest.go

@@ -26,8 +26,10 @@ import (
 )
 
 // DownloadRequestLister helps list DownloadRequests.
+// All objects returned here must be treated as read-only.
 type DownloadRequestLister interface {
 	// List lists all DownloadRequests in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.DownloadRequest, err error)
 	// DownloadRequests returns an object that can list and get DownloadRequests.
 	DownloadRequests(namespace string) DownloadRequestNamespaceLister
@@ -58,10 +60,13 @@ func (s *downloadRequestLister) DownloadRequests(namespace string) DownloadReque
 }
 
 // DownloadRequestNamespaceLister helps list and get DownloadRequests.
+// All objects returned here must be treated as read-only.
 type DownloadRequestNamespaceLister interface {
 	// List lists all DownloadRequests in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.DownloadRequest, err error)
 	// Get retrieves the DownloadRequest from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.DownloadRequest, error)
 	DownloadRequestNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/podvolumebackup.go

@@ -26,8 +26,10 @@ import (
 )
 
 // PodVolumeBackupLister helps list PodVolumeBackups.
+// All objects returned here must be treated as read-only.
 type PodVolumeBackupLister interface {
 	// List lists all PodVolumeBackups in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.PodVolumeBackup, err error)
 	// PodVolumeBackups returns an object that can list and get PodVolumeBackups.
 	PodVolumeBackups(namespace string) PodVolumeBackupNamespaceLister
@@ -58,10 +60,13 @@ func (s *podVolumeBackupLister) PodVolumeBackups(namespace string) PodVolumeBack
 }
 
 // PodVolumeBackupNamespaceLister helps list and get PodVolumeBackups.
+// All objects returned here must be treated as read-only.
 type PodVolumeBackupNamespaceLister interface {
 	// List lists all PodVolumeBackups in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.PodVolumeBackup, err error)
 	// Get retrieves the PodVolumeBackup from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.PodVolumeBackup, error)
 	PodVolumeBackupNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/podvolumerestore.go

@@ -26,8 +26,10 @@ import (
 )
 
 // PodVolumeRestoreLister helps list PodVolumeRestores.
+// All objects returned here must be treated as read-only.
 type PodVolumeRestoreLister interface {
 	// List lists all PodVolumeRestores in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.PodVolumeRestore, err error)
 	// PodVolumeRestores returns an object that can list and get PodVolumeRestores.
 	PodVolumeRestores(namespace string) PodVolumeRestoreNamespaceLister
@@ -58,10 +60,13 @@ func (s *podVolumeRestoreLister) PodVolumeRestores(namespace string) PodVolumeRe
 }
 
 // PodVolumeRestoreNamespaceLister helps list and get PodVolumeRestores.
+// All objects returned here must be treated as read-only.
 type PodVolumeRestoreNamespaceLister interface {
 	// List lists all PodVolumeRestores in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.PodVolumeRestore, err error)
 	// Get retrieves the PodVolumeRestore from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.PodVolumeRestore, error)
 	PodVolumeRestoreNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/resticrepository.go

@@ -26,8 +26,10 @@ import (
 )
 
 // ResticRepositoryLister helps list ResticRepositories.
+// All objects returned here must be treated as read-only.
 type ResticRepositoryLister interface {
 	// List lists all ResticRepositories in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.ResticRepository, err error)
 	// ResticRepositories returns an object that can list and get ResticRepositories.
 	ResticRepositories(namespace string) ResticRepositoryNamespaceLister
@@ -58,10 +60,13 @@ func (s *resticRepositoryLister) ResticRepositories(namespace string) ResticRepo
 }
 
 // ResticRepositoryNamespaceLister helps list and get ResticRepositories.
+// All objects returned here must be treated as read-only.
 type ResticRepositoryNamespaceLister interface {
 	// List lists all ResticRepositories in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.ResticRepository, err error)
 	// Get retrieves the ResticRepository from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.ResticRepository, error)
 	ResticRepositoryNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/restore.go

@@ -26,8 +26,10 @@ import (
 )
 
 // RestoreLister helps list Restores.
+// All objects returned here must be treated as read-only.
 type RestoreLister interface {
 	// List lists all Restores in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Restore, err error)
 	// Restores returns an object that can list and get Restores.
 	Restores(namespace string) RestoreNamespaceLister
@@ -58,10 +60,13 @@ func (s *restoreLister) Restores(namespace string) RestoreNamespaceLister {
 }
 
 // RestoreNamespaceLister helps list and get Restores.
+// All objects returned here must be treated as read-only.
 type RestoreNamespaceLister interface {
 	// List lists all Restores in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Restore, err error)
 	// Get retrieves the Restore from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.Restore, error)
 	RestoreNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/schedule.go

@@ -26,8 +26,10 @@ import (
 )
 
 // ScheduleLister helps list Schedules.
+// All objects returned here must be treated as read-only.
 type ScheduleLister interface {
 	// List lists all Schedules in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Schedule, err error)
 	// Schedules returns an object that can list and get Schedules.
 	Schedules(namespace string) ScheduleNamespaceLister
@@ -58,10 +60,13 @@ func (s *scheduleLister) Schedules(namespace string) ScheduleNamespaceLister {
 }
 
 // ScheduleNamespaceLister helps list and get Schedules.
+// All objects returned here must be treated as read-only.
 type ScheduleNamespaceLister interface {
 	// List lists all Schedules in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.Schedule, err error)
 	// Get retrieves the Schedule from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.Schedule, error)
 	ScheduleNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/serverstatusrequest.go

@@ -26,8 +26,10 @@ import (
 )
 
 // ServerStatusRequestLister helps list ServerStatusRequests.
+// All objects returned here must be treated as read-only.
 type ServerStatusRequestLister interface {
 	// List lists all ServerStatusRequests in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.ServerStatusRequest, err error)
 	// ServerStatusRequests returns an object that can list and get ServerStatusRequests.
 	ServerStatusRequests(namespace string) ServerStatusRequestNamespaceLister
@@ -58,10 +60,13 @@ func (s *serverStatusRequestLister) ServerStatusRequests(namespace string) Serve
 }
 
 // ServerStatusRequestNamespaceLister helps list and get ServerStatusRequests.
+// All objects returned here must be treated as read-only.
 type ServerStatusRequestNamespaceLister interface {
 	// List lists all ServerStatusRequests in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.ServerStatusRequest, err error)
 	// Get retrieves the ServerStatusRequest from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.ServerStatusRequest, error)
 	ServerStatusRequestNamespaceListerExpansion
 }

pkg/generated/listers/velero/v1/volumesnapshotlocation.go

@@ -26,8 +26,10 @@ import (
 )
 
 // VolumeSnapshotLocationLister helps list VolumeSnapshotLocations.
+// All objects returned here must be treated as read-only.
 type VolumeSnapshotLocationLister interface {
 	// List lists all VolumeSnapshotLocations in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.VolumeSnapshotLocation, err error)
 	// VolumeSnapshotLocations returns an object that can list and get VolumeSnapshotLocations.
 	VolumeSnapshotLocations(namespace string) VolumeSnapshotLocationNamespaceLister
@@ -58,10 +60,13 @@ func (s *volumeSnapshotLocationLister) VolumeSnapshotLocations(namespace string)
 }
 
 // VolumeSnapshotLocationNamespaceLister helps list and get VolumeSnapshotLocations.
+// All objects returned here must be treated as read-only.
 type VolumeSnapshotLocationNamespaceLister interface {
 	// List lists all VolumeSnapshotLocations in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1.VolumeSnapshotLocation, err error)
 	// Get retrieves the VolumeSnapshotLocation from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1.VolumeSnapshotLocation, error)
 	VolumeSnapshotLocationNamespaceListerExpansion
 }