diff --git a/ADOPTERS.md b/ADOPTERS.md index d311e8306..7bc46e9c3 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -17,6 +17,7 @@ If you're using Velero and want to add your organization to this list, replicated.com cloudcasa.io azure.com +broadcom.com ## Success Stories Below is a list of adopters of Velero in **production environments** that have @@ -68,6 +69,9 @@ Replicated uses the Velero open source project to enable snapshots in [KOTS][101 **[Microsoft Azure][105]**
[Azure Backup for AKS][106] is an Azure native, Kubernetes aware, Enterprise ready backup for containerized applications deployed on Azure Kubernetes Service (AKS). AKS Backup utilizes Velero to perform backup and restore operations to protect stateful applications in AKS clusters.
+**[Broadcom][107]**
+[VMware Cloud Foundation][108] (VCF) offers built-in [vSphere Kubernetes Service][109] (VKS), a Kubernetes runtime that includes a CNCF certified Kubernetes distribution, to deploy and manage containerized workloads. VCF empowers platform engineers with native [Kubernetes multi-cluster management][110] capability for managing Kubernetes (K8s) infrastructure at scale. VCF utilizes Velero for Kubernetes data protection enabling platform engineers to back up and restore containerized workloads manifests & persistent volumes, helping to increase the resiliency of stateful applications in VKS cluster. + ## Adding your organization to the list of Velero Adopters If you are using Velero and would like to be included in the list of `Velero Adopters`, add an SVG version of your logo to the `site/static/img/adopters` directory in this repo and submit a [pull request][3] with your change. Name the image file something that reflects your company (e.g., if your company is called Acme, name the image acme.png). See this for an example [PR][4]. @@ -125,3 +129,8 @@ If you would like to add your logo to a future `Adopters of Velero` section on [ [105]: https://azure.microsoft.com/ [106]: https://learn.microsoft.com/azure/backup/backup-overview + +[107]: https://www.broadcom.com/ +[108]: https://www.vmware.com/products/cloud-infrastructure/vmware-cloud-foundation +[109]: https://www.vmware.com/products/cloud-infrastructure/vsphere-kubernetes-service +[110]: https://blogs.vmware.com/cloud-foundation/2025/09/29/empowering-platform-engineers-with-native-kubernetes-multi-cluster-management-in-vmware-cloud-foundation/ \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index b8193933a..2789a573d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -7,11 +7,11 @@ | Maintainer | GitHub ID | Affiliation | |---------------------|---------------------------------------------------------------|--------------------------------------------------| | Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift) | -| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/) | -| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) | -| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | [VMware](https://www.github.com/vmware/) | +| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | Broadcom | +| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | Broadcom | +| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | Broadcom | | Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift) | -| Yonghui Li | [Lyndon-Li](https://github.com/Lyndon-Li) | [VMware](https://www.github.com/vmware/) | +| Yonghui Li | [Lyndon-Li](https://github.com/Lyndon-Li) | Broadcom | | Anshul Ahuja | [anshulahuja98](https://github.com/anshulahuja98) | [Microsoft Azure](https://www.github.com/azure/) | | Tiger Kaovilai | [kaovilai](https://github.com/kaovilai) | [OpenShift](https://github.com/openshift) | @@ -27,14 +27,3 @@ * JenTing Hsiao ([jenting](https://github.com/jenting)) * Dave Smith-Uchida ([dsu-igeek](https://github.com/dsu-igeek)) * Ming Qiu ([qiuming-best](https://github.com/qiuming-best)) - -## Velero Contributors & Stakeholders - -| Feature Area | Lead | -|------------------------|:------------------------------------------------------------------------------------:| -| Technical Lead | Daniel Jiang [reasonerjt](https://github.com/reasonerjt) | -| Kubernetes CSI Liaison | | -| Deployment | | -| Community Management | Orlin Vasilev [OrlinVasilev](https://github.com/OrlinVasilev) | -| Product Management | Pradeep Kumar Chaturvedi [pradeepkchaturvedi](https://github.com/pradeepkchaturvedi) | - diff --git a/README.md b/README.md index 046e55c91..691e11f3e 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,11 @@ The following is a list of the supported Kubernetes versions for each Velero ver | Velero version | Expected Kubernetes version compatibility | Tested on Kubernetes version | |----------------|-------------------------------------------|-------------------------------------| -| 1.17 | 1.18-latest | 1.31.7, 1.32.3, 1.33.1, and 1.34.0 | +| 1.18 | 1.18-latest | 1.33.7, 1.34.1, and 1.35.0 | +| 1.17 | 1.18-latest | 1.31.7, 1.32.3, 1.33.1, and 1.34.0 | | 1.16 | 1.18-latest | 1.31.4, 1.32.3, and 1.33.0 | | 1.15 | 1.18-latest | 1.28.8, 1.29.8, 1.30.4 and 1.31.1 | | 1.14 | 1.18-latest | 1.27.9, 1.28.9, and 1.29.4 | -| 1.13 | 1.18-latest | 1.26.5, 1.27.3, 1.27.8, and 1.28.3 | -| 1.12 | 1.18-latest | 1.25.7, 1.26.5, 1.26.7, and 1.27.3 | -| 1.11 | 1.18-latest | 1.23.10, 1.24.9, 1.25.5, and 1.26.1 | Velero supports IPv4, IPv6, and dual stack environments. Support for this was tested against Velero v1.8. diff --git a/changelogs/CHANGELOG-1.18.md b/changelogs/CHANGELOG-1.18.md new file mode 100644 index 000000000..66c486ddf --- /dev/null +++ b/changelogs/CHANGELOG-1.18.md @@ -0,0 +1,109 @@ +## v1.18 + +### Download +https://github.com/vmware-tanzu/velero/releases/tag/v1.18.0 + +### Container Image +`velero/velero:v1.18.0` + +### Documentation +https://velero.io/docs/v1.18/ + +### Upgrading +https://velero.io/docs/v1.18/upgrade-to-1.18/ + +### Highlights +#### Concurrent backup +In v1.18, Velero is capable to process multiple backups concurrently. This is a significant usability improvement, especially for multiple tenants or multiple users case, backups submitted from different users could run their backups simultaneously without interfering with each other. + +Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/concurrent-backup-processing.md for more details. + +#### Cache volume for data movers +In v1.18, Velero allows users to configure cache volumes for data mover pods during restore for CSI snapshot data movement and fs-backup. This brings below benefits: +- Solve the problem that data mover pods fail to when pod's ephemeral disk is limited +- Solve the problem that multiple data mover pods fail to run concurrently in one node when the node's ephemeral disk is limited +- Working together with backup repository's cache limit configuration, cache volume with appropriate size helps to improve the restore throughput + +Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/backup-repo-cache-volume.md for more details. + +#### Incremental size for data movers +In v1.18, Velero allows users to observe the incremental size of data movers backups for CSI snapshot data movement and fs-backup, so that users could visually see the data reduction due to incremental backup. + +#### Wildcard support for namespaces +In v1.18, Velero allows to use Glob regular expressions for namespace filters during backup and restore, so that users could filter namespaces in a batch manner. + +#### VolumePolicy for PVC phase +In v1.18, Velero VolumePolicy supports actions by PVC phase, which help users to do special operations for PVCs with a specific phase, e.g., skip PVCs in Pending/Lost status from the backup. + +#### Scalability and Resiliency improvements +##### Prevent Velero server OOM Kill for large backup repositories +In v1.18, some backup repository operations are delay executed out of Velero server, so Velero server won't be OOM Killed. + +#### Performance improvement for VolumePolicy +In v1.18, VolumePolicy is enhanced for large number of pods/PVCs so that the performance is significantly improved. + +#### Events for data mover pod diagnostic +In v1.18, events are recorded into data mover pod diagnostic, which allows user to see more information for troubleshooting when the data mover pod fails. + +### Runtime and dependencies +Golang runtime: 1.25.7 +kopia: 0.22.3 + +### Limitations/Known issues + +### Breaking changes +#### Deprecation of PVC selected node feature +According to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md#deprecation-policy), PVC selected node feature is deprecated in v1.18. Velero could appropriately handle PVC's selected-node annotation, so users don't need to do anything particularly. + +### All Changes +* Remove backup from running list when backup fails validation (#9498, @sseago) +* Maintenance Job only uses the first element of the LoadAffinity array (#9494, @blackpiglet) +* Fix issue #9478, add diagnose info on expose peek fails (#9481, @Lyndon-Li) +* Add Role, RoleBinding, ClusterRole, and ClusterRoleBinding in restore sequence. (#9474, @blackpiglet) +* Add maintenance job and data mover pod's labels and annotations setting. (#9452, @blackpiglet) +* Fix plugin init container names exceeding DNS-1123 limit (#9445, @mpryc) +* Add PVC-to-Pod cache to improve volume policy performance (#9441, @shubham-pampattiwar) +* Remove VolumeSnapshotClass from CSI B/R process. (#9431, @blackpiglet) +* Use hookIndex for recording multiple restore exec hooks. (#9366, @blackpiglet) +* Sanitize Azure HTTP responses in BSL status messages (#9321, @shubham-pampattiwar) +* Remove labels associated with previous backups (#9206, @Joeavaikath) +* Add VolumePolicy support for PVC Phase conditions to allow skipping Pending PVCs (#9166, @claude) +* feat: Enhance BackupStorageLocation with Secret-based CA certificate support (#9141, @kaovilai) +* Add `--apply` flag to `install` command, allowing usage of Kubernetes apply to make changes to existing installs (#9132, @mjnagel) +* Fix issue #9194, add doc for GOMAXPROCS behavior change (#9420, @Lyndon-Li) +* Apply volume policies to VolumeGroupSnapshot PVC filtering (#9419, @shubham-pampattiwar) +* Fix issue #9276, add doc for cache volume support (#9418, @Lyndon-Li) +* Add Prometheus metrics for maintenance jobs (#9414, @shubham-pampattiwar) +* Fix issue #9400, connect repo first time after creation so that init params could be written (#9407, @Lyndon-Li) +* Cache volume for PVR (#9397, @Lyndon-Li) +* Cache volume support for DataDownload (#9391, @Lyndon-Li) +* don't copy securitycontext from first container if configmap found (#9389, @sseago) +* Refactor repo provider interface for static configuration (#9379, @Lyndon-Li) +* Fix issue #9365, prevent fake completion notification due to multiple update of single PVR (#9375, @Lyndon-Li) +* Add cache volume configuration (#9370, @Lyndon-Li) +* Track actual resource names for GenerateName in restore status (#9368, @shubham-pampattiwar) +* Fix managed fields patch for resources using GenerateName (#9367, @shubham-pampattiwar) +* Support cache volume for generic restore exposer and pod volume exposer (#9362, @Lyndon-Li) +* Add incrementalSize to DU/PVB for reporting new/changed size (#9357, @sseago) +* Add snapshotSize for DataDownload, PodVolumeRestore (#9354, @Lyndon-Li) +* Add cache dir configuration for udmrepo (#9353, @Lyndon-Li) +* Fix the Job build error when BackupReposiotry name longer than 63. (#9350, @blackpiglet) +* Add cache configuration to VGDP (#9342, @Lyndon-Li) +* Fix issue #9332, add bytesDone for cache files (#9333, @Lyndon-Li) +* Fix typos in documentation (#9329, @T4iFooN-IX) +* Concurrent backup processing (#9307, @sseago) +* VerifyJSONConfigs verify every elements in Data. (#9302, @blackpiglet) +* Fix issue #9267, add events to data mover prepare diagnostic (#9296, @Lyndon-Li) +* Add option for privileged fs-backup pod (#9295, @sseago) +* Fix issue #9193, don't connect repo in repo controller (#9291, @Lyndon-Li) +* Implement concurrency control for cache of native VolumeSnapshotter plugin. (#9281, @0xLeo258) +* Fix issue #7904, remove the code and doc for PVC node selection (#9269, @Lyndon-Li) +* Fix schedule controller to prevent backup queue accumulation during extended blocking scenarios by properly handling empty backup phases (#9264, @shubham-pampattiwar) +* Fix repository maintenance jobs to inherit allowlisted tolerations from Velero deployment (#9256, @shubham-pampattiwar) +* Implement wildcard namespace pattern expansion for backup namespace includes/excludes. This change adds support for wildcard patterns (*, ?, [abc], {a,b,c}) in namespace includes and excludes during backup operations (#9255, @Joeavaikath) +* Protect VolumeSnapshot field from race condition during multi-thread backup (#9248, @0xLeo258) +* Update AzureAD Microsoft Authentication Library to v1.5.0 (#9244, @priyansh17) +* Get pod list once per namespace in pvc IBA (#9226, @sseago) +* Fix issue #7725, add design for backup repo cache configuration (#9148, @Lyndon-Li) +* Fix issue #9229, don't attach backupPVC to the source node (#9233, @Lyndon-Li) +* feat: Permit specifying annotations for the BackupPVC (#9173, @clementnuss) \ No newline at end of file diff --git a/changelogs/unreleased/9132-mjnagel b/changelogs/unreleased/9132-mjnagel deleted file mode 100644 index 24c09a3b1..000000000 --- a/changelogs/unreleased/9132-mjnagel +++ /dev/null @@ -1 +0,0 @@ -Add `--apply` flag to `install` command, allowing usage of Kubernetes apply to make changes to existing installs diff --git a/changelogs/unreleased/9141-kaovilai b/changelogs/unreleased/9141-kaovilai deleted file mode 100644 index 516612444..000000000 --- a/changelogs/unreleased/9141-kaovilai +++ /dev/null @@ -1 +0,0 @@ -feat: Enhance BackupStorageLocation with Secret-based CA certificate support diff --git a/changelogs/unreleased/9148-Lyndon-Li b/changelogs/unreleased/9148-Lyndon-Li deleted file mode 100644 index 5607e26d7..000000000 --- a/changelogs/unreleased/9148-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #7725, add design for backup repo cache configuration \ No newline at end of file diff --git a/changelogs/unreleased/9166-claude b/changelogs/unreleased/9166-claude deleted file mode 100644 index 8e3cb6610..000000000 --- a/changelogs/unreleased/9166-claude +++ /dev/null @@ -1 +0,0 @@ -Add VolumePolicy support for PVC Phase conditions to allow skipping Pending PVCs \ No newline at end of file diff --git a/changelogs/unreleased/9173-clementnuss b/changelogs/unreleased/9173-clementnuss deleted file mode 100644 index c7dece117..000000000 --- a/changelogs/unreleased/9173-clementnuss +++ /dev/null @@ -1 +0,0 @@ -feat: Permit specifying annotations for the BackupPVC diff --git a/changelogs/unreleased/9206-Joeavaikath b/changelogs/unreleased/9206-Joeavaikath deleted file mode 100644 index d080ddf8b..000000000 --- a/changelogs/unreleased/9206-Joeavaikath +++ /dev/null @@ -1 +0,0 @@ -Remove labels associated with previous backups diff --git a/changelogs/unreleased/9226-sseago b/changelogs/unreleased/9226-sseago deleted file mode 100644 index fe67d4cbf..000000000 --- a/changelogs/unreleased/9226-sseago +++ /dev/null @@ -1 +0,0 @@ -Get pod list once per namespace in pvc IBA diff --git a/changelogs/unreleased/9233-Lyndon-Li b/changelogs/unreleased/9233-Lyndon-Li deleted file mode 100644 index f6dd7213a..000000000 --- a/changelogs/unreleased/9233-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9229, don't attach backupPVC to the source node \ No newline at end of file diff --git a/changelogs/unreleased/9244-priyansh17 b/changelogs/unreleased/9244-priyansh17 deleted file mode 100644 index 931c43946..000000000 --- a/changelogs/unreleased/9244-priyansh17 +++ /dev/null @@ -1 +0,0 @@ -Update AzureAD Microsoft Authentication Library to v1.5.0 \ No newline at end of file diff --git a/changelogs/unreleased/9248-0xLeo258 b/changelogs/unreleased/9248-0xLeo258 deleted file mode 100644 index 8332dcade..000000000 --- a/changelogs/unreleased/9248-0xLeo258 +++ /dev/null @@ -1 +0,0 @@ -Protect VolumeSnapshot field from race condition during multi-thread backup diff --git a/changelogs/unreleased/9255-Joeavaikath b/changelogs/unreleased/9255-Joeavaikath deleted file mode 100644 index 4a2560051..000000000 --- a/changelogs/unreleased/9255-Joeavaikath +++ /dev/null @@ -1,10 +0,0 @@ -Implement wildcard namespace pattern expansion for backup namespace includes/excludes. - -This change adds support for wildcard patterns (*, ?, [abc], {a,b,c}) in namespace includes and excludes during backup operations. -When wildcard patterns are detected, they are expanded against the list of active namespaces in the cluster before the backup proceeds. - -Key features: -- Wildcard patterns in namespace includes/excludes are automatically detected and expanded -- Pattern validation ensures unsupported patterns (regex, consecutive asterisks) are rejected -- Empty wildcard results (e.g., "invalid*" matching no namespaces) correctly result in empty backups -- Exact namespace names and "*" continue to work as before (no expansion needed) diff --git a/changelogs/unreleased/9256-shubham-pampattiwar b/changelogs/unreleased/9256-shubham-pampattiwar deleted file mode 100644 index 2400fb6b6..000000000 --- a/changelogs/unreleased/9256-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Fix repository maintenance jobs to inherit allowlisted tolerations from Velero deployment diff --git a/changelogs/unreleased/9264-shubham-pampattiwar b/changelogs/unreleased/9264-shubham-pampattiwar deleted file mode 100644 index 711ea4b57..000000000 --- a/changelogs/unreleased/9264-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Fix schedule controller to prevent backup queue accumulation during extended blocking scenarios by properly handling empty backup phases \ No newline at end of file diff --git a/changelogs/unreleased/9269-Lyndon-Li b/changelogs/unreleased/9269-Lyndon-Li deleted file mode 100644 index 2ed7cdd4f..000000000 --- a/changelogs/unreleased/9269-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #7904, remove the code and doc for PVC node selection \ No newline at end of file diff --git a/changelogs/unreleased/9281-0xLeo258 b/changelogs/unreleased/9281-0xLeo258 deleted file mode 100644 index eb5bf3f5d..000000000 --- a/changelogs/unreleased/9281-0xLeo258 +++ /dev/null @@ -1 +0,0 @@ -Implement concurrency control for cache of native VolumeSnapshotter plugin. diff --git a/changelogs/unreleased/9291-Lyndon-Li b/changelogs/unreleased/9291-Lyndon-Li deleted file mode 100644 index 8dd7c8090..000000000 --- a/changelogs/unreleased/9291-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9193, don't connect repo in repo controller \ No newline at end of file diff --git a/changelogs/unreleased/9295-sseago b/changelogs/unreleased/9295-sseago deleted file mode 100644 index 92f44c7ba..000000000 --- a/changelogs/unreleased/9295-sseago +++ /dev/null @@ -1 +0,0 @@ -Add option for privileged fs-backup pod diff --git a/changelogs/unreleased/9296-Lyndon-Li b/changelogs/unreleased/9296-Lyndon-Li deleted file mode 100644 index 696943ede..000000000 --- a/changelogs/unreleased/9296-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9267, add events to data mover prepare diagnostic \ No newline at end of file diff --git a/changelogs/unreleased/9302-blackpiglet b/changelogs/unreleased/9302-blackpiglet deleted file mode 100644 index 63576a535..000000000 --- a/changelogs/unreleased/9302-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -VerifyJSONConfigs verify every elements in Data. diff --git a/changelogs/unreleased/9307-sseago b/changelogs/unreleased/9307-sseago deleted file mode 100644 index c69891510..000000000 --- a/changelogs/unreleased/9307-sseago +++ /dev/null @@ -1 +0,0 @@ -Concurrent backup processing diff --git a/changelogs/unreleased/9321-shubham-pampattiwar b/changelogs/unreleased/9321-shubham-pampattiwar deleted file mode 100644 index 7fd9cec10..000000000 --- a/changelogs/unreleased/9321-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Sanitize Azure HTTP responses in BSL status messages diff --git a/changelogs/unreleased/9329-T4iFooN-IX b/changelogs/unreleased/9329-T4iFooN-IX deleted file mode 100644 index 2209ecb73..000000000 --- a/changelogs/unreleased/9329-T4iFooN-IX +++ /dev/null @@ -1 +0,0 @@ -Fix typos in documentation diff --git a/changelogs/unreleased/9333-Lyndon-Li b/changelogs/unreleased/9333-Lyndon-Li deleted file mode 100644 index 91d551881..000000000 --- a/changelogs/unreleased/9333-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9332, add bytesDone for cache files \ No newline at end of file diff --git a/changelogs/unreleased/9342-Lyndon-Li b/changelogs/unreleased/9342-Lyndon-Li deleted file mode 100644 index 444643232..000000000 --- a/changelogs/unreleased/9342-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Add cache configuration to VGDP \ No newline at end of file diff --git a/changelogs/unreleased/9350-blackpiglet b/changelogs/unreleased/9350-blackpiglet deleted file mode 100644 index eb3e5fde7..000000000 --- a/changelogs/unreleased/9350-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Fix the Job build error when BackupReposiotry name longer than 63. diff --git a/changelogs/unreleased/9353-Lyndon-Li b/changelogs/unreleased/9353-Lyndon-Li deleted file mode 100644 index ba022edb0..000000000 --- a/changelogs/unreleased/9353-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Add cache dir configuration for udmrepo \ No newline at end of file diff --git a/changelogs/unreleased/9354-Lyndon-Li b/changelogs/unreleased/9354-Lyndon-Li deleted file mode 100644 index 33871a2c3..000000000 --- a/changelogs/unreleased/9354-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Add snapshotSize for DataDownload, PodVolumeRestore \ No newline at end of file diff --git a/changelogs/unreleased/9357-sseago b/changelogs/unreleased/9357-sseago deleted file mode 100644 index 9787a9024..000000000 --- a/changelogs/unreleased/9357-sseago +++ /dev/null @@ -1 +0,0 @@ -Add incrementalSize to DU/PVB for reporting new/changed size diff --git a/changelogs/unreleased/9362-Lyndon-Li b/changelogs/unreleased/9362-Lyndon-Li deleted file mode 100644 index 791b3836d..000000000 --- a/changelogs/unreleased/9362-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Support cache volume for generic restore exposer and pod volume exposer \ No newline at end of file diff --git a/changelogs/unreleased/9366-blackpiglet b/changelogs/unreleased/9366-blackpiglet deleted file mode 100644 index 644ef6a48..000000000 --- a/changelogs/unreleased/9366-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Use hookIndex for recording multiple restore exec hooks. diff --git a/changelogs/unreleased/9367-shubham-pampattiwar b/changelogs/unreleased/9367-shubham-pampattiwar deleted file mode 100644 index f9dac16f6..000000000 --- a/changelogs/unreleased/9367-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Fix managed fields patch for resources using GenerateName diff --git a/changelogs/unreleased/9368-shubham-pampattiwar b/changelogs/unreleased/9368-shubham-pampattiwar deleted file mode 100644 index f8262126a..000000000 --- a/changelogs/unreleased/9368-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Track actual resource names for GenerateName in restore status diff --git a/changelogs/unreleased/9370-Lyndon-Li b/changelogs/unreleased/9370-Lyndon-Li deleted file mode 100644 index 43025c589..000000000 --- a/changelogs/unreleased/9370-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Add cache volume configuration \ No newline at end of file diff --git a/changelogs/unreleased/9375-Lyndon-Li b/changelogs/unreleased/9375-Lyndon-Li deleted file mode 100644 index b60f84894..000000000 --- a/changelogs/unreleased/9375-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9365, prevent fake completion notification due to multiple update of single PVR \ No newline at end of file diff --git a/changelogs/unreleased/9379-Lyndon-Li b/changelogs/unreleased/9379-Lyndon-Li deleted file mode 100644 index 481bf8bdd..000000000 --- a/changelogs/unreleased/9379-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Refactor repo provider interface for static configuration \ No newline at end of file diff --git a/changelogs/unreleased/9389-sseago b/changelogs/unreleased/9389-sseago deleted file mode 100644 index f054c96e3..000000000 --- a/changelogs/unreleased/9389-sseago +++ /dev/null @@ -1 +0,0 @@ -don't copy securitycontext from first container if configmap found diff --git a/changelogs/unreleased/9391-Lyndon-Li b/changelogs/unreleased/9391-Lyndon-Li deleted file mode 100644 index a5176d5ef..000000000 --- a/changelogs/unreleased/9391-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Cache volume support for DataDownload \ No newline at end of file diff --git a/changelogs/unreleased/9397-Lyndon-Li b/changelogs/unreleased/9397-Lyndon-Li deleted file mode 100644 index b5291f6c8..000000000 --- a/changelogs/unreleased/9397-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Cache volume for PVR \ No newline at end of file diff --git a/changelogs/unreleased/9407-Lyndon-Li b/changelogs/unreleased/9407-Lyndon-Li deleted file mode 100644 index ee6c403ef..000000000 --- a/changelogs/unreleased/9407-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9400, connect repo first time after creation so that init params could be written \ No newline at end of file diff --git a/changelogs/unreleased/9414-shubham-pampattiwar b/changelogs/unreleased/9414-shubham-pampattiwar deleted file mode 100644 index bf7ccbfec..000000000 --- a/changelogs/unreleased/9414-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Add Prometheus metrics for maintenance jobs \ No newline at end of file diff --git a/changelogs/unreleased/9418-Lyndon-Li b/changelogs/unreleased/9418-Lyndon-Li deleted file mode 100644 index 7fa2c380e..000000000 --- a/changelogs/unreleased/9418-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9276, add doc for cache volume support \ No newline at end of file diff --git a/changelogs/unreleased/9419-shubham-pampattiwar b/changelogs/unreleased/9419-shubham-pampattiwar deleted file mode 100644 index 9f21ac8ca..000000000 --- a/changelogs/unreleased/9419-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Apply volume policies to VolumeGroupSnapshot PVC filtering diff --git a/changelogs/unreleased/9420-Lyndon-Li b/changelogs/unreleased/9420-Lyndon-Li deleted file mode 100644 index 5575369cc..000000000 --- a/changelogs/unreleased/9420-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9194, add doc for GOMAXPROCS behavior change \ No newline at end of file diff --git a/changelogs/unreleased/9431-blackpiglet b/changelogs/unreleased/9431-blackpiglet deleted file mode 100644 index f82eb459c..000000000 --- a/changelogs/unreleased/9431-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Remove VolumeSnapshotClass from CSI B/R process. diff --git a/changelogs/unreleased/9441-shubham-pampattiwar b/changelogs/unreleased/9441-shubham-pampattiwar deleted file mode 100644 index b7e821a0b..000000000 --- a/changelogs/unreleased/9441-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Add PVC-to-Pod cache to improve volume policy performance diff --git a/changelogs/unreleased/9445-mpryc b/changelogs/unreleased/9445-mpryc deleted file mode 100644 index 9bef666cb..000000000 --- a/changelogs/unreleased/9445-mpryc +++ /dev/null @@ -1 +0,0 @@ -Fix plugin init container names exceeding DNS-1123 limit diff --git a/changelogs/unreleased/9452-blackpiglet b/changelogs/unreleased/9452-blackpiglet deleted file mode 100644 index d057c9eaf..000000000 --- a/changelogs/unreleased/9452-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Add maintenance job and data mover pod's labels and annotations setting. \ No newline at end of file diff --git a/changelogs/unreleased/9474-blackpiglet b/changelogs/unreleased/9474-blackpiglet deleted file mode 100644 index 9bfc11756..000000000 --- a/changelogs/unreleased/9474-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Add Role, RoleBinding, ClusterRole, and ClusterRoleBinding in restore sequence. \ No newline at end of file diff --git a/changelogs/unreleased/9481-Lyndon-Li b/changelogs/unreleased/9481-Lyndon-Li deleted file mode 100644 index 6cadfe64f..000000000 --- a/changelogs/unreleased/9481-Lyndon-Li +++ /dev/null @@ -1 +0,0 @@ -Fix issue #9478, add diagnose info on expose peek fails \ No newline at end of file diff --git a/changelogs/unreleased/9494-blackpiglet b/changelogs/unreleased/9494-blackpiglet deleted file mode 100644 index ac062ddaf..000000000 --- a/changelogs/unreleased/9494-blackpiglet +++ /dev/null @@ -1 +0,0 @@ -Maintenance Job only uses the first element of the LoadAffinity array \ No newline at end of file diff --git a/changelogs/unreleased/9502-Joeavaikath b/changelogs/unreleased/9502-Joeavaikath new file mode 100644 index 000000000..419a5850a --- /dev/null +++ b/changelogs/unreleased/9502-Joeavaikath @@ -0,0 +1 @@ +Support all glob wildcard characters in namespace validation diff --git a/changelogs/unreleased/9508-kaovilai b/changelogs/unreleased/9508-kaovilai new file mode 100644 index 000000000..3c224aee3 --- /dev/null +++ b/changelogs/unreleased/9508-kaovilai @@ -0,0 +1 @@ +Fix VolumePolicy PVC phase condition filter for unbound PVCs (#9507) diff --git a/changelogs/unreleased/9532-Lyndon-Li b/changelogs/unreleased/9532-Lyndon-Li new file mode 100644 index 000000000..0d5094c22 --- /dev/null +++ b/changelogs/unreleased/9532-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9343, include PV topology to data mover pod affinities \ No newline at end of file diff --git a/changelogs/unreleased/9533-Lyndon-Li‎‎ b/changelogs/unreleased/9533-Lyndon-Li‎‎ new file mode 100644 index 000000000..acd2b37cb --- /dev/null +++ b/changelogs/unreleased/9533-Lyndon-Li‎‎ @@ -0,0 +1 @@ +Fix issue #9496, support customized host os \ No newline at end of file diff --git a/design/backup-repo-cache-volume.md b/design/Implemented/backup-repo-cache-volume.md similarity index 100% rename from design/backup-repo-cache-volume.md rename to design/Implemented/backup-repo-cache-volume.md diff --git a/design/bsl-certificate-support_design.md b/design/Implemented/bsl-certificate-support_design.md similarity index 100% rename from design/bsl-certificate-support_design.md rename to design/Implemented/bsl-certificate-support_design.md diff --git a/design/concurrent-backup-processing.md b/design/Implemented/concurrent-backup-processing.md similarity index 100% rename from design/concurrent-backup-processing.md rename to design/Implemented/concurrent-backup-processing.md diff --git a/design/wildcard-namespace-support-design.md b/design/Implemented/wildcard-namespace-support-design.md similarity index 100% rename from design/wildcard-namespace-support-design.md rename to design/Implemented/wildcard-namespace-support-design.md diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index 89477d2fb..65378b22e 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -21,9 +21,11 @@ ENV GO111MODULE=on ENV GOPROXY=${GOPROXY} # kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test. -RUN curl -sSLo envtest-bins.tar.gz https://go.kubebuilder.io/test-tools/1.22.1/linux/$(go env GOARCH) && \ - mkdir /usr/local/kubebuilder && \ - tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz +# Using setup-envtest to download envtest binaries +RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest && \ + mkdir -p /usr/local/kubebuilder/bin && \ + ENVTEST_ASSETS_DIR=$(setup-envtest use 1.33.0 --bin-dir /usr/local/kubebuilder/bin -p path) && \ + cp -r ${ENVTEST_ASSETS_DIR}/* /usr/local/kubebuilder/bin/ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_$(go env GOARCH) && \ mv kubebuilder_linux_$(go env GOARCH) /usr/local/kubebuilder/bin/kubebuilder && \ diff --git a/hack/fix_restic_cve.txt b/hack/fix_restic_cve.txt index c05992097..eeee2ecc6 100644 --- a/hack/fix_restic_cve.txt +++ b/hack/fix_restic_cve.txt @@ -1,5 +1,5 @@ diff --git a/go.mod b/go.mod -index 5f939c481..6ae17f4a1 100644 +index 5f939c481..f6205aa3c 100644 --- a/go.mod +++ b/go.mod @@ -24,32 +24,31 @@ require ( @@ -14,13 +14,13 @@ index 5f939c481..6ae17f4a1 100644 - golang.org/x/term v0.4.0 - golang.org/x/text v0.6.0 - google.golang.org/api v0.106.0 -+ golang.org/x/crypto v0.36.0 -+ golang.org/x/net v0.38.0 ++ golang.org/x/crypto v0.45.0 ++ golang.org/x/net v0.47.0 + golang.org/x/oauth2 v0.28.0 -+ golang.org/x/sync v0.12.0 -+ golang.org/x/sys v0.31.0 -+ golang.org/x/term v0.30.0 -+ golang.org/x/text v0.23.0 ++ golang.org/x/sync v0.18.0 ++ golang.org/x/sys v0.38.0 ++ golang.org/x/term v0.37.0 ++ golang.org/x/text v0.31.0 + google.golang.org/api v0.114.0 ) @@ -64,11 +64,11 @@ index 5f939c481..6ae17f4a1 100644 ) -go 1.18 -+go 1.23.0 ++go 1.24.0 + -+toolchain go1.23.7 ++toolchain go1.24.11 diff --git a/go.sum b/go.sum -index 026e1d2fa..805792055 100644 +index 026e1d2fa..4a37e7ac7 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,24 @@ @@ -170,8 +170,8 @@ index 026e1d2fa..805792055 100644 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= ++golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= ++golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -181,8 +181,8 @@ index 026e1d2fa..805792055 100644 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= ++golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= ++golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= @@ -194,8 +194,8 @@ index 026e1d2fa..805792055 100644 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= ++golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= ++golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -205,21 +205,21 @@ index 026e1d2fa..805792055 100644 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= ++golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= ++golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -+golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -+golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= ++golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= ++golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= ++golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= ++golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go index 160a2005e..a47f7be83 100644 --- a/internal/volumehelper/volume_policy_helper.go +++ b/internal/volumehelper/volume_policy_helper.go @@ -134,6 +134,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group pv := new(corev1api.PersistentVolume) var err error + var pvNotFoundErr error if groupResource == kuberesource.PersistentVolumeClaims { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { v.logger.WithError(err).Error("fail to convert unstructured into PVC") @@ -142,8 +143,10 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group pv, err = kubeutil.GetPVForPVC(pvc, v.client) if err != nil { - v.logger.WithError(err).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) - return false, err + // Any error means PV not available - save to return later if no policy matches + v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) + pvNotFoundErr = err + pv = nil } } @@ -158,7 +161,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) action, err := v.volumePolicy.GetMatchAction(vfd) if err != nil { - v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for PV %s", pv.Name) + v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for %+v", vfd) return false, err } @@ -167,15 +170,21 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group // If there is no match action, go on to the next check. if action != nil { if action.Type == resourcepolicies.Snapshot { - v.logger.Infof(fmt.Sprintf("performing snapshot action for pv %s", pv.Name)) + v.logger.Infof("performing snapshot action for %+v", vfd) return true, nil } else { - v.logger.Infof("Skip snapshot action for pv %s as the action type is %s", pv.Name, action.Type) + v.logger.Infof("Skip snapshot action for %+v as the action type is %s", vfd, action.Type) return false, nil } } } + // If resource is PVC, and PV is nil (e.g., Pending/Lost PVC with no matching policy), return the original error + if groupResource == kuberesource.PersistentVolumeClaims && pv == nil && pvNotFoundErr != nil { + v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, pvNotFoundErr + } + // If this PV is claimed, see if we've already taken a (pod volume backup) // snapshot of the contents of this PV. If so, don't take a snapshot. if pv.Spec.ClaimRef != nil { @@ -209,7 +218,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group return true, nil } - v.logger.Infof(fmt.Sprintf("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name)) + v.logger.Infof("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name) return false, nil } @@ -219,6 +228,7 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor return false, nil } + var pvNotFoundErr error if v.volumePolicy != nil { var resource any var err error @@ -230,10 +240,13 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor v.logger.WithError(err).Errorf("fail to get PVC for pod %s", pod.Namespace+"/"+pod.Name) return false, err } - resource, err = kubeutil.GetPVForPVC(pvc, v.client) + pvResource, err := kubeutil.GetPVForPVC(pvc, v.client) if err != nil { - v.logger.WithError(err).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) - return false, err + // Any error means PV not available - save to return later if no policy matches + v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) + pvNotFoundErr = err + } else { + resource = pvResource } } @@ -260,6 +273,12 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor return false, nil } } + + // If no policy matched and PV was not found, return the original error + if pvNotFoundErr != nil { + v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, pvNotFoundErr + } } if v.shouldPerformFSBackupLegacy(volume, pod) { diff --git a/internal/volumehelper/volume_policy_helper_test.go b/internal/volumehelper/volume_policy_helper_test.go index 8d6073c2b..5e52ae73b 100644 --- a/internal/volumehelper/volume_policy_helper_test.go +++ b/internal/volumehelper/volume_policy_helper_test.go @@ -286,7 +286,7 @@ func TestVolumeHelperImpl_ShouldPerformSnapshot(t *testing.T) { expectedErr: false, }, { - name: "PVC not having PV, return false and error case PV not found", + name: "PVC not having PV, return false and error when no matching policy", inputObj: builder.ForPersistentVolumeClaim("default", "example-pvc").StorageClass("gp2-csi").Result(), groupResource: kuberesource.PersistentVolumeClaims, resourcePolicies: &resourcepolicies.ResourcePolicies{ @@ -1234,3 +1234,312 @@ func TestNewVolumeHelperImplWithCache_UsesCache(t *testing.T) { require.NoError(t, err) require.False(t, shouldSnapshot, "Expected snapshot to be skipped due to fs-backup selection via cache") } + +// TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC tests that Pending and Lost PVCs with +// phase-based skip policies don't cause errors when GetPVForPVC would fail. +func TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC(t *testing.T) { + testCases := []struct { + name string + inputPVC *corev1api.PersistentVolumeClaim + resourcePolicies *resourcepolicies.ResourcePolicies + shouldSnapshot bool + expectedErr bool + }{ + { + name: "Pending PVC with phase-based skip policy should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Pending PVC without matching skip policy should error (no PV)", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: true, + }, + { + name: "Lost PVC with phase-based skip policy should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Lost PVC with policy for Pending and Lost should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputPVC) + require.NoError(t, err) + + actualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumeClaims) + if tc.expectedErr { + require.Error(t, actualError, "Want error; Got nil error") + return + } + + require.NoError(t, actualError) + require.Equalf(t, tc.shouldSnapshot, actualShouldSnapshot, "Want shouldSnapshot as %t; Got shouldSnapshot as %t", tc.shouldSnapshot, actualShouldSnapshot) + }) + } +} + +// TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC tests that Pending and Lost PVCs with +// phase-based skip policies don't cause errors when GetPVForPVC would fail. +func TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC(t *testing.T) { + testCases := []struct { + name string + pod *corev1api.Pod + pvc *corev1api.PersistentVolumeClaim + resourcePolicies *resourcepolicies.ResourcePolicies + shouldFSBackup bool + expectedErr bool + }{ + { + name: "Pending PVC with phase-based skip policy should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-pending", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-pending", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + { + name: "Pending PVC without matching skip policy should error (no PV)", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-pending", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-pending-no-policy", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: true, + }, + { + name: "Lost PVC with phase-based skip policy should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-lost", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-lost", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + { + name: "Lost PVC with policy for Pending and Lost should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-lost", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-lost", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.pvc) + require.NoError(t, fakeClient.Create(t.Context(), tc.pod)) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + actualShouldFSBackup, actualError := vh.ShouldPerformFSBackup(tc.pod.Spec.Volumes[0], *tc.pod) + if tc.expectedErr { + require.Error(t, actualError, "Want error; Got nil error") + return + } + + require.NoError(t, actualError) + require.Equalf(t, tc.shouldFSBackup, actualShouldFSBackup, "Want shouldFSBackup as %t; Got shouldFSBackup as %t", tc.shouldFSBackup, actualShouldFSBackup) + }) + } +} diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 7b1ea69bd..feae0e01c 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -687,15 +687,14 @@ func (ib *itemBackupper) getMatchAction(obj runtime.Unstructured, groupResource return nil, errors.WithStack(err) } - pvName := pvc.Spec.VolumeName - if pvName == "" { - return nil, errors.Errorf("PVC has no volume backing this claim") - } - - pv := &corev1api.PersistentVolume{} - if err := ib.kbClient.Get(context.Background(), kbClient.ObjectKey{Name: pvName}, pv); err != nil { - return nil, errors.WithStack(err) + var pv *corev1api.PersistentVolume + if pvName := pvc.Spec.VolumeName; pvName != "" { + pv = &corev1api.PersistentVolume{} + if err := ib.kbClient.Get(context.Background(), kbClient.ObjectKey{Name: pvName}, pv); err != nil { + return nil, errors.WithStack(err) + } } + // If pv is nil for unbound PVCs - policy matching will use PVC-only conditions vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return ib.backupRequest.ResPolicies.GetMatchAction(vfd) } @@ -709,7 +708,10 @@ func (ib *itemBackupper) trackSkippedPV(obj runtime.Unstructured, groupResource if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Track(name, approach, reason) } else if err != nil { - log.WithError(err).Warnf("unable to get PV name, skip tracking.") + // Log at info level for tracking purposes. This is not an error because + // it's expected for some resources (e.g., PVCs in Pending or Lost phase) + // to not have a PV name. This occurs when volume policy skips unbound PVCs. + log.WithError(err).Infof("unable to get PV name, skip tracking.") } } @@ -719,6 +721,17 @@ func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResourc if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Untrack(name) } else if err != nil { + // For PVCs in Pending or Lost phase, it's expected that there's no PV name. + // Log at debug level instead of warning to reduce noise. + if groupResource == kuberesource.PersistentVolumeClaims { + pvc := new(corev1api.PersistentVolumeClaim) + if convErr := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); convErr == nil { + if pvc.Status.Phase == corev1api.ClaimPending || pvc.Status.Phase == corev1api.ClaimLost { + log.WithError(err).Debugf("unable to get PV name for %s PVC, skip untracking.", pvc.Status.Phase) + return + } + } + } log.WithError(err).Warnf("unable to get PV name, skip untracking.") } } diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index b76536baa..be91b6d34 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -17,12 +17,15 @@ limitations under the License. package backup import ( + "bytes" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/stretchr/testify/assert" @@ -269,3 +272,225 @@ func TestAddVolumeInfo(t *testing.T) { }) } } + +func TestGetMatchAction_PendingLostPVC(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1api.AddToScheme(scheme)) + + // Create resource policies that skip Pending/Lost PVCs + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + pv *corev1api.PersistentVolume + expectedAction *resourcepolicies.Action + expectError bool + }{ + { + name: "Pending PVC with no VolumeName should match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Lost PVC with no VolumeName should match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimLost). + Result(), + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Bound PVC with VolumeName and matching PV should not match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + StorageClass("test-sc"). + VolumeName("test-pv"). + Phase(corev1api.ClaimBound). + Result(), + pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), + expectedAction: nil, + expectError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Build fake client with PV if present + clientBuilder := ctrlfake.NewClientBuilder().WithScheme(scheme) + if tc.pv != nil { + clientBuilder = clientBuilder.WithObjects(tc.pv) + } + fakeClient := clientBuilder.Build() + + ib := &itemBackupper{ + kbClient: fakeClient, + backupRequest: &Request{ + ResPolicies: policies, + }, + } + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + action, err := ib.getMatchAction(obj, kuberesource.PersistentVolumeClaims, csiBIAPluginName) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tc.expectedAction == nil { + assert.Nil(t, action) + } else { + require.NotNil(t, action) + assert.Equal(t, tc.expectedAction.Type, action.Type) + } + }) + } +} + +func TestTrackSkippedPV_PendingLostPVC(t *testing.T) { + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + }{ + { + name: "Pending PVC should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + Phase(corev1api.ClaimPending). + Result(), + }, + { + name: "Lost PVC should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + Phase(corev1api.ClaimLost). + Result(), + }, + { + name: "Bound PVC without VolumeName should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + Phase(corev1api.ClaimBound). + Result(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ib := &itemBackupper{ + backupRequest: &Request{ + SkippedPVTracker: NewSkipPVTracker(), + }, + } + + // Set up log capture + logOutput := &bytes.Buffer{} + logger := logrus.New() + logger.SetOutput(logOutput) + logger.SetLevel(logrus.DebugLevel) + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + ib.trackSkippedPV(obj, kuberesource.PersistentVolumeClaims, "", "test reason", logger) + + logStr := logOutput.String() + assert.Contains(t, logStr, "level=info") + assert.Contains(t, logStr, "unable to get PV name, skip tracking.") + }) + } +} + +func TestUnTrackSkippedPV_PendingLostPVC(t *testing.T) { + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + expectWarningLog bool + expectDebugMessage string + }{ + { + name: "Pending PVC should log at debug level, not warning", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + Phase(corev1api.ClaimPending). + Result(), + expectWarningLog: false, + expectDebugMessage: "unable to get PV name for Pending PVC, skip untracking.", + }, + { + name: "Lost PVC should log at debug level, not warning", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + Phase(corev1api.ClaimLost). + Result(), + expectWarningLog: false, + expectDebugMessage: "unable to get PV name for Lost PVC, skip untracking.", + }, + { + name: "Bound PVC without VolumeName should log warning", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + Phase(corev1api.ClaimBound). + Result(), + expectWarningLog: true, + expectDebugMessage: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ib := &itemBackupper{ + backupRequest: &Request{ + SkippedPVTracker: NewSkipPVTracker(), + }, + } + + // Set up log capture + logOutput := &bytes.Buffer{} + logger := logrus.New() + logger.SetOutput(logOutput) + logger.SetLevel(logrus.DebugLevel) + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + ib.unTrackSkippedPV(obj, kuberesource.PersistentVolumeClaims, logger) + + logStr := logOutput.String() + if tc.expectWarningLog { + assert.Contains(t, logStr, "level=warning") + assert.Contains(t, logStr, "unable to get PV name, skip untracking.") + } else { + assert.NotContains(t, logStr, "level=warning") + if tc.expectDebugMessage != "" { + assert.Contains(t, logStr, "level=debug") + assert.Contains(t, logStr, tc.expectDebugMessage) + } + } + }) + } +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 285c5271d..74e003572 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -340,20 +340,16 @@ func (s *nodeAgentServer) run() { } } + var cachePVCConfig *velerotypes.CachePVC if s.dataPathConfigs != nil && s.dataPathConfigs.CachePVCConfig != nil { if err := s.validateCachePVCConfig(*s.dataPathConfigs.CachePVCConfig); err != nil { s.logger.WithError(err).Warnf("Ignore cache config %v", s.dataPathConfigs.CachePVCConfig) } else { + cachePVCConfig = s.dataPathConfigs.CachePVCConfig s.logger.Infof("Using cache volume configs %v", s.dataPathConfigs.CachePVCConfig) } } - var cachePVCConfig *velerotypes.CachePVC - if s.dataPathConfigs != nil && s.dataPathConfigs.CachePVCConfig != nil { - cachePVCConfig = s.dataPathConfigs.CachePVCConfig - s.logger.Infof("Using customized cachePVC config %v", cachePVCConfig) - } - var podLabels map[string]string if s.dataPathConfigs != nil && len(s.dataPathConfigs.PodLabels) > 0 { podLabels = s.dataPathConfigs.PodLabels @@ -368,6 +364,8 @@ func (s *nodeAgentServer) run() { if s.backupRepoConfigs != nil { s.logger.Infof("Using backup repo config %v", s.backupRepoConfigs) + } else if cachePVCConfig != nil { + s.logger.Info("Backup repo config is not provided, using default values for cache volume configs") } pvbReconciler := controller.NewPodVolumeBackupReconciler( diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 37faafd68..496308a6e 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -307,6 +307,16 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr backupScheduleName := request.GetLabels()[velerov1api.ScheduleNameLabel] + b.backupTracker.Add(request.Namespace, request.Name) + defer func() { + switch request.Status.Phase { + case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: + b.backupTracker.Delete(request.Namespace, request.Name) + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: + b.backupTracker.AddPostProcessing(request.Namespace, request.Name) + } + }() + if request.Status.Phase == velerov1api.BackupPhaseFailedValidation { log.Debug("failed to validate backup status") b.metrics.RegisterBackupValidationFailure(backupScheduleName) @@ -318,16 +328,6 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // store ref to just-updated item for creating patch original = request.Backup.DeepCopy() - b.backupTracker.Add(request.Namespace, request.Name) - defer func() { - switch request.Status.Phase { - case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: - b.backupTracker.Delete(request.Namespace, request.Name) - case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: - b.backupTracker.AddPostProcessing(request.Namespace, request.Name) - } - }() - log.Debug("Running backup") b.metrics.RegisterBackupAttempt(backupScheduleName) diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index e0b554ff1..386498900 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -246,6 +246,7 @@ func TestProcessBackupValidationFailures(t *testing.T) { clock: &clock.RealClock{}, formatFlag: formatFlag, metrics: metrics.NewServerMetrics(), + backupTracker: NewBackupTracker(), } require.NotNil(t, test.backup) diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 5acb229d2..079a4b527 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -124,6 +124,15 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O "owner": ownerObject.Name, }) + volumeTopology, err := kube.GetVolumeTopology(ctx, e.kubeClient.CoreV1(), e.kubeClient.StorageV1(), csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + if err != nil { + return errors.Wrapf(err, "error getting volume topology for PV %s, storage class %s", csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + } + + if volumeTopology != nil { + curLog.Infof("Using volume topology %v", volumeTopology) + } + curLog.Info("Exposing CSI snapshot") volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog) @@ -254,6 +263,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O csiExposeParam.NodeOS, csiExposeParam.PriorityClassName, intoleratableNodes, + volumeTopology, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -320,7 +330,8 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1a curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i) var nodeOS *string - if os, found := pod.Spec.NodeSelector[kube.NodeOSLabel]; found { + if pod.Spec.OS != nil { + os := string(pod.Spec.OS.Name) nodeOS = &os } @@ -588,6 +599,7 @@ func (e *csiSnapshotExposer) createBackupPod( nodeOS string, priorityClassName string, intoleratableNodes []string, + volumeTopology *corev1api.NodeSelector, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -643,6 +655,10 @@ func (e *csiSnapshotExposer) createBackupPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + if affinity == nil { + affinity = &kube.LoadAffinity{} + } + var securityCtx *corev1api.PodSecurityContext nodeSelector := map[string]string{} podOS := corev1api.PodOS{} @@ -654,9 +670,14 @@ func (e *csiSnapshotExposer) createBackupPod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -683,11 +704,15 @@ func (e *csiSnapshotExposer) createBackupPod( } } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } - var podAffinity *corev1api.Affinity if len(intoleratableNodes) > 0 { if affinity == nil { affinity = &kube.LoadAffinity{} @@ -700,9 +725,7 @@ func (e *csiSnapshotExposer) createBackupPod( }) } - if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) - } + podAffinity := kube.ToSystemAffinity(affinity, volumeTopology) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index 345d5b327..d1ffa4700 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -154,6 +154,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { kube.NodeOSLinux, tc.expectedPriorityClass, nil, + nil, ) require.NoError(t, err, tc.description) @@ -239,6 +240,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { kube.NodeOSLinux, "", // empty priority class since config map is missing nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index b4dd92c3f..e1a9860eb 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -68,6 +68,12 @@ func TestExpose(t *testing.T) { var restoreSize int64 = 123456 + scObj := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-sc", + }, + } + snapshotClass := "fake-snapshot-class" vsObject := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ @@ -199,6 +205,18 @@ func TestExpose(t *testing.T) { expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string }{ + { + name: "get volume topology fail", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + }, + err: "error getting volume topology for PV fake-pv, storage class fake-sc: error getting storage class fake-sc: storageclasses.storage.k8s.io \"fake-sc\" not found", + }, { name: "wait vs ready fail", ownerBackup: backup, @@ -206,6 +224,11 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + }, + kubeClientObj: []runtime.Object{ + scObj, }, err: "error wait volume snapshot ready: error to get VolumeSnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, @@ -217,10 +240,15 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to get volume snapshot content: error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io \"fake-vsc\" not found", }, { @@ -231,6 +259,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -245,6 +275,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to delete volume snapshot: error to delete volume snapshot: fake-delete-error", }, { @@ -255,6 +288,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -269,6 +304,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to delete volume snapshot content: error to delete volume snapshot content: fake-delete-error", }, { @@ -279,6 +317,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -293,6 +333,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup volume snapshot: fake-create-error", }, { @@ -303,6 +346,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -317,6 +362,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup volume snapshot content: fake-create-error", }, { @@ -326,11 +374,16 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: "fake-mode", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup pvc: unsupported access mode fake-mode", }, { @@ -342,6 +395,8 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, AccessMode: AccessModeFileSystem, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -356,6 +411,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup pvc: error to create pvc: fake-create-error", }, { @@ -367,6 +425,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -374,6 +434,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, kubeReactors: []reactor{ { @@ -395,6 +456,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -402,6 +465,24 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, }, { @@ -413,6 +494,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -420,6 +503,24 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, }, { @@ -432,6 +533,8 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, VolumeSize: *resource.NewQuantity(567890, ""), + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObjectWithoutRestoreSize, @@ -439,8 +542,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedVolumeSize: resource.NewQuantity(567890, ""), + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts read only backupPVC", @@ -449,6 +570,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -465,8 +587,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedReadOnlyPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts read only backupPVC and storageClass specified in backupPVC config", @@ -475,6 +615,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -491,9 +632,27 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedReadOnlyPVC: true, expectedBackupPVCStorageClass: "fake-sc-read-only", + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts backupPVC with storageClass specified in backupPVC config", @@ -502,6 +661,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -517,8 +677,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "Affinity per StorageClass", @@ -527,6 +705,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -551,6 +730,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ @@ -563,6 +743,11 @@ func TestExpose(t *testing.T) { Operator: corev1api.NodeSelectorOpIn, Values: []string{"Linux"}, }, + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, }, }, }, @@ -577,6 +762,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -606,6 +792,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ @@ -619,6 +806,11 @@ func TestExpose(t *testing.T) { Operator: corev1api.NodeSelectorOpIn, Values: []string{"amd64"}, }, + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, }, }, }, @@ -633,6 +825,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -649,9 +842,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", - expectedAffinity: nil, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "IntolerateSourceNode, get source node fail", @@ -677,6 +887,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, kubeReactors: []reactor{ { @@ -687,7 +898,23 @@ func TestExpose(t *testing.T) { }, }, }, - expectedAffinity: nil, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, expectedPVCAnnotation: nil, }, { @@ -714,8 +941,25 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, - expectedAffinity: nil, expectedPVCAnnotation: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, { @@ -744,6 +988,7 @@ func TestExpose(t *testing.T) { daemonSet, volumeAttachement1, volumeAttachement2, + scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ @@ -751,6 +996,11 @@ func TestExpose(t *testing.T) { NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, { Key: "kubernetes.io/hostname", Operator: corev1api.NodeSelectorOpNotIn, @@ -844,6 +1094,8 @@ func TestExpose(t *testing.T) { if test.expectedAffinity != nil { assert.Equal(t, test.expectedAffinity, backupPod.Spec.Affinity) + } else { + assert.Nil(t, backupPod.Spec.Affinity) } if test.expectedPVCAnnotation != nil { diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index c10370072..186815d77 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -493,13 +493,13 @@ func (e *genericRestoreExposer) createRestorePod( containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) - var podAffinity *corev1api.Affinity - if selectedNode == "" { - e.log.Infof("No selected node for restore pod. Try to get affinity from the node-agent config.") + if selectedNode != "" { + affinity = nil + e.log.Infof("Selected node for restore pod. Ignore affinity from the node-agent config.") + } - if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) - } + if affinity == nil { + affinity = &kube.LoadAffinity{} } podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) @@ -576,9 +576,14 @@ func (e *genericRestoreExposer) createRestorePod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -599,10 +604,17 @@ func (e *genericRestoreExposer) createRestorePod( RunAsUser: &userID, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } + podAffinity := kube.ToSystemAffinity(affinity, nil) + pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: restorePodName, diff --git a/pkg/exposer/pod_volume.go b/pkg/exposer/pod_volume.go index 246d04e28..aeb6f1903 100644 --- a/pkg/exposer/pod_volume.go +++ b/pkg/exposer/pod_volume.go @@ -434,6 +434,8 @@ func (e *podVolumeExposer) createHostingPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + affinity := &kube.LoadAffinity{} + var securityCtx *corev1api.PodSecurityContext var containerSecurityCtx *corev1api.SecurityContext nodeSelector := map[string]string{} @@ -446,9 +448,14 @@ func (e *podVolumeExposer) createHostingPod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -472,10 +479,17 @@ func (e *podVolumeExposer) createHostingPod( Privileged: &privileged, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } + podAffinity := kube.ToSystemAffinity(affinity, nil) + pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: hostingPodName, @@ -495,6 +509,7 @@ func (e *podVolumeExposer) createHostingPod( Spec: corev1api.PodSpec{ NodeSelector: nodeSelector, OS: &podOS, + Affinity: podAffinity, Containers: []corev1api.Container{ { Name: containerName, diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index ee63f3736..771114e82 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -235,12 +235,28 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet if c.forWindows { daemonSet.Spec.Template.Spec.SecurityContext = nil daemonSet.Spec.Template.Spec.Containers[0].SecurityContext = nil - daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/os": "windows", - } daemonSet.Spec.Template.Spec.OS = &corev1api.PodOS{ Name: "windows", } + + daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpIn, + }, + }, + }, + }, + }, + }, + } + daemonSet.Spec.Template.Spec.Tolerations = []corev1api.Toleration{ { Key: "os", @@ -256,11 +272,22 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet }, } } else { - daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/os": "linux", - } - daemonSet.Spec.Template.Spec.OS = &corev1api.PodOS{ - Name: "linux", + daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, } } diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 139d3dcd0..0f4de11bd 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -34,8 +34,23 @@ func TestDaemonSet(t *testing.T) { assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) - assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) - assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, corev1api.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, corev1api.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) assert.Len(t, ds.Spec.Template.Spec.Volumes, 3) @@ -80,8 +95,24 @@ func TestDaemonSet(t *testing.T) { assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) - assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpIn, + }, + }, + }, + }, + }, + }, + }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, (*corev1api.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, (*corev1api.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext) } diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d1010d294..04ea40e04 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -364,12 +364,26 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme Spec: corev1api.PodSpec{ RestartPolicy: corev1api.RestartPolicyAlways, ServiceAccountName: c.serviceAccountName, - NodeSelector: map[string]string{ - "kubernetes.io/os": "linux", - }, OS: &corev1api.PodOS{ Name: "linux", }, + Affinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, Containers: []corev1api.Container{ { Name: "velero", diff --git a/pkg/install/deployment_test.go b/pkg/install/deployment_test.go index b8aeaa9dd..53b696f72 100644 --- a/pkg/install/deployment_test.go +++ b/pkg/install/deployment_test.go @@ -100,8 +100,23 @@ func TestDeployment(t *testing.T) { assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) - assert.Equal(t, "linux", deploy.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) - assert.Equal(t, "linux", string(deploy.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, deploy.Spec.Template.Spec.Affinity) } func TestDeploymentWithPriorityClassName(t *testing.T) { diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index fba3cb19a..1747f1b33 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -210,11 +210,9 @@ func resultsKey(ns, name string) string { func (b *backupper) getMatchAction(resPolicies *resourcepolicies.Policies, pvc *corev1api.PersistentVolumeClaim, volume *corev1api.Volume) (*resourcepolicies.Action, error) { if pvc != nil { - pv := new(corev1api.PersistentVolume) - err := b.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv) - if err != nil { - return nil, errors.Wrapf(err, "error getting pv for pvc %s", pvc.Spec.VolumeName) - } + // Ignore err, if the PV is not available (Pending/Lost PVC or PV fetch failed) - try matching with PVC only + // GetPVForPVC returns nil for all error cases + pv, _ := kube.GetPVForPVC(pvc, b.crClient) vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return resPolicies.GetMatchAction(vfd) } diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 6359df696..846f65796 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -309,8 +309,8 @@ func createNodeObj() *corev1api.Node { func TestBackupPodVolumes(t *testing.T) { scheme := runtime.NewScheme() - velerov1api.AddToScheme(scheme) - corev1api.AddToScheme(scheme) + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) log := logrus.New() tests := []struct { @@ -778,7 +778,7 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{}) if c.pvb != nil { - backuper.pvbIndexer.Add(c.pvb) + require.NoError(t, backuper.pvbIndexer.Add(c.pvb)) backuper.wg.Add(1) } @@ -833,3 +833,185 @@ func TestPVCBackupSummary(t *testing.T) { assert.Empty(t, pbs.Skipped) assert.Len(t, pbs.Backedup, 2) } + +func TestGetMatchAction_PendingPVC(t *testing.T) { + // Create resource policies that skip Pending/Lost PVCs + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + volume *corev1api.Volume + pv *corev1api.PersistentVolume + expectedAction *resourcepolicies.Action + expectError bool + }{ + { + name: "Pending PVC with pvcPhase skip policy should return skip action", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Lost PVC with pvcPhase skip policy should return skip action", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimLost). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "lost-pvc", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Bound PVC with matching PV should not match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + StorageClass("test-sc"). + VolumeName("test-pv"). + Phase(corev1api.ClaimBound). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "bound-pvc", + }, + }, + }, + pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), + expectedAction: nil, + expectError: false, + }, + { + name: "Pending PVC with no matching policy should return nil action", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc-no-match"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc-no-match", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, // Will match the pvcPhase policy + expectError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Build fake client with PV if present + var objs []runtime.Object + if tc.pv != nil { + objs = append(objs, tc.pv) + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + b := &backupper{ + crClient: fakeClient, + } + + action, err := b.getMatchAction(policies, tc.pvc, tc.volume) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tc.expectedAction == nil { + assert.Nil(t, action) + } else { + require.NotNil(t, action) + assert.Equal(t, tc.expectedAction.Type, action.Type) + } + }) + } +} + +func TestGetMatchAction_PVCWithoutPVLookupError(t *testing.T) { + // Test that when a PVC has a VolumeName but the PV doesn't exist, + // the function ignores the error and tries to match with PVC only + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + // Pending PVC without a matching PV in the cluster + pvc := builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result() + + volume := &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc", + }, + }, + } + + // Empty client - no PV exists + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + b := &backupper{ + crClient: fakeClient, + } + + // Should succeed even though PV lookup would fail + // because the function ignores PV lookup errors and uses PVC-only matching + action, err := b.getMatchAction(policies, pvc, volume) + require.NoError(t, err) + require.NotNil(t, action) + assert.Equal(t, resourcepolicies.Skip, action.Type) +} diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 496d07703..426cb44d4 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -671,8 +671,7 @@ func buildJob( } if config != nil && len(config.LoadAffinities) > 0 { - // Maintenance job only takes the first loadAffinity. - affinity := kube.ToSystemAffinity([]*kube.LoadAffinity{config.LoadAffinities[0]}) + affinity := kube.ToSystemAffinity(config.LoadAffinities[0], nil) job.Spec.Template.Spec.Affinity = affinity } diff --git a/pkg/util/collections/includes_excludes.go b/pkg/util/collections/includes_excludes.go index b3d18d068..ab63eaa72 100644 --- a/pkg/util/collections/includes_excludes.go +++ b/pkg/util/collections/includes_excludes.go @@ -666,10 +666,22 @@ func validateNamespaceName(ns string) []error { return nil } - // Kubernetes does not allow asterisks in namespaces but Velero uses them as - // wildcards. Replace asterisks with an arbitrary letter to pass Kubernetes - // validation. - tmpNamespace := strings.ReplaceAll(ns, "*", "x") + // Validate the namespace name to ensure it is a valid wildcard pattern + if err := wildcard.ValidateNamespaceName(ns); err != nil { + return []error{err} + } + + // Kubernetes does not allow wildcard characters in namespaces but Velero uses them + // for glob patterns. Replace wildcard characters with valid characters to pass + // Kubernetes validation. + tmpNamespace := ns + + // Replace glob wildcard characters with valid alphanumeric characters + // Note: Validation of wildcard patterns is handled by the wildcard package. + tmpNamespace = strings.ReplaceAll(tmpNamespace, "*", "x") // matches any sequence + tmpNamespace = strings.ReplaceAll(tmpNamespace, "?", "x") // matches single character + tmpNamespace = strings.ReplaceAll(tmpNamespace, "[", "x") // character class start + tmpNamespace = strings.ReplaceAll(tmpNamespace, "]", "x") // character class end if errMsgs := validation.ValidateNamespaceName(tmpNamespace, false); errMsgs != nil { for _, msg := range errMsgs { diff --git a/pkg/util/collections/includes_excludes_test.go b/pkg/util/collections/includes_excludes_test.go index b7fdcd3af..1d700a729 100644 --- a/pkg/util/collections/includes_excludes_test.go +++ b/pkg/util/collections/includes_excludes_test.go @@ -289,6 +289,54 @@ func TestValidateNamespaceIncludesExcludes(t *testing.T) { excludes: []string{"bar"}, wantErr: true, }, + { + name: "glob characters in includes should not error", + includes: []string{"kube-*", "test-?", "ns-[0-9]"}, + excludes: []string{}, + wantErr: false, + }, + { + name: "glob characters in excludes should not error", + includes: []string{"default"}, + excludes: []string{"test-*", "app-?", "ns-[1-5]"}, + wantErr: false, + }, + { + name: "character class in includes should not error", + includes: []string{"ns-[abc]", "test-[0-9]"}, + excludes: []string{}, + wantErr: false, + }, + { + name: "mixed glob patterns should not error", + includes: []string{"kube-*", "test-?"}, + excludes: []string{"*-test", "debug-[0-9]"}, + wantErr: false, + }, + { + name: "pipe character in includes should error", + includes: []string{"namespace|other"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "parentheses in includes should error", + includes: []string{"namespace(prod)", "test-(dev)"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "exclamation mark in includes should error", + includes: []string{"!namespace", "test!"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "unsupported characters in excludes should error", + includes: []string{"default"}, + excludes: []string{"test|prod", "app(staging)"}, + wantErr: true, + }, } for _, tc := range tests { @@ -1082,16 +1130,6 @@ func TestExpandIncludesExcludes(t *testing.T) { expectedWildcardExpanded: true, expectError: false, }, - { - name: "brace wildcard pattern", - includes: []string{"app-{prod,dev}"}, - excludes: []string{}, - activeNamespaces: []string{"app-prod", "app-dev", "app-test", "default"}, - expectedIncludes: []string{"app-prod", "app-dev"}, - expectedExcludes: []string{}, - expectedWildcardExpanded: true, - expectError: false, - }, { name: "empty activeNamespaces with wildcards", includes: []string{"kube-*"}, @@ -1233,13 +1271,6 @@ func TestResolveNamespaceList(t *testing.T) { expectedNamespaces: []string{"kube-system", "kube-public"}, preExpandWildcards: true, }, - { - name: "complex wildcard pattern", - includes: []string{"app-{prod,dev}", "kube-*"}, - excludes: []string{"*-test"}, - activeNamespaces: []string{"app-prod", "app-dev", "app-test", "kube-system", "kube-test", "default"}, - expectedNamespaces: []string{"app-prod", "app-dev", "kube-system"}, - }, { name: "question mark wildcard pattern", includes: []string{"ns-?"}, diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index da68183a5..ba6853624 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -17,7 +17,6 @@ package kube import ( "context" - "fmt" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,6 +33,11 @@ const ( NodeOSLabel = "kubernetes.io/os" ) +var realNodeOSMap = map[string]string{ + "linux": NodeOSLinux, + "windows": NodeOSWindows, +} + func IsLinuxNode(ctx context.Context, nodeName string, client client.Client) error { node := &corev1api.Node{} if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { @@ -41,12 +45,11 @@ func IsLinuxNode(ctx context.Context, nodeName string, client client.Client) err } os, found := node.Labels[NodeOSLabel] - if !found { return errors.Errorf("no os type label for node %s", nodeName) } - if os != NodeOSLinux { + if getRealOS(os) != NodeOSLinux { return errors.Errorf("os type %s for node %s is not linux", os, nodeName) } @@ -72,7 +75,7 @@ func withOSNode(ctx context.Context, client client.Client, osType string, log lo for _, node := range nodeList.Items { os, found := node.Labels[NodeOSLabel] - if os == osType { + if getRealOS(os) == osType { return true } @@ -98,7 +101,7 @@ func GetNodeOS(ctx context.Context, nodeName string, nodeClient corev1client.Cor return "", nil } - return node.Labels[NodeOSLabel], nil + return getRealOS(node.Labels[NodeOSLabel]), nil } func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV1Interface) error { @@ -106,14 +109,29 @@ func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV return errors.New("invalid node OS") } - nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", NodeOSLabel, os)}) + nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrapf(err, "error listing nodes with OS %s", os) } - if len(nodes.Items) == 0 { - return errors.Errorf("node with OS %s doesn't exist", os) + for _, node := range nodes.Items { + osLabel, found := node.Labels[NodeOSLabel] + if !found { + continue + } + + if getRealOS(osLabel) == os { + return nil + } } - return nil + return errors.Errorf("node with OS %s doesn't exist", os) +} + +func getRealOS(osLabel string) string { + if os, found := realNodeOSMap[osLabel]; !found { + return NodeOSLinux + } else { + return os + } } diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 2aeb45a1c..a57e2cea9 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -230,14 +230,9 @@ func CollectPodLogs(ctx context.Context, podGetter corev1client.CoreV1Interface, return nil } -func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { - if len(loadAffinities) == 0 { - return nil - } - nodeSelectorTermList := make([]corev1api.NodeSelectorTerm, 0) - - for _, loadAffinity := range loadAffinities { - requirements := []corev1api.NodeSelectorRequirement{} +func ToSystemAffinity(loadAffinity *LoadAffinity, volumeTopology *corev1api.NodeSelector) *corev1api.Affinity { + requirements := []corev1api.NodeSelectorRequirement{} + if loadAffinity != nil { for k, v := range loadAffinity.NodeSelector.MatchLabels { requirements = append(requirements, corev1api.NodeSelectorRequirement{ Key: k, @@ -253,25 +248,25 @@ func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { Operator: corev1api.NodeSelectorOperator(exp.Operator), }) } - - nodeSelectorTermList = append( - nodeSelectorTermList, - corev1api.NodeSelectorTerm{ - MatchExpressions: requirements, - }, - ) } - if len(nodeSelectorTermList) > 0 { - result := new(corev1api.Affinity) - result.NodeAffinity = new(corev1api.NodeAffinity) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTermList + result := new(corev1api.Affinity) + result.NodeAffinity = new(corev1api.NodeAffinity) + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - return result + if volumeTopology != nil { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, volumeTopology.NodeSelectorTerms...) + } else if len(requirements) > 0 { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = make([]corev1api.NodeSelectorTerm, 1) + } else { + return nil } - return nil + for i := range result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions, requirements...) + } + + return result } func DiagnosePod(pod *corev1api.Pod, events *corev1api.EventList) string { diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 6751e8b6e..aa8d4db99 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -747,24 +747,23 @@ func TestCollectPodLogs(t *testing.T) { func TestToSystemAffinity(t *testing.T) { tests := []struct { name string - loadAffinities []*LoadAffinity + loadAffinity *LoadAffinity + volumeTopology *corev1api.NodeSelector expected *corev1api.Affinity }{ { name: "loadAffinity is nil", }, { - name: "loadAffinity is empty", - loadAffinities: []*LoadAffinity{}, + name: "loadAffinity is empty", + loadAffinity: &LoadAffinity{}, }, { name: "with match label", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", - }, + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-1": "value-1", }, }, }, @@ -788,23 +787,21 @@ func TestToSystemAffinity(t *testing.T) { }, { name: "with match expression", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "key-3", - Values: []string{"value-3-1", "value-3-2"}, - Operator: metav1.LabelSelectorOpNotIn, - }, - { - Key: "key-4", - Values: []string{"value-4-1", "value-4-2", "value-4-3"}, - Operator: metav1.LabelSelectorOpDoesNotExist, - }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, }, }, }, @@ -838,19 +835,49 @@ func TestToSystemAffinity(t *testing.T) { }, }, { - name: "multiple load affinities", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", + name: "with olume topology", + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, - }, - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -862,10 +889,177 @@ func TestToSystemAffinity(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "key-1", - Values: []string{"value-1"}, + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "with match expression and volume topology", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + }, + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + expected: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, + Operator: corev1api.NodeSelectorOpIn, + }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, }, }, { @@ -875,6 +1069,28 @@ func TestToSystemAffinity(t *testing.T) { Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -886,7 +1102,7 @@ func TestToSystemAffinity(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - affinity := ToSystemAffinity(test.loadAffinities) + affinity := ToSystemAffinity(test.loadAffinity, test.volumeTopology) assert.True(t, reflect.DeepEqual(affinity, test.expected)) }) } diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 786cef2a5..d5d2e2041 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -417,19 +417,19 @@ func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVol return volumeMounts, volumeDevices, volumePath } +// GetPVForPVC returns the PersistentVolume backing a PVC +// returns PV, error. +// PV will be nil on error func GetPVForPVC( pvc *corev1api.PersistentVolumeClaim, crClient crclient.Client, ) (*corev1api.PersistentVolume, error) { if pvc.Spec.VolumeName == "" { - return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", - pvc.Namespace, pvc.Name) + return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", pvc.Namespace, pvc.Name) } if pvc.Status.Phase != corev1api.ClaimBound { - // TODO: confirm if this PVC should be snapshotted if it has no PV bound - return nil, - errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", - pvc.Namespace, pvc.Name, pvc.Status.Phase) + return nil, errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", + pvc.Namespace, pvc.Name, pvc.Status.Phase) } pv := &corev1api.PersistentVolume{} @@ -580,3 +580,29 @@ func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1. return nodes, nil } + +func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) { + if pvName == "" || scName == "" { + return nil, errors.Errorf("invalid parameter, pv %s, sc %s", pvName, scName) + } + + sc, err := storageClient.StorageClasses().Get(ctx, scName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting storage class %s", scName) + } + + if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != storagev1api.VolumeBindingWaitForFirstConsumer { + return nil, nil + } + + pv, err := volumeClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting PV %s", pvName) + } + + if pv.Spec.NodeAffinity == nil { + return nil, nil + } + + return pv.Spec.NodeAffinity.Required, nil +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index d94efa62e..63b8e1edd 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1909,3 +1909,143 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { }) } } + +func TestGetVolumeTopology(t *testing.T) { + pvWithoutNodeAffinity := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + } + + pvWithNodeAffinity := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + NodeAffinity: &corev1api.VolumeNodeAffinity{ + Required: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "fake-key", + }, + }, + }, + }, + }, + }, + }, + } + + scObjWithoutVolumeBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + } + + volumeBindImmediate := storagev1api.VolumeBindingImmediate + scObjWithImeediateBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + VolumeBindingMode: &volumeBindImmediate, + } + + volumeBindWffc := storagev1api.VolumeBindingWaitForFirstConsumer + scObjWithWffcBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + VolumeBindingMode: &volumeBindWffc, + } + + tests := []struct { + name string + pvName string + scName string + kubeClientObj []runtime.Object + expectedErr string + expected *corev1api.NodeSelector + }{ + { + name: "invalid pvName", + scName: "fake-storage-class", + expectedErr: "invalid parameter, pv , sc fake-storage-class", + }, + { + name: "invalid scName", + pvName: "fake-pv", + expectedErr: "invalid parameter, pv fake-pv, sc ", + }, + { + name: "no sc", + pvName: "fake-pv", + scName: "fake-storage-class", + expectedErr: "error getting storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found", + }, + { + name: "sc without binding mode", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithoutVolumeBind}, + }, + { + name: "sc without immediate binding mode", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithImeediateBind}, + }, + { + name: "get pv fail", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithWffcBind}, + expectedErr: "error getting PV fake-pv: persistentvolumes \"fake-pv\" not found", + }, + { + name: "pv with no affinity", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{ + scObjWithWffcBind, + pvWithoutNodeAffinity, + }, + }, + { + name: "pv with affinity", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{ + scObjWithWffcBind, + pvWithNodeAffinity, + }, + expected: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "fake-key", + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + + var kubeClient kubernetes.Interface = fakeKubeClient + + affinity, err := GetVolumeTopology(t.Context(), kubeClient.CoreV1(), kubeClient.StorageV1(), test.pvName, test.scName) + + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + assert.Equal(t, test.expected, affinity) + } + }) + } +} diff --git a/pkg/util/wildcard/expand.go b/pkg/util/wildcard/expand.go index 632e05aa1..8767c8ed3 100644 --- a/pkg/util/wildcard/expand.go +++ b/pkg/util/wildcard/expand.go @@ -31,70 +31,77 @@ func ShouldExpandWildcards(includes []string, excludes []string) bool { } // containsWildcardPattern checks if a pattern contains any wildcard symbols -// Supported patterns: *, ?, [abc], {a,b,c} +// Supported patterns: *, ?, [abc] // Note: . and + are treated as literal characters (not wildcards) // Note: ** and consecutive asterisks are NOT supported (will cause validation error) func containsWildcardPattern(pattern string) bool { - return strings.ContainsAny(pattern, "*?[{") + return strings.ContainsAny(pattern, "*?[") } func validateWildcardPatterns(patterns []string) error { for _, pattern := range patterns { - // Check for invalid regex-only patterns that we don't support - if strings.ContainsAny(pattern, "|()") { - return errors.New("wildcard pattern contains unsupported regex symbols: |, (, )") - } - - // Check for consecutive asterisks (2 or more) - if strings.Contains(pattern, "**") { - return errors.New("wildcard pattern contains consecutive asterisks (only single * allowed)") - } - - // Check for malformed brace patterns - if err := validateBracePatterns(pattern); err != nil { + if err := ValidateNamespaceName(pattern); err != nil { return err } } return nil } +func ValidateNamespaceName(pattern string) error { + // Check for invalid characters that are not supported in glob patterns + if strings.ContainsAny(pattern, "|()!{},") { + return errors.New("wildcard pattern contains unsupported characters: |, (, ), !, {, }, ,") + } + + // Check for consecutive asterisks (2 or more) + if strings.Contains(pattern, "**") { + return errors.New("wildcard pattern contains consecutive asterisks (only single * allowed)") + } + + // Check for malformed brace patterns + if err := validateBracePatterns(pattern); err != nil { + return err + } + + return nil +} + // validateBracePatterns checks for malformed brace patterns like unclosed braces or empty braces +// Also validates bracket patterns [] for character classes func validateBracePatterns(pattern string) error { - depth := 0 + bracketDepth := 0 for i := 0; i < len(pattern); i++ { - if pattern[i] == '{' { - braceStart := i - depth++ + if pattern[i] == '[' { + bracketStart := i + bracketDepth++ - // Scan ahead to find the matching closing brace and validate content - for j := i + 1; j < len(pattern) && depth > 0; j++ { - if pattern[j] == '{' { - depth++ - } else if pattern[j] == '}' { - depth-- - if depth == 0 { - // Found matching closing brace - validate content - content := pattern[braceStart+1 : j] - if strings.Trim(content, ", \t") == "" { - return errors.New("wildcard pattern contains empty brace pattern '{}'") + // Scan ahead to find the matching closing bracket and validate content + for j := i + 1; j < len(pattern) && bracketDepth > 0; j++ { + if pattern[j] == ']' { + bracketDepth-- + if bracketDepth == 0 { + // Found matching closing bracket - validate content + content := pattern[bracketStart+1 : j] + if content == "" { + return errors.New("wildcard pattern contains empty bracket pattern '[]'") } - // Skip to the closing brace + // Skip to the closing bracket i = j break } } } - // If we exited the loop without finding a match (depth > 0), brace is unclosed - if depth > 0 { - return errors.New("wildcard pattern contains unclosed brace '{'") + // If we exited the loop without finding a match (bracketDepth > 0), bracket is unclosed + if bracketDepth > 0 { + return errors.New("wildcard pattern contains unclosed bracket '['") } - // i is now positioned at the closing brace; the outer loop will increment it - } else if pattern[i] == '}' { - // Found a closing brace without a matching opening brace - return errors.New("wildcard pattern contains unmatched closing brace '}'") + // i is now positioned at the closing bracket; the outer loop will increment it + } else if pattern[i] == ']' { + // Found a closing bracket without a matching opening bracket + return errors.New("wildcard pattern contains unmatched closing bracket ']'") } } diff --git a/pkg/util/wildcard/expand_test.go b/pkg/util/wildcard/expand_test.go index f6c7ed434..317020648 100644 --- a/pkg/util/wildcard/expand_test.go +++ b/pkg/util/wildcard/expand_test.go @@ -90,7 +90,7 @@ func TestShouldExpandWildcards(t *testing.T) { name: "brace alternatives wildcard", includes: []string{"ns{prod,staging}"}, excludes: []string{}, - expected: true, // brace alternatives are considered wildcard + expected: false, // brace alternatives are not supported }, { name: "dot is literal - not wildcard", @@ -237,9 +237,9 @@ func TestExpandWildcards(t *testing.T) { activeNamespaces: []string{"app-prod", "app-staging", "app-dev", "db-prod"}, includes: []string{"app-{prod,staging}"}, excludes: []string{}, - expectedIncludes: []string{"app-prod", "app-staging"}, // {prod,staging} matches either + expectedIncludes: nil, expectedExcludes: nil, - expectError: false, + expectError: true, }, { name: "literal dot and plus patterns", @@ -259,33 +259,6 @@ func TestExpandWildcards(t *testing.T) { expectedExcludes: nil, expectError: true, // |, (, ) are not supported }, - { - name: "unclosed brace patterns should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-{prod,staging"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // unclosed brace - }, - { - name: "empty brace patterns should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-{}"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // empty braces - }, - { - name: "unmatched closing brace should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-prod}"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // unmatched closing brace - }, } for _, tt := range tests { @@ -354,13 +327,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: []string{}, // returns empty slice, not nil expectError: false, }, - { - name: "brace patterns work correctly", - patterns: []string{"app-{prod,staging}"}, - activeNamespaces: []string{"app-prod", "app-staging", "app-dev", "app-{prod,staging}"}, - expected: []string{"app-prod", "app-staging"}, // brace patterns do expand - expectError: false, - }, { name: "duplicate matches from multiple patterns", patterns: []string{"app-*", "*-prod"}, @@ -389,20 +355,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: []string{"nsa", "nsb", "nsc"}, // [a-c] matches a to c expectError: false, }, - { - name: "negated character class", - patterns: []string{"ns[!abc]"}, - activeNamespaces: []string{"nsa", "nsb", "nsc", "nsd", "ns1"}, - expected: []string{"nsd", "ns1"}, // [!abc] matches anything except a, b, c - expectError: false, - }, - { - name: "brace alternatives", - patterns: []string{"app-{prod,test}"}, - activeNamespaces: []string{"app-prod", "app-test", "app-staging", "db-prod"}, - expected: []string{"app-prod", "app-test"}, // {prod,test} matches either - expectError: false, - }, { name: "double asterisk should error", patterns: []string{"**"}, @@ -410,13 +362,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: nil, expectError: true, // ** is not allowed }, - { - name: "literal dot and plus", - patterns: []string{"app.prod", "service+"}, - activeNamespaces: []string{"app.prod", "appXprod", "service+", "service"}, - expected: []string{"app.prod", "service+"}, // . and + are literal - expectError: false, - }, { name: "unsupported regex symbols should error", patterns: []string{"ns(1|2)"}, @@ -468,153 +413,101 @@ func TestValidateBracePatterns(t *testing.T) { expectError bool errorMsg string }{ - // Valid patterns + // Valid square bracket patterns { - name: "valid single brace pattern", - pattern: "app-{prod,staging}", + name: "valid square bracket pattern", + pattern: "ns[abc]", expectError: false, }, { - name: "valid brace with single option", - pattern: "app-{prod}", + name: "valid square bracket pattern with range", + pattern: "ns[a-z]", expectError: false, }, { - name: "valid brace with three options", - pattern: "app-{prod,staging,dev}", + name: "valid square bracket pattern with numbers", + pattern: "ns[0-9]", expectError: false, }, { - name: "valid pattern with text before and after brace", - pattern: "prefix-{a,b}-suffix", + name: "valid square bracket pattern with mixed", + pattern: "ns[a-z0-9]", expectError: false, }, { - name: "valid pattern with no braces", - pattern: "app-prod", + name: "valid square bracket pattern with single character", + pattern: "ns[a]", expectError: false, }, { - name: "valid pattern with asterisk", - pattern: "app-*", + name: "valid square bracket pattern with text before and after", + pattern: "prefix-[abc]-suffix", expectError: false, }, + // Unclosed opening brackets { - name: "valid brace with spaces around content", - pattern: "app-{ prod , staging }", - expectError: false, + name: "unclosed opening bracket at end", + pattern: "ns[abc", + expectError: true, + errorMsg: "unclosed bracket", }, { - name: "valid brace with numbers", - pattern: "ns-{1,2,3}", - expectError: false, + name: "unclosed opening bracket at start", + pattern: "[abc", + expectError: true, + errorMsg: "unclosed bracket", }, { - name: "valid brace with hyphens in options", - pattern: "{app-prod,db-staging}", - expectError: false, + name: "unclosed opening bracket in middle", + pattern: "ns[abc-test", + expectError: true, + errorMsg: "unclosed bracket", }, - // Unclosed opening braces + // Unmatched closing brackets { - name: "unclosed opening brace at end", - pattern: "app-{prod,staging", + name: "unmatched closing bracket at end", + pattern: "ns-abc]", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "unclosed opening brace at start", - pattern: "{prod,staging", + name: "unmatched closing bracket at start", + pattern: "]ns-abc", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "unclosed opening brace in middle", - pattern: "app-{prod-test", + name: "unmatched closing bracket in middle", + pattern: "ns-]abc", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "multiple unclosed braces", - pattern: "app-{prod-{staging", + name: "extra closing bracket after valid pair", + pattern: "ns[abc]]", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, - // Unmatched closing braces + // Empty bracket patterns { - name: "unmatched closing brace at end", - pattern: "app-prod}", + name: "completely empty brackets", + pattern: "ns[]", expectError: true, - errorMsg: "unmatched closing brace", + errorMsg: "empty bracket pattern", }, { - name: "unmatched closing brace at start", - pattern: "}app-prod", + name: "empty brackets at start", + pattern: "[]ns", expectError: true, - errorMsg: "unmatched closing brace", + errorMsg: "empty bracket pattern", }, { - name: "unmatched closing brace in middle", - pattern: "app-}prod", + name: "empty brackets standalone", + pattern: "[]", expectError: true, - errorMsg: "unmatched closing brace", - }, - { - name: "extra closing brace after valid pair", - pattern: "app-{prod,staging}}", - expectError: true, - errorMsg: "unmatched closing brace", - }, - - // Empty brace patterns - { - name: "completely empty braces", - pattern: "app-{}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only spaces", - pattern: "app-{ }", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only comma", - pattern: "app-{,}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only commas", - pattern: "app-{,,,}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with commas and spaces", - pattern: "app-{ , , }", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with tabs and commas", - pattern: "app-{\t,\t}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "empty braces at start", - pattern: "{}app-prod", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "empty braces standalone", - pattern: "{}", - expectError: true, - errorMsg: "empty brace pattern", + errorMsg: "empty bracket pattern", }, // Edge cases @@ -623,58 +516,6 @@ func TestValidateBracePatterns(t *testing.T) { pattern: "", expectError: false, }, - { - name: "pattern with only opening brace", - pattern: "{", - expectError: true, - errorMsg: "unclosed brace", - }, - { - name: "pattern with only closing brace", - pattern: "}", - expectError: true, - errorMsg: "unmatched closing brace", - }, - { - name: "valid brace with special characters inside", - pattern: "app-{prod-1,staging_2,dev.3}", - expectError: false, - }, - { - name: "brace with asterisk inside option", - pattern: "app-{prod*,staging}", - expectError: false, - }, - { - name: "multiple valid brace patterns", - pattern: "{app,db}-{prod,staging}", - expectError: false, - }, - { - name: "brace with single character", - pattern: "app-{a}", - expectError: false, - }, - { - name: "brace with trailing comma but has content", - pattern: "app-{prod,staging,}", - expectError: false, // Has content, so it's valid - }, - { - name: "brace with leading comma but has content", - pattern: "app-{,prod,staging}", - expectError: false, // Has content, so it's valid - }, - { - name: "brace with leading comma but has content", - pattern: "app-{{,prod,staging}", - expectError: true, // unclosed brace - }, - { - name: "brace with leading comma but has content", - pattern: "app-{,prod,staging}}", - expectError: true, // unmatched closing brace - }, } for _, tt := range tests { @@ -723,20 +564,6 @@ func TestExpandWildcardsEdgeCases(t *testing.T) { assert.ElementsMatch(t, []string{"ns-1", "ns_2", "ns.3", "ns@4"}, result) }) - t.Run("complex glob combinations", func(t *testing.T) { - activeNamespaces := []string{"app1-prod", "app2-prod", "app1-test", "db-prod", "service"} - result, err := expandWildcards([]string{"app?-{prod,test}"}, activeNamespaces) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"app1-prod", "app2-prod", "app1-test"}, result) - }) - - t.Run("escaped characters", func(t *testing.T) { - activeNamespaces := []string{"app*", "app-prod", "app?test", "app-test"} - result, err := expandWildcards([]string{"app\\*"}, activeNamespaces) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"app*"}, result) - }) - t.Run("mixed literal and wildcard patterns", func(t *testing.T) { activeNamespaces := []string{"app.prod", "app-prod", "app_prod", "test.ns"} result, err := expandWildcards([]string{"app.prod", "app?prod"}, activeNamespaces) @@ -777,12 +604,8 @@ func TestExpandWildcardsEdgeCases(t *testing.T) { shouldError bool }{ {"unclosed bracket", "ns[abc", true}, - {"unclosed brace", "app-{prod,staging", true}, - {"nested unclosed", "ns[a{bc", true}, {"valid bracket", "ns[abc]", false}, - {"valid brace", "app-{prod,staging}", false}, {"empty bracket", "ns[]", true}, // empty brackets are invalid - {"empty brace", "app-{}", true}, // empty braces are invalid } for _, tt := range tests { diff --git a/site/config.yaml b/site/config.yaml index c7f25f9ad..ed80914a4 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -15,6 +15,7 @@ params: latest: v1.17 versions: - main + - v1.18 - v1.17 - v1.16 - v1.15 diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index 1768f0934..3bad516e3 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -16,6 +16,8 @@ Backup belongs to the API group version `velero.io/v1`. Here is a sample `Backup` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -42,11 +44,12 @@ spec: resourcePolicy: kind: configmap name: resource-policy-configmap - # Array of namespaces to include in the backup. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the backup. Accepts glob patterns (*, ?, [abc]). + # Note: '*' alone is reserved for empty fields, which means all namespaces. + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the backup. Optional. + # Array of namespaces to exclude from the backup. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/main/api-types/restore.md b/site/content/docs/main/api-types/restore.md index ec3e19511..1c80a0ee8 100644 --- a/site/content/docs/main/api-types/restore.md +++ b/site/content/docs/main/api-types/restore.md @@ -16,6 +16,8 @@ Restore belongs to the API group version `velero.io/v1`. Here is a sample `Restore` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -45,11 +47,11 @@ spec: writeSparseFiles: true # ParallelFilesDownload is the concurrency number setting for restore parallelFilesDownload: 10 - # Array of namespaces to include in the restore. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the restore. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the restore. Optional. + # Array of namespaces to exclude from the restore. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/main/namespace-glob-patterns.md b/site/content/docs/main/namespace-glob-patterns.md new file mode 100644 index 000000000..4695124ea --- /dev/null +++ b/site/content/docs/main/namespace-glob-patterns.md @@ -0,0 +1,71 @@ +--- +title: "Namespace Glob Patterns" +layout: docs +--- + +When using `--include-namespaces` and `--exclude-namespaces` flags with backup and restore commands, you can use glob patterns to match multiple namespaces. + +## Supported Patterns + +Velero supports the following glob pattern characters: + +- `*` - Matches any sequence of characters + ```bash + velero backup create my-backup --include-namespaces "app-*" + # Matches: app-prod, app-staging, app-dev, etc. + ``` + +- `?` - Matches exactly one character + ```bash + velero backup create my-backup --include-namespaces "ns?" + # Matches: ns1, ns2, nsa, but NOT ns10 + ``` + +- `[abc]` - Matches any single character in the brackets + ```bash + velero backup create my-backup --include-namespaces "ns[123]" + # Matches: ns1, ns2, ns3 + ``` + +- `[a-z]` - Matches any single character in the range + ```bash + velero backup create my-backup --include-namespaces "ns[a-c]" + # Matches: nsa, nsb, nsc + ``` + +## Unsupported Patterns + +The following patterns are **not supported** and will cause validation errors: + +- `**` - Consecutive asterisks +- `|` - Alternation (regex operator) +- `()` - Grouping (regex operators) +- `!` - Negation +- `{}` - Brace expansion +- `,` - Comma (used in brace expansion) + +## Special Cases + +- `*` alone means "all namespaces" and is not expanded +- Empty brackets `[]` are invalid +- Unmatched or unclosed brackets will cause validation errors + +## Examples + +Combine patterns with include and exclude flags: + +```bash +# Backup all production namespaces except test +velero backup create prod-backup \ + --include-namespaces "*-prod" \ + --exclude-namespaces "test-*" + +# Backup specific numbered namespaces +velero backup create numbered-backup \ + --include-namespaces "app-[0-9]" + +# Restore namespaces matching multiple patterns +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces "frontend-*,backend-*" +``` diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index a9e65d157..cbfdb2816 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -17,7 +17,11 @@ Wildcard takes precedence when both a wildcard and specific resource are include ### --include-namespaces -Namespaces to include. Default is `*`, all namespaces. +Namespaces to include. Accepts glob patterns (`*`, `?`, `[abc]`). Default is `*`, all namespaces. + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. + +Note: `*` alone is reserved for empty fields, which means all namespaces. * Backup a namespace and it's objects. @@ -158,7 +162,9 @@ Wildcard excludes are ignored. ### --exclude-namespaces -Namespaces to exclude. +Namespaces to exclude. Accepts glob patterns (`*`, `?`, `[abc]`). + +See [Namespace Glob Patterns](namespace-glob-patterns.md) for more details on supported patterns. * Exclude kube-system from the cluster backup. diff --git a/site/content/docs/main/upgrade-to-1.17.md b/site/content/docs/main/upgrade-to-1.18.md similarity index 70% rename from site/content/docs/main/upgrade-to-1.17.md rename to site/content/docs/main/upgrade-to-1.18.md index f6738d55c..0bf839fe3 100644 --- a/site/content/docs/main/upgrade-to-1.17.md +++ b/site/content/docs/main/upgrade-to-1.18.md @@ -1,13 +1,13 @@ --- -title: "Upgrading to Velero 1.17" +title: "Upgrading to Velero 1.18" layout: docs --- ## Prerequisites -- Velero [v1.16.x][9] installed. +- Velero [v1.17.x][9] installed. -If you're not yet running at least Velero v1.16, see the following: +If you're not yet running at least Velero v1.17, see the following: - [Upgrading to v1.8][1] - [Upgrading to v1.9][2] @@ -18,13 +18,14 @@ If you're not yet running at least Velero v1.16, see the following: - [Upgrading to v1.14][7] - [Upgrading to v1.15][8] - [Upgrading to v1.16][9] +- [Upgrading to v1.17][10] Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. ## Instructions -### Upgrade from v1.16 -1. Install the Velero v1.17 command-line interface (CLI) by following the [instructions here][0]. +### Upgrade from v1.17 +1. Install the Velero v1.18 command-line interface (CLI) by following the [instructions here][0]. Verify that you've properly installed it by running: @@ -36,7 +37,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: ``` @@ -46,28 +47,21 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw velero install --crds-only --dry-run -o yaml | kubectl apply -f - ``` -3. (optional) Update the `uploader-type` to `kopia` if you are using `restic`: - ```bash - kubectl get deploy -n velero -ojson \ - | sed "s/\"--uploader-type=restic\"/\"--uploader-type=kopia\"/g" \ - | kubectl apply -f - - ``` - -4. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: +3. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: ```bash # set the container and image of the init container for plugin accordingly, # if you are using other plugin kubectl set image deployment/velero \ - velero=velero/velero:v1.17.0 \ - velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.13.0 \ + velero=velero/velero:v1.18.0 \ + velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.14.0 \ --namespace velero # optional, if using the node agent daemonset kubectl set image daemonset/node-agent \ - node-agent=velero/velero:v1.17.0 \ + node-agent=velero/velero:v1.18.0 \ --namespace velero ``` -5. Confirm that the deployment is up and running with the correct version by running: +4. Confirm that the deployment is up and running with the correct version by running: ```bash velero version @@ -77,11 +71,11 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: Server: - Version: v1.17.0 + Version: v1.18.0 ``` [0]: basic-install.md#install-the-cli @@ -93,4 +87,5 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw [6]: https://velero.io/docs/v1.13/upgrade-to-1.13 [7]: https://velero.io/docs/v1.14/upgrade-to-1.14 [8]: https://velero.io/docs/v1.15/upgrade-to-1.15 -[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 \ No newline at end of file +[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 +[10]: https://velero.io/docs/v1.17/upgrade-to-1.17 \ No newline at end of file diff --git a/site/content/docs/v1.18/_index.md b/site/content/docs/v1.18/_index.md new file mode 100644 index 000000000..10f2d53f3 --- /dev/null +++ b/site/content/docs/v1.18/_index.md @@ -0,0 +1,58 @@ +--- +toc: "false" +cascade: + version: v1.18 + toc: "true" +--- +![100] + +[![Build Status][1]][2] + +## Overview + +Velero (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a cloud provider or on-premises. Velero lets you: + +* Take backups of your cluster and restore in case of loss. +* Migrate cluster resources to other clusters. +* Replicate your production cluster to development and testing clusters. + +Velero consists of: + +* A server that runs on your cluster +* A command-line client that runs locally + +## Documentation + +This site is our documentation home with installation instructions, plus information about customizing Velero for your needs, architecture, extending Velero, contributing to Velero and more. + +Please use the version selector at the top of the site to ensure you are using the appropriate documentation for your version of Velero. + +## Troubleshooting + +If you encounter issues, review the [troubleshooting docs][30], [file an issue][4], or talk to us on the [#velero-users channel][25] on the Kubernetes Slack server. + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, follow the instructions on our [Start contributing](https://velero.io/docs/v1.18.0/start-contributing/) documentation for guidance on how to setup Velero for development. + +## Changelog + +See [the list of releases][6] to find out about feature changes. + +[1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg +[2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI" + +[4]: https://github.com/vmware-tanzu/velero/issues +[6]: https://github.com/vmware-tanzu/velero/releases + +[9]: https://kubernetes.io/docs/setup/ +[10]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-homebrew-on-macos +[11]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#tabset-1 +[12]: https://github.com/kubernetes/kubernetes/blob/main/cluster/addons/dns/README.md +[14]: https://github.com/kubernetes/kubernetes +[24]: https://groups.google.com/forum/#!forum/projectvelero +[25]: https://kubernetes.slack.com/messages/velero-users + +[30]: troubleshooting.md + +[100]: img/velero.png diff --git a/site/content/docs/v1.18/api-types/README.md b/site/content/docs/v1.18/api-types/README.md new file mode 100644 index 000000000..54c23544d --- /dev/null +++ b/site/content/docs/v1.18/api-types/README.md @@ -0,0 +1,21 @@ +--- +title: "Table of Contents" +layout: docs +--- + +## API types + +Here we list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.18/api-types/_index.md b/site/content/docs/v1.18/api-types/_index.md new file mode 100644 index 000000000..e60832500 --- /dev/null +++ b/site/content/docs/v1.18/api-types/_index.md @@ -0,0 +1,19 @@ +--- +layout: docs +title: API types +--- + +Here's a list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.18/api-types/backup.md b/site/content/docs/v1.18/api-types/backup.md new file mode 100644 index 000000000..634264ee1 --- /dev/null +++ b/site/content/docs/v1.18/api-types/backup.md @@ -0,0 +1,213 @@ +--- +title: "Backup API Type" +layout: docs +--- + +## Use + +Use the `Backup` API type to request the Velero server to perform a backup. Once created, the +Velero Server immediately starts the backup process. + +## API GroupVersion + +Backup belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Backup` object with each of the fields documented: + +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Backup +# Standard Kubernetes metadata. Required. +metadata: + # Backup name. May be any valid Kubernetes object name. Required. + name: a + # Backup namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the backup. Required. +spec: + # CSISnapshotTimeout specifies the time used to wait for + # CSI VolumeSnapshot status turns to ReadyToUse during creation, before + # returning error as timeout. The default value is 10 minute. + csiSnapshotTimeout: 10m + # ItemOperationTimeout specifies the time used to wait for + # asynchronous BackupItemAction operations + # The default value is 4 hour. + itemOperationTimeout: 4h + # resourcePolicy specifies the referenced resource policies that backup should follow + # optional + resourcePolicy: + kind: configmap + name: resource-policy-configmap + # Array of namespaces to include in the backup. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the backup. Accepts glob patterns (*, ?, [abc]). Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Order of the resources to be collected during the backup process. It's a map with key being the plural resource + # name, and the value being a list of object names separated by comma. Each resource name has format "namespace/objectname". + # For cluster resources, simply use "objectname". Optional + orderedResources: + pods: mysql/mysql-cluster-replica-0,mysql/mysql-cluster-replica-1,mysql/mysql-cluster-source-0 + persistentvolumes: pvc-87ae0832-18fd-4f40-a2a4-5ed4242680c4,pvc-63be1bb0-90f5-4629-a7db-b8ce61ee29b3 + # Whether to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Array of cluster-scoped resources to exclude from the backup. Resources may be shortcuts + # (for example 'sc' for 'storageclasses'), or fully-qualified. If unspecified, + # no additional cluster-scoped resources are excluded. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + excludedClusterScopedResources: {} + # Array of cluster-scoped resources to include from the backup. Resources may be shortcuts + # (for example 'sc' for 'storageclasses'), or fully-qualified. If unspecified, + # no additional cluster-scoped resources are included. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + includedClusterScopedResources: {} + # Array of namespace-scoped resources to exclude from the backup. Resources may be shortcuts + # (for example 'cm' for 'configmaps'), or fully-qualified. If unspecified, + # no namespace-scoped resources are excluded. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + excludedNamespaceScopedResources: {} + # Array of namespace-scoped resources to include from the backup. Resources may be shortcuts + # (for example 'cm' for 'configmaps'), or fully-qualified. If unspecified, + # all namespace-scoped resources are included. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + includedNamespaceScopedResources: {} + # Individual objects must match this label selector to be included in the backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Individual object when matched with any of the label selector specified in the set are to be included in the backup. Optional. + # orLabelSelectors as well as labelSelector cannot co-exist, only one of them can be specified in the backup request + orLabelSelectors: + - matchLabels: + app: velero + - matchLabels: + app: data-protection + # Whether or not to snapshot volumes. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for this backup. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before this backup is eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # whether pod volume file system backup should be used for all volumes by default. + defaultVolumesToFsBackup: true + # Whether snapshot data should be moved. If set, data movement is launched after the snapshot is created. + snapshotMoveData: true + # The data mover to be used by the backup. If the value is "" or "velero", the built-in data mover will be used. + datamover: velero + # UploaderConfig specifies the configuration for the uploader + uploaderConfig: + # ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. + parallelFilesUpload: 10 + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +# Status about the Backup. Users should not set any data here. +status: + # The version of this Backup. The only version supported is 1. + version: 1 + # The date and time when the Backup is eligible for garbage collection. + expiration: null + # The current phase. + # Valid values are New, FailedValidation, InProgress, WaitingForPluginOperations, + # WaitingForPluginOperationsPartiallyFailed, FinalizingafterPluginOperations, + # FinalizingPartiallyFailed, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Date/time when the backup started being processed. + startTimestamp: 2019-04-29T15:58:43Z + # Date/time when the backup finished being processed. + completionTimestamp: 2019-04-29T15:58:56Z + # Number of volume snapshots that Velero tried to create for this backup. + volumeSnapshotsAttempted: 2 + # Number of volume snapshots that Velero successfully created for this backup. + volumeSnapshotsCompleted: 1 + # Number of attempted BackupItemAction operations for this backup. + backupItemOperationsAttempted: 2 + # Number of BackupItemAction operations that Velero successfully completed for this backup. + backupItemOperationsCompleted: 1 + # Number of BackupItemAction operations that ended in failure for this backup. + backupItemOperationsFailed: 0 + # Number of warnings that were logged by the backup. + warnings: 2 + # Number of errors that were logged by the backup. + errors: 0 + # An error that caused the entire backup to fail. + failureReason: "" +``` diff --git a/site/content/docs/v1.18/api-types/backupstoragelocation.md b/site/content/docs/v1.18/api-types/backupstoragelocation.md new file mode 100644 index 000000000..92a78aa49 --- /dev/null +++ b/site/content/docs/v1.18/api-types/backupstoragelocation.md @@ -0,0 +1,112 @@ +--- +title: "Velero Backup Storage Locations" +layout: docs +--- + +## Backup Storage Location + +Velero can store backups in a number of locations. These are represented in the cluster via the `BackupStorageLocation` CRD. + +Velero must have at least one `BackupStorageLocation`. By default, this is expected to be named `default`, however the name can be changed by specifying `--default-backup-storage-location` on `velero server`. Backups that do not explicitly specify a storage location will be saved to this `BackupStorageLocation`. + +A sample YAML `BackupStorageLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + backupSyncPeriod: 2m0s + provider: aws + objectStorage: + bucket: myBucket + credential: + name: secret-name + key: key-in-secret + config: + region: us-west-2 + profile: "default" +``` + +### Example with self-signed certificate + +When using object storage with self-signed certificates, you can specify the CA certificate: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + provider: aws + objectStorage: + bucket: velero-backups + # Base64 encoded CA certificate (deprecated - use caCertRef instead) + caCert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR1VENDQXFHZ0F3SUJBZ0lVTWRiWkNaYnBhcE9lYThDR0NMQnhhY3dVa213d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JERUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2tOaGJHbG1iM0p1YVdFeEZqQVVCZ05WQkFjTQpEVk5oYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFvTUQwVjRZVzF3YkdVZ1EyOXRjR0Z1ZVRFV01CUUdBMVVFCkF3d05aWGhoYlhCc1pTNXNiMk5oYkRBZUZ3MHlNekEzTVRBeE9UVXlNVGhhRncweU5EQTNNRGt4T1RVeU1UaGEKTUd3eEN6QUpCZ05WQkFZVEFsVlRNUk13RVFZRFZRUUNEQXBEWEJ4cG1iM0p1YVdFeEZqQVVCZ05WQkFjTURWTmgKYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFvTUQwVjRZVzF3YkdVZ1EyOXRjR0Z1ZVRFV01CUUdBMVVFQXd3TgpaWGhoYlhCc1pTNXNiMk5oYkRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBS1dqCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + config: + region: us-east-1 + s3Url: https://minio.example.com +``` + +#### Using a CA Certificate with Secret Reference (Recommended) + +The recommended approach is to use `caCertRef` to reference a Secret containing the CA certificate: + +```yaml +# First, create a Secret containing the CA certificate +apiVersion: v1 +kind: Secret +metadata: + name: storage-ca-cert + namespace: velero +type: Opaque +data: + ca-bundle.crt: + +--- +# Then reference it in the BackupStorageLocation +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + provider: aws + objectStorage: + bucket: myBucket + caCertRef: + name: storage-ca-cert + key: ca-bundle.crt + # ... other configuration +``` + +**Note:** You cannot specify both `caCert` and `caCertRef` in the same BackupStorageLocation. The `caCert` field is deprecated and will be removed in a future version. + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever object storage provider will be used to store the backups. See [your object storage provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `objectStorage` | ObjectStorageLocation | Required Field | Specification of the object storage for the given provider. | +| `objectStorage/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | +| `objectStorage/prefix` | String | Optional Field | The directory inside a storage bucket where backups are to be uploaded. | +| `objectStorage/caCert` | String | Optional Field | **Deprecated**: Use `caCertRef` instead. A base64 encoded CA bundle to be used when verifying TLS connections | +| `objectStorage/caCertRef` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | Reference to a Secret containing a CA bundle to be used when verifying TLS connections. The Secret must be in the same namespace as the BackupStorageLocation. | +| `objectStorage/caCertRef/name` | String | Required Field (when using caCertRef) | The name of the Secret containing the CA certificate bundle | +| `objectStorage/caCertRef/key` | String | Required Field (when using caCertRef) | The key within the Secret that contains the CA certificate bundle | +| `config` | map[string]string | None (Optional) | Provider-specific configuration keys/values to be passed to the object store plugin. See [your object storage provider's plugin documentation](../supported-providers) for details. | +| `accessMode` | String | `ReadWrite` | How Velero can access the backup storage location. Valid values are `ReadWrite`, `ReadOnly`. | +| `backupSyncPeriod` | metav1.Duration | Optional Field | How frequently Velero should synchronize backups in object storage. Default is Velero's server backup sync period. Set this to `0s` to disable sync. | +| `validationFrequency` | metav1.Duration | Optional Field | How frequently Velero should validate the object storage . Default is Velero's server validation frequency. Set this to `0s` to disable validation. Default 1 minute. | +| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. | +| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. | +| `credential/key` | String | Optional Field | The key to use within the secret. | +{{< /table >}} \ No newline at end of file diff --git a/site/content/docs/v1.18/api-types/restore.md b/site/content/docs/v1.18/api-types/restore.md new file mode 100644 index 000000000..1c80a0ee8 --- /dev/null +++ b/site/content/docs/v1.18/api-types/restore.md @@ -0,0 +1,221 @@ +--- +title: "Restore API Type" +layout: docs +--- + +## Use + +The `Restore` API type is used as a request for the Velero server to perform a Restore. Once created, the +Velero Server immediately starts the Restore process. + +## API GroupVersion + +Restore belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Restore` object with each of the fields documented: + +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Restore +# Standard Kubernetes metadata. Required. +metadata: + # Restore name. May be any valid Kubernetes object name. Required. + name: a-very-special-backup-0000111122223333 + # Restore namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the restore. Required. +spec: + # The unique name of the Velero backup to restore from. + backupName: a-very-special-backup + # The unique name of the Velero schedule + # to restore from. If specified, and BackupName is empty, Velero will + # restore from the most recent successful backup created from this schedule. + scheduleName: my-scheduled-backup-name + # ItemOperationTimeout specifies the time used to wait for + # asynchronous BackupItemAction operations + # The default value is 4 hour. + itemOperationTimeout: 4h + # UploaderConfig specifies the configuration for the restore. + uploaderConfig: + # WriteSparseFiles is a flag to indicate whether write files sparsely or not + writeSparseFiles: true + # ParallelFilesDownload is the concurrency number setting for restore + parallelFilesDownload: 10 + # Array of namespaces to include in the restore. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the restore. Accepts glob patterns (*, ?, [abc]). Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + + # restoreStatus selects resources to restore not only the specification, but + # the status of the manifest. This is specially useful for CRDs that maintain + # external references. By default, it excludes all resources. + restoreStatus: + # Array of resources to include in the restore status. Just like above, + # resources may be shortcuts (for example 'po' for 'pods') or fully-qualified. + # If unspecified, no resources are included. Optional. + includedResources: + - workflows + # Array of resources to exclude from the restore status. Resources may be + # shortcuts (for example 'po' for 'pods') or fully-qualified. + # If unspecified, all resources are excluded. Optional. + excludedResources: [] + + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the restore. For example, if a + # PersistentVolumeClaim is included in the restore, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the restore. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Individual object when matched with any of the label selector specified in the set are to be included in the restore. Optional. + # orLabelSelectors as well as labelSelector cannot co-exist, only one of them can be specified in the restore request + orLabelSelectors: + - matchLabels: + app: velero + - matchLabels: + app: data-protection + # namespaceMapping is a map of source namespace names to + # target namespace names to restore into. Any source namespaces not + # included in the map will be restored into namespaces of the same name. + namespaceMapping: + namespace-backup-from: namespace-to-restore-to + # restorePVs specifies whether to restore all included PVs + # from snapshot. Optional + restorePVs: true + # preserveNodePorts specifies whether to restore old nodePorts from backup, + # so that the exposed port numbers on the node will remain the same after restore. Optional + preserveNodePorts: true + # existingResourcePolicy specifies the restore behaviour + # for the Kubernetes resource to be restored. Optional + existingResourcePolicy: none + # ResourceModifier specifies the reference to JSON resource patches + # that should be applied to resources before restoration. Optional + resourceModifier: + kind: ConfigMap + name: resource-modifier-configmap + # Actions to perform during or post restore. The only hooks currently supported are + # adding an init container to a pod before it can be restored and executing a command in a + # restored pod's container. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + # Name is the name of this hook. + - name: restore-hook-1 + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - ns1 + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - ns3 + # Array of resources to which this hook applies. If unspecified, the hook applies to all resources in the backup. Optional. + # The only resource supported at this time is pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run during or after restores. Currently only "init" and "exec" hooks + # are supported. + postHooks: + # The type of the hook. This must be "init" or "exec". + - init: + # An array of container specs to be added as init containers to pods to which this hook applies to. + initContainers: + - name: restore-hook-init1 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc1-vm + # Volume name from the podSpec + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc2-vm + # Volume name from the podSpec + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed + - exec: + # The container name where the hook will be executed. Defaults to the first container. + # Optional. + container: foo + # The command that will be executed in the container. Required. + command: + - /bin/bash + - -c + - "psql < /backup/backup.sql" + # How long to wait for a container to become ready. This should be long enough for the + # container to start plus any preceding hooks in the same container to complete. The wait + # timeout begins when the container is restored and may require time for the image to pull + # and volumes to mount. If not set the restore will wait indefinitely. Optional. + waitTimeout: 5m + # How long to wait once execution begins. Defaults to 30 seconds. Optional. + execTimeout: 1m + # How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to + # `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, + # no more restore hooks will be executed in any container in any pod and the status of the + # Restore will be `PartiallyFailed`. Optional. + onError: Continue +# RestoreStatus captures the current status of a Velero restore. Users should not set any data here. +status: + # The current phase. + # Valid values are New, FailedValidation, InProgress, WaitingForPluginOperations, + # WaitingForPluginOperationsPartiallyFailed, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Number of attempted RestoreItemAction operations for this restore. + restoreItemOperationsAttempted: 2 + # Number of RestoreItemAction operations that Velero successfully completed for this restore. + restoreItemOperationsCompleted: 1 + # Number of RestoreItemAction operations that ended in failure for this restore. + restoreItemOperationsFailed: 0 + # Number of warnings that were logged by the restore. + warnings: 2 + # Errors is a count of all error messages that were generated + # during execution of the restore. The actual errors are stored in object + # storage. + errors: 0 + # FailureReason is an error that caused the entire restore + # to fail. + failureReason: + +``` diff --git a/site/content/docs/v1.18/api-types/schedule.md b/site/content/docs/v1.18/api-types/schedule.md new file mode 100644 index 000000000..c89fe60d7 --- /dev/null +++ b/site/content/docs/v1.18/api-types/schedule.md @@ -0,0 +1,216 @@ +--- +title: "Schedule API Type" +layout: docs +--- + +## Use + +The `Schedule` API type is used as a repeatable request for the Velero server to perform a backup for a given cron notation. Once created, the +Velero Server will start the backup process. It will then wait for the next valid point of the given cron expression and execute the backup +process on a repeating basis. + +### Schedule Control Fields + +The Schedule API provides several fields to control backup execution behavior: + +- **paused**: When set to `true`, the schedule is paused and no new backups will be created. When set back to `false`, the schedule is unpaused and will resume creating backups according to the cron schedule. + +- **skipImmediately**: Controls whether to skip an immediate backup when a schedule is created or unpaused. By default (when `false`), if a backup is due immediately upon creation or unpausing, it will be executed right away. When set to `true`, the controller will: + 1. Skip the immediate backup + 2. Record the current time in the `lastSkipped` status field + 3. Automatically reset `skipImmediately` back to `false` (one-time use) + 4. Schedule the next backup based on the cron expression, using `lastSkipped` as the reference time + +- **lastSkipped**: A status field (not directly settable) that records when a backup was last skipped due to `skipImmediately` being `true`. The controller uses this timestamp, if more recent than `lastBackup`, to calculate the next scheduled backup time. + +This "consume and reset" pattern for `skipImmediately` ensures that after skipping one immediate backup, the schedule returns to normal behavior for subsequent runs without requiring user intervention. + +## API GroupVersion + +Schedule belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Schedule` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Schedule +# Standard Kubernetes metadata. Required. +metadata: + # Schedule name. May be any valid Kubernetes object name. Required. + name: a + # Schedule namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the scheduled backup. Required. +spec: + # Paused specifies whether the schedule is paused or not + paused: false + # SkipImmediately specifies whether to skip backup if schedule is due immediately when unpaused or created. + # This is a one-time flag that will be automatically reset to false after being consumed. + # When true, the controller will skip the immediate backup, set LastSkipped timestamp, and reset this to false. + skipImmediately: false + # Schedule is a Cron expression defining when to run the Backup + schedule: 0 7 * * * + # Specifies whether to use OwnerReferences on backups created by this Schedule. + # Notice: if set to true, when schedule is deleted, backups will be deleted too. Optional. + useOwnerReferencesInBackup: false + # Template is the spec that should be used for each backup triggered by this schedule. + template: + # CSISnapshotTimeout specifies the time used to wait for + # CSI VolumeSnapshot status turns to ReadyToUse during creation, before + # returning error as timeout. The default value is 10 minute. + csiSnapshotTimeout: 10m + # resourcePolicy specifies the referenced resource policies that backup should follow + # optional + resourcePolicy: + kind: configmap + name: resource-policy-configmap + # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the scheduled backup. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + orderedResources: + pods: mysql/mysql-cluster-replica-0,mysql/mysql-cluster-replica-1,mysql/mysql-cluster-source-0 + persistentvolumes: pvc-87ae0832-18fd-4f40-a2a4-5ed4242680c4,pvc-63be1bb0-90f5-4629-a7db-b8ce61ee29b3 + # Whether to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the scheduled backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Array of cluster-scoped resources to exclude from the backup. Resources may be shortcuts + # (for example 'sc' for 'storageclasses'), or fully-qualified. If unspecified, + # no additional cluster-scoped resources are excluded. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + excludedClusterScopedResources: {} + # Array of cluster-scoped resources to include from the backup. Resources may be shortcuts + # (for example 'sc' for 'storageclasses'), or fully-qualified. If unspecified, + # no additional cluster-scoped resources are included. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + includedClusterScopedResources: {} + # Array of namespace-scoped resources to exclude from the backup. Resources may be shortcuts + # (for example 'cm' for 'configmaps'), or fully-qualified. If unspecified, + # no namespace-scoped resources are excluded. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + excludedNamespaceScopedResources: {} + # Array of namespace-scoped resources to include from the backup. Resources may be shortcuts + # (for example 'cm' for 'configmaps'), or fully-qualified. If unspecified, + # all namespace-scoped resources are included. Optional. + # Cannot work with include-resources, exclude-resources and include-cluster-resources. + includedNamespaceScopedResources: {} + # Individual objects must match this label selector to be included in the scheduled backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Individual object when matched with any of the label selector specified in the set are to be included in the backup. Optional. + # orLabelSelectors as well as labelSelector cannot co-exist, only one of them can be specified in the backup request + orLabelSelectors: + - matchLabels: + app: velero + - matchLabels: + app: data-protection + # Whether to snapshot volumes. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for backups under this schedule. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before backups created on this schedule are eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # whether pod volume file system backup should be used for all volumes by default. + defaultVolumesToFsBackup: true + # Whether snapshot data should be moved. If set, data movement is launched after the snapshot is created. + snapshotMoveData: true + # The data mover to be used by the backup. If the value is "" or "velero", the built-in data mover will be used. + datamover: velero + # UploaderConfig specifies the configuration for the uploader + uploaderConfig: + # ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. + parallelFilesUpload: 10 + # The labels you want on backup objects, created from this schedule (instead of copying the labels you have on schedule object itself). + # When this field is set, the labels from the Schedule resource are not copied to the Backup resource. + metadata: + labels: + labelname: somelabelvalue + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +status: + # The current phase. + # Valid values are New, Enabled, FailedValidation. + phase: "" + # Date/time of the last backup for a given schedule + lastBackup: + # Date/time when a backup was last skipped due to skipImmediately being true + lastSkipped: + # An array of any validation errors encountered. + validationErrors: +``` diff --git a/site/content/docs/v1.18/api-types/volumesnapshotlocation.md b/site/content/docs/v1.18/api-types/volumesnapshotlocation.md new file mode 100644 index 000000000..e6758f8fa --- /dev/null +++ b/site/content/docs/v1.18/api-types/volumesnapshotlocation.md @@ -0,0 +1,46 @@ +--- +title: "Velero Volume Snapshot Location" +layout: docs +--- + +## Volume Snapshot Location + +A volume snapshot location is the location in which to store the volume snapshots created for a backup. + +Velero can be configured to take snapshots of volumes from multiple providers. Velero also allows you to configure multiple possible `VolumeSnapshotLocation` per provider, although you can only select one location per provider at backup time. + +Each VolumeSnapshotLocation describes a provider + location. These are represented in the cluster via the `VolumeSnapshotLocation` CRD. Velero must have at least one `VolumeSnapshotLocation` per cloud provider. + +A sample YAML `VolumeSnapshotLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +metadata: + name: aws-default + namespace: velero +spec: + provider: aws + credential: + name: secret-name + key: key-in-secret + config: + region: us-west-2 + profile: "default" +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever storage provider will be used to create/store the volume snapshots. See [your volume snapshot provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `config` | map string string | None (Optional) | Provider-specific configuration keys/values to be passed to the volume snapshotter plugin. See [your volume snapshot provider's plugin documentation](../supported-providers) for details. | +| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. | +| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. | +| `credential/key` | String | Optional Field | The key to use within the secret. | +{{< /table >}} diff --git a/site/content/docs/v1.18/backup-hooks.md b/site/content/docs/v1.18/backup-hooks.md new file mode 100644 index 000000000..465dedbad --- /dev/null +++ b/site/content/docs/v1.18/backup-hooks.md @@ -0,0 +1,126 @@ +--- +title: "Backup Hooks" +layout: docs +--- + +Velero supports executing commands in containers in pods during a backup. + +## Backup Hooks + +When performing a backup, you can specify one or more commands to execute in a container in a pod +when that pod is being backed up. The commands can be configured to run *before* any custom action +processing ("pre" hooks), or after all custom actions have been completed and any additional items +specified by custom action have been backed up ("post" hooks). Note that hooks are _not_ executed within a shell +on the containers. + +As of Velero 1.15, related items that must be backed up together are grouped into ItemBlocks, and pod hooks run before and after the ItemBlock is backed up. +In particular, this means that if an ItemBlock contains more than one pod (such as in a scenario where an RWX volume is mounted by multiple pods), pre hooks are run for all pods in the ItemBlock, then the items are backed up, then all post hooks are run. + +There are two ways to specify hooks: annotations on the pod itself, and in the Backup spec. + +### Specifying Hooks As Pod Annotations + +You can use the following annotations on a pod to make Velero execute a hook when backing up the pod: + +#### Pre hooks + +* `pre.hook.backup.velero.io/container` + * The container where the command should be executed. Defaults to the first container in the pod. Optional. +* `pre.hook.backup.velero.io/command` + * The command to execute. This command is not executed within a shell by default. If a shell is needed to run your command, include a shell command, like `/bin/sh`, that is supported by the container at the beginning of your command. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]`. See [examples of using pre hook commands](#backup-hook-commands-examples). Optional. +* `pre.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to `Fail`. Valid values are Fail and Continue. Optional. +* `pre.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + + +#### Post hooks + +* `post.hook.backup.velero.io/container` + * The container where the command should be executed. Default is the first container in the pod. Optional. +* `post.hook.backup.velero.io/command` + * The command to execute. This command is not executed within a shell by default. If a shell is needed to run your command, include a shell command, like `/bin/sh`, that is supported by the container at the beginning of your command. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]`. See [examples of using pre hook commands](#backup-hook-commands-examples). Optional. +* `post.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to `Fail`. Valid values are Fail and Continue. Optional. +* `post.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + +### Specifying Hooks in the Backup Spec + +Please see the documentation on the [Backup API Type][1] for how to specify hooks in the Backup +spec. + +## Hook Example with fsfreeze + +This examples walks you through using both pre and post hooks for freezing a file system. Freezing the +file system is useful to ensure that all pending disk I/O operations have completed prior to taking a snapshot. + +### Annotations + +The Velero [example/nginx-app/with-pv.yaml][2] serves as an example of adding the pre and post hook annotations directly +to your declarative deployment. Below is an example of what updating an object in place might look like. + +```shell +kubectl annotate pod -n nginx-example -l app=nginx \ + pre.hook.backup.velero.io/command='["/sbin/fsfreeze", "--freeze", "/var/log/nginx"]' \ + pre.hook.backup.velero.io/container=fsfreeze \ + post.hook.backup.velero.io/command='["/sbin/fsfreeze", "--unfreeze", "/var/log/nginx"]' \ + post.hook.backup.velero.io/container=fsfreeze +``` + +Now test the pre and post hooks by creating a backup. You can use the Velero logs to verify that the pre and post +hooks are running and exiting without error. + +```shell +velero backup create nginx-hook-test + +velero backup get nginx-hook-test +velero backup logs nginx-hook-test | grep hookCommand +``` + +## Backup hook commands examples + +### Multiple commands + +To use multiple commands, wrap your target command in a shell and separate them with `;`, `&&`, or other shell conditional constructs. + +```shell + pre.hook.backup.velero.io/command='["/bin/bash", "-c", "echo hello > hello.txt && echo goodbye > goodbye.txt"]' +``` + +#### Using environment variables + +You are able to use environment variables from your pods in your pre and post hook commands by including a shell command before using the environment variable. For example, `MYSQL_ROOT_PASSWORD` is an environment variable defined in pod called `mysql`. To use `MYSQL_ROOT_PASSWORD` in your pre-hook, you'd include a shell, like `/bin/sh`, before calling your environment variable: + +``` +pre: +- exec: + container: mysql + command: + - /bin/sh + - -c + - mysql --password=$MYSQL_ROOT_PASSWORD -e "FLUSH TABLES WITH READ LOCK" + onError: Fail +``` + +Note that the container must support the shell command you use. + +## Backup Hook Execution Results +### Viewing Results + +Velero records the execution results of hooks, allowing users to obtain this information by running the following command: + +```bash +$ velero backup describe +``` + +The displayed results include the number of hooks that were attempted to be executed and the number of hooks that failed execution. Any detailed failure reasons will be present in `Errors` section if applicable. + +```bash +HooksAttempted: 1 +HooksFailed: 0 +``` + + +[1]: api-types/backup.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/examples/nginx-app/with-pv.yaml diff --git a/site/content/docs/v1.18/backup-reference.md b/site/content/docs/v1.18/backup-reference.md new file mode 100644 index 000000000..220d97acc --- /dev/null +++ b/site/content/docs/v1.18/backup-reference.md @@ -0,0 +1,167 @@ +--- +title: "Backup Reference" +layout: docs +--- + +## Exclude Specific Items from Backup + +It is possible to exclude individual items from being backed up, even if they match the resource/namespace/label selectors defined in the backup spec. To do this, label the item as follows: + +```bash +kubectl label -n / velero.io/exclude-from-backup=true +``` +## Parallel Files Upload +If using fs-backup with Kopia uploader or CSI snapshot data movements, it's allowed to configure the option for parallel files upload, which could accelerate the backup: +```bash +velero backup create --include-namespaces --parallel-files-upload --wait +``` + +## Specify Backup Orders of Resources of Specific Kind + +To backup resources of specific Kind in a specific order, use option --ordered-resources to specify a mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Kind name is in plural form. + +```bash +velero backup create backupName --include-cluster-resources=true --ordered-resources 'pods=ns1/pod1,ns1/pod2;persistentvolumes=pv4,pv8' --include-namespaces=ns1 +velero backup create backupName --ordered-resources 'statefulsets=ns1/sts1,ns1/sts0' --include-namespaces=ns1 +``` +## Schedule a Backup + +The **schedule** operation allows you to create a backup of your data at a specified time, defined by a [Cron expression](https://en.wikipedia.org/wiki/Cron). + +``` +velero schedule create NAME --schedule="* * * * *" [flags] +``` + +Cron schedules use the following format. + +``` +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * +``` + +For example, the command below creates a backup that runs every day at 3am. + +``` +velero schedule create example-schedule --schedule="0 3 * * *" +``` + +This command will create the backup, `example-schedule`, within Velero, but the backup will not be taken until the next scheduled time, 3am. Backups created by a schedule are saved with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For a full list of available configuration flags use the Velero CLI help command. + +``` +velero schedule create --help +``` + +Once you create the scheduled backup, you can then trigger it manually using the `velero backup` command. + +``` +velero backup create --from-schedule example-schedule +``` + +This command will immediately trigger a new backup based on your template for `example-schedule`. This will not affect the backup schedule, and another backup will trigger at the scheduled time. + +### Time zone specification +Time zone can be specified in the schedule cron. The format is `CRON_TZ= `. + +Specifying timezones can reduce disputes in the case of daylight saving time changes. For example, if the schedule is set to run at 3am, and daylight saving time changes, the schedule will still run at 3am in the timezone specified. + +Be aware that jobs scheduled during daylight-savings leap-ahead transitions will not be run! + +For example, the command below creates a backup that runs every day at 3am in the timezone `America/New_York`. + +``` +velero schedule create example-schedule --schedule="CRON_TZ=America/New_York 0 3 * * *" +``` + +Another example, the command below creates a backup that runs every day at 3am in the timezone `Asia/Shanghai`. + +``` +velero schedule create example-schedule --schedule="CRON_TZ=Asia/Shanghai 0 3 * * *" +``` + +The supported timezone names are listed in the [IANA Time Zone Database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) under 'TZ identifier'. + + +### Limitation + +#### Backup's OwnerReference with Schedule +Backups created from schedule can have owner reference to the schedule. This can be achieved by command: + +``` +velero schedule create --use-owner-references-in-backup +``` +By this way, schedule is the owner of it created backups. This is useful for some GitOps scenarios, or the resource tree of k8s synchronized from other places. + +Please do notice there is also side effect that may not be expected. Because schedule is the owner, when the schedule is deleted, the related backups CR (Just backup CR is deleted. Backup data still exists in object store and snapshots) will be deleted by k8s GC controller, too, but Velero controller will sync these backups from object store's metadata into k8s. Then k8s GC controller and Velero controller will fight over whether these backups should exist all through. + +If there is possibility the schedule will be disable to not create backup anymore, and the created backups are still useful. Please do not enable this option. For detail, please reference to [Backups created by a schedule with useOwnerReferenceInBackup set do not get synced properly](https://github.com/vmware-tanzu/velero/issues/4093). + +Some GitOps tools have configurations to avoid pruning the day 2 backups generated from the schedule. +For example, the ArgoCD has two ways to do that: +* Add annotations to schedule. This method makes ArgoCD ignore the schedule from syncing, so the generated backups are ignored too, but it has a side effect. When deleting the schedule from the GitOps manifest, the schedule can not be deleted. User needs to do it manually. +``` yaml + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Delete=false,Prune=false +``` +* If ArgoCD is deployed by ArgoCD-Operator, there is another option: [resourceExclusions](https://argocd-operator.readthedocs.io/en/latest/reference/argocd/#resource-exclusions-example). This is an example, which means ArgoCD operator should ignore `Backup` and `Restore` in `velero.io` group in the `velero` namespace for all managed k8s cluster. +``` yaml +apiVersion: argoproj.io/v1alpha1 +kind: ArgoCD +metadata: + name: velero-argocd + namespace: velero +spec: + resourceExclusions: | + - apiGroups: + - velero.io + kinds: + - Backup + - Restore + clusters: + - "*" +``` + +#### Cannot support backup data immutability +Starting from 1.11, Velero's backups may not work as expected when the target object storage has some kind of an "immutability" option configured. These options are known by different names (see links below for some examples). The main reason is that Velero first saves the state of a backup as Finalizing and then checks whether there are any async operations in progress. If there are, it needs to wait for all of them to be finished before moving the backup state to Complete. If there are no async operations, the state is moved to Complete right away. In either case, Velero needs to modify the metadata in object storage and that will not be possible if some kind of immutability is configured on the object storage. + +Even with versions prior to 1.11, there was no explicit support in Velero to work with object storage that has "immutability" configuration. As a result, you may see some problems even though backups seem to work (e.g. versions objects not being deleted when backup is deleted). + +Note that backups may still work in some cases depending on specific providers and configurations. + +* For AWS S3 service, backups work because S3's object lock only applies to versioned buckets, and the object data can still be updated as the new version. But when backups are deleted, old versions of the objects will not be deleted. +* Azure Storage Blob supports both versioned-level immutability and container-level immutability. For the versioned-level scenario, data immutability can still work in Velero, but the container-level cannot. +* GCP Cloud storage policy only supports bucket-level immutability, so there is no way to make it work in the GCP environment. + +The following are the links to cloud providers' documentation in this regard: + +* [AWS S3 Using S3 Object Lock](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html) +* [Azure Storage Blob Containers - Lock Immutability Policy](https://learn.microsoft.com/en-us/azure/storage/blobs/immutable-policy-configure-version-scope?tabs=azure-portal) +* [GCP cloud storage Retention policies and retention policy locks](https://cloud.google.com/storage/docs/bucket-lock) + +## Kubernetes API Pagination + +By default, Velero will paginate the LIST API call for each resource type in the Kubernetes API when collecting items into a backup. The `--client-page-size` flag for the Velero server configures the size of each page. + +Depending on the cluster's scale, tuning the page size can improve backup performance. You can experiment with higher values, noting their impact on the relevant `apiserver_request_duration_seconds_*` metrics from the Kubernetes apiserver. + +Pagination can be entirely disabled by setting `--client-page-size` to `0`. This will request all items in a single unpaginated LIST call. + +## Deleting Backups + +Use the following commands to delete Velero backups and data: + +* `kubectl delete backup -n ` will delete the backup custom resource only and will not delete any associated data from object/block storage +* `velero backup delete ` will delete the backup resource including all data in object/block storage diff --git a/site/content/docs/v1.18/backup-repository-configuration.md b/site/content/docs/v1.18/backup-repository-configuration.md new file mode 100644 index 000000000..fd6cf0b78 --- /dev/null +++ b/site/content/docs/v1.18/backup-repository-configuration.md @@ -0,0 +1,63 @@ +--- +title: "Backup Repository Configuration" +layout: docs +--- + +Velero uses selectable backup repositories for various backup/restore methods, i.e., [file-system backup][1], [CSI snapshot data movement][2], etc. To achieve the best performance, backup repositories may need to be configured according to the running environments. + +Velero uses a BackupRepository CR to represent the instance of the backup repository. Now, a new field `repositoryConfig` is added to support various configurations to the underlying backup repository. + +Velero also allows you to specify configurations before the BackupRepository CR is created through a configMap. The configurations in the configMap will be copied to the BackupRepository CR when it is created at the due time. +The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to Velero instance in that namespace only. The name of the configMap should be specified in the Velero server parameter `--backup-repository-configmap`. + + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --backup-repository-configmap=` + +Conclusively, you have two ways to add/change/delete configurations of a backup repository: +- If the BackupRepository CR for the backup repository is already there, you should modify the `repositoryConfig` field. The new changes will be applied to the backup repository at the due time, it doesn't require Velero server to restart. +- Otherwise, you can create the backup repository configMap as a template for the BackupRepository CRs that are going to be created. + +The backup repository configMap is repository type (i.e., kopia, restic) specific, so for one repository type, you only need to create one set of configurations, they will be applied to all BackupRepository CRs of the same type. Whereas, the changes of `repositoryConfig` field apply to the specific BackupRepository CR only, you may need to change every BackupRepository CR of the same type. + +Below is an example of the BackupRepository configMap with the configurations: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: + namespace: velero +data: + : | + { + "cacheLimitMB": 2048, + "fullMaintenanceInterval": "fastGC" + } + : | + { + "cacheLimitMB": 1024 + } +``` + +To create the configMap, you need to save something like the above sample to a file and then run below commands: +```shell +kubectl apply -f +``` + +When and how the configurations are used is decided by the backup repository itself. Though you can specify any configuration to the configMap or `repositoryConfig`, the configuration may/may not be used by the backup repository, or the configuration may be used at an arbitrary time. + +Below is the supported configurations by Velero and the specific backup repository. +***Kopia repository:*** +`cacheLimitMB`: specifies the size limit(in MB) for the local data cache. The more data is cached locally, the less data may be downloaded from the backup storage, so the better performance may be achieved. Practically, you can specify any size that is smaller than the free space so that the disk space won't run out. This parameter is for repository connection, that is, you could change it before connecting to the repository. E.g., before a backup/restore/maintenance. + +`fullMaintenanceInterval`: The full maintenance interval defaults to kopia defaults of 24 hours. Override options below allows for faster removal of deleted velero backups from kopia repo. +- normalGC: 24 hours +- fastGC: 12 hours +- eagerGC: 6 hours + +Per kopia [Maintenance Safety](https://kopia.io/docs/advanced/maintenance/#maintenance-safety), it is expected that velero backup deletion will not result in immediate kopia repository data removal. Reducing full maintenance interval using above options should help reduce time taken to remove blobs not in use. + +On the other hand, the not-in-use data will be deleted permanently after the full maintenance, so shorter full maintenance intervals may weaken the data safety if they are used incorrectly. + +[1]: file-system-backup.md +[2]: csi-snapshot-data-movement.md diff --git a/site/content/docs/v1.18/backup-restore-windows.md b/site/content/docs/v1.18/backup-restore-windows.md new file mode 100644 index 000000000..9d700f472 --- /dev/null +++ b/site/content/docs/v1.18/backup-restore-windows.md @@ -0,0 +1,79 @@ +--- +title: "Backup Restore Windows Workloads" +layout: docs +--- + +## Prerequisites + +Velero supports to backup and restore Windows workloads, either stateless or stateful. +To keep compatibility to the existing Velero plugins, Velero server runs in linux nodes only, so Velero requires at least one linux node in the cluster. And it is not recommended to run Velero server in control plane, so a linux worker node is required. For resource requirement of the linux node for Velero server, see [Customize resource requests and limits][1]. + +Velero is built and tested with `windows/amd64/ltsc2022` container only, older Windows versions, i.e., Windows Server 2019, are not supported. + +For volume backups, CSI and CSI snapshot should be supported by the storage. + +## Installation + +As mentioned in [Image building][2], a hybrid image is provided for all platforms, so you don't need to set different images for linux and Windows clusters, you can always use the all-in-one image, e.g., `velero/velero:v1.16.0` or `velero/velero:main`. + +In order to backup/restore volumes for stateful workloads, Velero node-agent needs to run in the Windows nodes. Velero provides a dedicated daemonset for Windows nodes, called `node-agent-windows`. +Therefore, in a typical cluster with linux and Windows nodes, there are two daemonsets for Velero node-agent, the existing `node-agent` deamonset for linux nodes, and the `node-agent-windows` daemonset for Windows nodes. +If you want to install `node-agent` deamonset, specify `--use-node-agent` parameter in `velero install` command; and if you want to install `node-agent-windows` daemonset, specify `--use-node-agent-windows` parameter. + +## Resource backup restore + +Resource backup/restore for Windows workloads are done by Velero server as same as linux workloads. + +Since Velero server is running in linux nodes only, all the existing plugins, i.e., BIA, RIA, BackupStore plugins, could be started by Velero in a cluster with Windows nodes. However, whether or how the plugins are functional to Windows workloads are decided by the plugins themselves. +It is recommended that plugin providers do a well round test with Velero in Windows cluster environments, and: +- If they need to support Windows workloads, make the necessary modification to ensure their plugins work well with Windows workloads +- If they don't want to support Windows workloads, or part of the Windows workloads, they need to ensure the plugins won't cause any failure or crash when they process the undesired Windows workload items + +## Volume backup restore + +Below are the status of supportive of Windows workload volumes for different backup methods: +- CSI snapshot data movement: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are full supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same with linux workloads +- CSI snapshot backup: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are full supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same with linux workloads +- native snapshot backup: supported as same as linux workloads +- file system backup: at present, NOT supported + +For volume backups/restores conducted through Velero plugins, the supportive status is decided by the plugin themselves. + +### CSI snapshot data movement + +During backup, Velero automatically identifies the OS type of the workload and schedules data mover pods to the right nodes. Specifically, for a linux workload, linux nodes in the cluster will be used; for a Windows workload, Windows nodes in the cluster will be used. +You could view the OS type that a data mover pod is running with from the DataUpload status's `nodeOS` field. + +Velero takes several measures to deduce the OS type for volumes of workloads, from PVCs, VolumeAttach CRs, nodes and storage classes. If Velero fails to deduce the OS type, it fallbacks to linux, then the data mover pods will be scheduled to linux nodes. As a result, the data mover pods may not be able to start and the corresponding DataUploads will be cancelled because of timeout, so the backup will be partially failed. + +Therefore, it is highly recommended you provide a dedicated storage class for Windows workloads volumes, and set `csi.storage.k8s.io/fstype` correctly. E.g., for linux workload volumes, set `csi.storage.k8s.io/fstype=ext4`; for Windows workload volumes set `csi.storage.k8s.io/fstype=ntfs`. +Specifically, if you have X number of storage classes for linux workloads, you need to create another X number of storage classes for Windows workloads. +This is helpful for Velero to deduce the right OS type successfully all the time, especially when you are backing up below kind of volumes belonging to a Windows workload: +- The PVC is with Immediate mode +- There is no pod mounting the PVC at the time of backup + +For restore, Velero automatically inherits the OS type from backup, so no deduction process is required. + +For other information, check [CSI Snapshot Data Movement][3]. + + +## Backup Repository Maintenance job + +Backup Repository Maintenance jobs and pods are supported to run in Windows nodes, that is, you can take full node resources in a cluster with Windows nodes for Backup Repository Maintenance. For more information, check [Repository Maintenance][4]. + +## Backup restore hooks + +Pre/post backup/restore hooks are supported for Windows workloads, the commands run in the same Windows nodes hosting the workload pods. For more information, check [Backup Hooks][5] and [Restore Hooks][6]. + +## Limitations + +NTFS extended attributes/advanced features are not supported, i.e., Security Descriptors, System/Hidden/ReadOnly attributes, Creation Time, NTFS Streams, etc. That is, after backup/restore, these data will be lost. + + + +[1]: customize-installation.md#customize-resource-requests-and-limits +[2]: build-from-source.md#image-building +[3]: csi-snapshot-data-movement.md +[4]: repository-maintenance.md +[5]: backup-hooks.md +[6]: restore-hooks.md \ No newline at end of file diff --git a/site/content/docs/v1.18/basic-install.md b/site/content/docs/v1.18/basic-install.md new file mode 100644 index 000000000..018fea341 --- /dev/null +++ b/site/content/docs/v1.18/basic-install.md @@ -0,0 +1,73 @@ +--- +title: "Basic Install" +layout: docs +--- + +Use this doc to get a basic installation of Velero. +Refer [this document](customize-installation.md) to customize your installation, including setting priority classes for Velero components. + +## Prerequisites + +- Access to a Kubernetes cluster, v1.16 or later, with DNS and container networking enabled. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix). +- `kubectl` installed locally + +Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. + +Velero supports storage providers for both cloud-provider environments and on-premises environments. For more details on on-premises scenarios, see the [on-premises documentation][2]. + +### Velero on Windows + +Velero supports to backup and restore Windows workloads, either stateless or stateful. +Velero node-agent and data mover pods could run in Windows nodes. To keep compatibility to the existing Velero plugins, Velero server runs in linux nodes only, so Velero requires at least one linux node in the cluster. Velero provides Windows images for specific Windows versions. For more information see [Backup Restore Windows Workloads][6]. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest release][1]'s tarball for your client platform. +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz + ``` + +1. Move the extracted `velero` binary to somewhere in your `$PATH` (`/usr/local/bin` for most users). + +### Option 3: Windows - Chocolatey + +On Windows, you can use [Chocolatey](https://chocolatey.org/install) to install the [velero](https://chocolatey.org/packages/velero) client: + +```powershell +choco install velero +``` + +## Install and configure the server components + +There are two supported methods for installing the Velero server components: + +- the `velero install` CLI command +- the [Helm chart](https://vmware-tanzu.github.io/helm-charts/) + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. The steps to install and configure the Velero server components along with the appropriate plugins are specific to your chosen storage provider. To find installation instructions for your chosen storage provider, follow the documentation link for your provider at our [supported storage providers][0] page + +_Note: if your object storage provider is different than your volume snapshot provider, follow the installation instructions for your object storage provider first, then return here and follow the instructions to [add your volume snapshot provider][4]._ + +## Command line Autocompletion + +Please refer to [this part of the documentation][5]. + +[0]: supported-providers.md +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: on-premises.md +[3]: overview-plugins.md +[4]: customize-installation.md#install-an-additional-volume-snapshot-provider +[5]: customize-installation.md#optional-velero-cli-configurations +[6]: backup-restore-windows.md diff --git a/site/content/docs/v1.18/build-from-source.md b/site/content/docs/v1.18/build-from-source.md new file mode 100644 index 000000000..5f617c8ae --- /dev/null +++ b/site/content/docs/v1.18/build-from-source.md @@ -0,0 +1,198 @@ +--- +title: "Build from source" +layout: docs +--- + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. +* A DNS server on the cluster +* `kubectl` installed +* [Go][5] installed (minimum version 1.8) + +## Get the source + +### Option 1) Get latest (recommended) + +```bash +mkdir $HOME/go +export GOPATH=$HOME/go +go get github.com/vmware-tanzu/velero +``` + +Where `go` is your [import path][4] for Go. + +For Go development, it is recommended to add the Go import path (`$HOME/go` in this example) to your path. + +### Option 2) Release archive + +Download the archive named `Source code` from the [release page][22] and extract it in your Go import path as `src/github.com/vmware-tanzu/velero`. + +Note that the Makefile targets assume building from a git repository. When building from an archive, you will be limited to the `go build` commands described below. + +## Build + +There are a number of different ways to build `velero` depending on your needs. This section outlines the main possibilities. + +When building by using `make`, it will place the binaries under `_output/bin/$GOOS/$GOARCH`. For example, you will find the binary for darwin here: `_output/bin/darwin/amd64/velero`, and the binary for linux here: `_output/bin/linux/amd64/velero`. `make` will also splice version and git commit information in so that `velero version` displays proper output. + +Note: `velero install` will also use the version information to determine which tagged image to deploy. If you would like to overwrite what image gets deployed, use the `image` flag (see below for instructions on how to build images). + +### Build the binary + +To build the `velero` binary on your local machine, compiled for your OS and architecture, run one of these two commands: + +```bash +go build ./cmd/velero +``` + +```bash +make local +``` + +### Cross compiling + +To build the velero binary targeting linux/amd64 within a build container on your local machine, run: + +```bash +make build +``` + +For any specific platform, run `make build--`. + +For example, to build for the Mac, run `make build-darwin-amd64`. + +Velero's `Makefile` has a convenience target, `all-build`, that builds the following platforms: + +* linux-amd64 +* linux-arm +* linux-arm64 +* linux-ppc64le +* darwin-amd64 +* windows-amd64 + +## Making images and updating Velero + +If after installing Velero you would like to change the image used by its deployment to one that contains your code changes, you may do so by updating the image: + +```bash +kubectl -n velero set image deploy/velero velero=myimagerepo/velero:$VERSION +``` + +To build a Velero container image, you need to configure `buildx` first. + +### Buildx + +Docker Buildx is a CLI plugin that extends the docker command with the full support of the features provided by Moby BuildKit builder toolkit. It provides the same user experience as docker build with many new features like creating scoped builder instances and building against multiple nodes concurrently. + +More information in the [docker docs][23] and in the [buildx github][24] repo. + +### Image building + +#### Build local image + +If you want to build an image with the same OS type and CPU architecture with your local machine, you can keep most the build parameters as default. +Run below command to build the local image: +```bash +make container +``` +Optionally, set the `$VERSION` environment variable to change the image tag or `$BIN` to change which binary to build a container image for. +Optionally, you can set the `$REGISTRY` environment variable. For example, if you want to build the `gcr.io/my-registry/velero:main` image, set `$REGISTRY` to `gcr.io/my-registry`. If this variable is not set, the default is `velero`. +The image is preserved in the local machine, you can run `docker push` to push the image to the specified registry, or if not specified, docker hub by default. + +#### Build hybrid image + +You can also build a hybrid image that supports multiple OS types or CPU architectures. A hybrid image contains a manifest list with one or more manifests each of which maps to a single `os type/arch/os version` configuration. +Below `os type/arch/os version` configurations are tested and supported: +* `linux/amd64` +* `linux/arm64` +* `windows/amd64/ltsc2022` + +The hybrid image must be pushed to a registry as the local system doesn't support all the manifests in the image. So `BUILDX_OUTPUT_TYPE` parameter must be set as `registry`. +By default, `$REGISTRY` is set as `velero`, you can change it to your own registry. + +To build a hybrid image, the following one time setup is necessary: + +1. If you are building cross platform container images + ```bash + $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ``` +2. Create and bootstrap a new docker buildx builder + ```bash + $ docker buildx create --use --name builder + builder + $ docker buildx inspect --bootstrap + [+] Building 2.6s (1/1) FINISHED + => [internal] booting buildkit 2.6s + => => pulling image moby/buildkit:buildx-stable-1 1.9s + => => creating container buildx_buildkit_builder0 0.7s + Name: builder + Driver: docker-container + + Nodes: + Name: builder0 + Endpoint: unix:///var/run/docker.sock + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + NOTE: Without the above setup, the output of `docker buildx inspect --bootstrap` will be: + ```bash + $ docker buildx inspect --bootstrap + Name: default + Driver: docker + + Nodes: + Name: default + Endpoint: default + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + And the `REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container` will fail with the below error: + ```bash + $ REGISTRY=ashishamarnath BUILDX_PLATFORMS=linux/arm64 BUILDX_OUTPUT_TYPE=registry make container + auto-push is currently not implemented for docker driver + make: *** [container] Error 1 + ``` + +Having completed the above one time setup, now the output of `docker buildx inspect --bootstrap` should be like + +```bash +$ docker buildx inspect --bootstrap +Name: builder +Driver: docker-container + +Nodes: +Name: builder0 +Endpoint: unix:///var/run/docker.sock +Status: running +Platforms: linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v +``` + +Now build and push the container image by running the `make container` command with `$BUILDX_OUTPUT_TYPE` set to `registry`. + +Blow command builds a hybrid image with single configuration `linux/amd64`: +```bash +$ REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container +``` + +Blow command builds a hybrid image with configurations `linux/amd64` and `linux/arm64`: +```bash +$ REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry BUILD_ARCH=amd64,arm64 make container +``` + +Blow command builds a hybrid image with configurations `linux/amd64`, `linux/arm64` and `windows/amd64/ltsc2022`: +```bash +$ REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container +``` + +Note: if you want to update the image but not change its name, you will have to trigger Kubernetes to pick up the new image. One way of doing so is by deleting the Velero deployment pod and node-agent pods: + +```bash +kubectl -n velero delete pods -l deploy=velero +``` + +[4]: https://blog.golang.org/organizing-go-code +[5]: https://golang.org/doc/install +[22]: https://github.com/vmware-tanzu/velero/releases +[23]: https://docs.docker.com/buildx/working-with-buildx/ +[24]: https://github.com/docker/buildx diff --git a/site/content/docs/v1.18/code-standards.md b/site/content/docs/v1.18/code-standards.md new file mode 100644 index 000000000..9a6e0a014 --- /dev/null +++ b/site/content/docs/v1.18/code-standards.md @@ -0,0 +1,171 @@ +--- +title: "Code Standards" +layout: docs +toc: "true" +--- + +## Opening PRs + +When opening a pull request, please fill out the checklist supplied the template. This will help others properly categorize and review your pull request. + +### PR title + +Make sure that the pull request title summarizes the change made (and not just "fixes issue #xxxx"): + +Example PR titles: + + - "Check for nil when validating foo" + - "Issue #1234: Check for nil when validating foo" + +### Cherry-pick PRs + +When a PR to main needs to be cherry-picked to a release branch, please wait until the main PR is merged first before creating the CP PR. If the CP PR is made before the main PR is merged, there is a risk that PR modifications in response to review comments will not make it into the CP PR. + +The Cherry-pick PR title should reference the branch it's cherry-picked to and the fact that it's a CP of a commit to main: + + - "[release-1.13 CP] Issue #1234: Check for nil when validating foo" + + +## Adding a changelog + +Authors are expected to include a changelog file with their pull requests. The changelog file +should be a new file created in the `changelogs/unreleased` folder. The file should follow the +naming convention of `pr-username` and the contents of the file should be your text for the +changelog. + + velero/changelogs/unreleased <- folder + 000-username <- file + +Add that to the PR. + +A command to do this is `make new-changelog CHANGELOG_BODY="Changes you have made"` + +If a PR does not warrant a changelog, the CI check for a changelog can be skipped by applying a `changelog-not-required` label on the PR. If you are making a PR on a release branch, you should still make a new file in the `changelogs/unreleased` folder on the release branch for your change. + +## Copyright header + +Whenever a source code file is being modified, the copyright notice should be updated to our standard copyright notice. That is, it should read “Copyright the Velero contributors.” + +For new files, the entire copyright and license header must be added. + +Please note that doc files do not need a copyright header. + +## Code + +- Log messages are capitalized. + +- Error messages are kept lower-cased. + +- Wrap/add a stack only to errors that are being directly returned from non-velero code, such as an API call to the Kubernetes server. + + ```bash + errors.WithStack(err) + ``` + +- Prefer to use the utilities in the Kubernetes package [`sets`](https://godoc.org/github.com/kubernetes/apimachinery/pkg/util/sets). + + ```bash + k8s.io/apimachinery/pkg/util/sets + ``` + +## Imports + +For imports, we use the following convention: + +`` + +Example: + + import ( + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + ) + +## Mocks + +We use a package to generate mocks for our interfaces. + +Example: if you want to change this mock: https://github.com/vmware-tanzu/velero/blob/v1.18.0/pkg/podvolume/mocks/restorer.go + +Run: + +```bash +go get github.com/vektra/mockery/.../ +cd pkg/podvolume +mockery -name=Restorer +``` + +Might need to run `make update` to update the imports. + +## Kubernetes Labels + +When generating label values, be sure to pass them through the `label.GetValidName()` helper function. + +This will help ensure that the values are the proper length and format to be stored and queried. + +In general, UIDs are safe to persist as label values. + +This function is not relevant to annotation values, which do not have restrictions. + +## DCO Sign off + +All authors to the project retain copyright to their work. However, to ensure +that they are only submitting work that they have rights to, we are requiring +everyone to acknowledge this by signing their work. + +Any copyright notices in this repo should specify the authors as "the Velero contributors". + +To sign your work, just add a line like this at the end of your commit message: + +``` +Signed-off-by: Joe Beda +``` + +This can easily be done with the `--signoff` option to `git commit`. + +By doing this you state that you can certify the following (from [https://developercertificate.org/](https://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` diff --git a/site/content/docs/v1.18/contributions/ibm-config.md b/site/content/docs/v1.18/contributions/ibm-config.md new file mode 100644 index 000000000..464f53c82 --- /dev/null +++ b/site/content/docs/v1.18/contributions/ibm-config.md @@ -0,0 +1,102 @@ +--- +title: "Use IBM Cloud Object Storage as Velero's storage destination." +layout: docs +--- +You can deploy Velero on IBM [Public][5] or [Private][4] clouds, or even on any other Kubernetes cluster, but anyway you can use IBM Cloud Object Store as a destination for Velero's backups. + +To set up IBM Cloud Object Storage (COS) as Velero's destination, you: + +* Download an official release of Velero +* Create your COS instance +* Create an S3 bucket +* Define a service that can store data in the bucket +* Configure and start the Velero server + +## Download Velero + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Create COS instance +If you don’t have a COS instance, you can create a new one, according to the detailed instructions in [Creating a new resource instance][1]. + +## Create an S3 bucket +Velero requires an object storage bucket to store backups in. See instructions in [Create some buckets to store your data][2]. + +## Define a service that can store data in the bucket. +The process of creating service credentials is described in [Service credentials][3]. +Several comments: + +1. The Velero service will write its backup into the bucket, so it requires the “Writer” access role. + +2. Velero uses an AWS S3 compatible API. Which means it authenticates using a signature created from a pair of access and secret keys — a set of HMAC credentials. You can create these HMAC credentials by specifying `{“HMAC”:true}` as an optional inline parameter. See [HMAC credentials][31] guide. + +3. After successfully creating a Service credential, you can view the JSON definition of the credential. Under the `cos_hmac_keys` entry there are `access_key_id` and `secret_access_key`. Use them in the next step. + +4. Create a Velero-specific credentials file (`credentials-velero`) in your local directory: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + + Where the access key id and secret are the values that you got above. + +## Install and start Velero + +Install Velero, including all prerequisites, into the cluster and start the deployment. This will create a namespace called `velero`, and place a deployment named `velero` in it. + +```bash +velero install \ + --provider aws \ + --bucket \ + --secret-file ./credentials-velero \ + --plugins velero/velero-plugin-for-aws:v1.10.0\ + --use-volume-snapshots=false \ + --backup-location-config region=,s3ForcePathStyle="true",s3Url=,checksumAlgorithm="" +``` + +Velero does not have a volume snapshot plugin for IBM Cloud, so creating volume snapshots is disabled. + +Additionally, you can specify `--use-node-agent` to enable [File System Backup][16], and `--wait` to wait for the deployment to be ready. + +(Optional) Specify [CPU and memory resource requests and limits][15] for the Velero/node-agent pods. + +Once the installation is complete, remove the default `VolumeSnapshotLocation` that was created by `velero install`, since it's specific to AWS and won't work for IBM Cloud: + +```bash +kubectl -n velero delete volumesnapshotlocation.velero.io default +``` + +For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. + +## Installing the nginx example (optional) + +If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`: + +Uncomment `storageClassName: ` and replace with your `StorageClass` name. + +[0]: ../namespace.md +[1]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html +[2]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html#create-buckets +[3]: https://cloud.ibm.com/docs/cloud-object-storage/iam?topic=cloud-object-storage-service-credentials +[31]: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-uhc-hmac-credentials-main +[4]: https://www.ibm.com/docs/en/cloud-private +[5]: https://cloud.ibm.com/docs/containers/container_index.html#container_index +[14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html +[15]: ../customize-installation.md#customize-resource-requests-and-limits +[16]: ../file-system-backup.md diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png b/site/content/docs/v1.18/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png new file mode 100644 index 000000000..61859ca50 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png b/site/content/docs/v1.18/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png new file mode 100644 index 000000000..57de0cde0 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png b/site/content/docs/v1.18/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png new file mode 100644 index 000000000..f363eb1a7 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png b/site/content/docs/v1.18/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png new file mode 100644 index 000000000..921b54a16 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png b/site/content/docs/v1.18/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png new file mode 100644 index 000000000..36805831b Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png b/site/content/docs/v1.18/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png new file mode 100644 index 000000000..c64f47a81 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png b/site/content/docs/v1.18/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png new file mode 100644 index 000000000..188c2db52 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png b/site/content/docs/v1.18/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png new file mode 100644 index 000000000..3eaee3553 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png b/site/content/docs/v1.18/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png new file mode 100644 index 000000000..28e1bbaad Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png differ diff --git a/site/content/docs/v1.18/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png b/site/content/docs/v1.18/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png new file mode 100644 index 000000000..b732cf252 Binary files /dev/null and b/site/content/docs/v1.18/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png differ diff --git a/site/content/docs/v1.18/contributions/minio.md b/site/content/docs/v1.18/contributions/minio.md new file mode 100644 index 000000000..4b5aaa8e5 --- /dev/null +++ b/site/content/docs/v1.18/contributions/minio.md @@ -0,0 +1,301 @@ +--- +title: "Quick start evaluation install with Minio" +layout: docs +--- + +The following example sets up the Velero server and client, then backs up and restores a sample application. + +For simplicity, the example uses Minio, an S3-compatible storage service that runs locally on your cluster. +For additional functionality with this setup, see the section below on how to [expose Minio outside your cluster][1]. + +**NOTE** The example lets you explore basic Velero functionality. Configuring Minio for production is out of scope. + +See [Set up Velero on your platform][3] for how to configure Velero for a production environment. + +If you encounter issues with installing or configuring, see [Debugging Installation Issues](debugging-install.md). + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. **Note:** File System Backup support requires Kubernetes version 1.10 or later, or an earlier version with the mount propagation feature enabled. File System Backup support is not required for this example, but may be of interest later. See [File System Backup][17]. +* A DNS server on the cluster +* `kubectl` installed +* Sufficient disk space to store backups in Minio. You will need sufficient disk space available to handle any +backups plus at least 1GB additional. Minio will not operate if less than 1GB of free disk space is available. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Set up server + +These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster](#expose-minio-outside-your-cluster-with-a-service) for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. + +1. Create a Velero-specific credentials file (`credentials-velero`) in your Velero directory: + + ``` + [default] + aws_access_key_id = minio + aws_secret_access_key = minio123 + ``` + +1. Start the server and the local storage service. In the Velero directory, run: + + ``` + kubectl apply -f examples/minio/00-minio-deployment.yaml + ``` + _Note_: The example Minio yaml provided uses "empty dir". Your node needs to have enough space available to store the + data being backed up plus 1GB of free space. If the node does not have enough space, you can modify the example yaml to + use a Persistent Volume instead of "empty dir" + + ``` + velero install \ + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.2.1 \ + --bucket velero \ + --secret-file ./credentials-velero \ + --use-volume-snapshots=false \ + --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://minio.velero.svc:9000 + ``` + + * This example assumes that it is running within a local cluster without a volume provider capable of snapshots, so no `VolumeSnapshotLocation` is created (`--use-volume-snapshots=false`). You may need to update AWS plugin version to one that is [compatible](https://github.com/vmware-tanzu/velero-plugin-for-aws#compatibility) with the version of Velero you are installing. + + * Additionally, you can specify `--use-node-agent` to enable File System Backup support, and `--wait` to wait for the deployment to be ready. + + * This example also assumes you have named your Minio bucket "velero". + + * Please make sure to set parameter `s3ForcePathStyle=true`. The parameter is used to set the Velero integrated AWS SDK data query address style. There are two types of the address: [virtual-host and path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). If the `s3ForcePathStyle=true` is not set, the default value is false, then the AWS SDK will query in virtual-host style, but the MinIO server only support path-style address by default. The miss match will mean Velero can upload data to MinIO, but **cannot download from MinIO**. This [link](https://github.com/vmware-tanzu/velero/issues/7268) is an example of this issue. + It can be resolved by two ways: + * Set `s3ForcePathStyle=true` for parameter `--backup-location-config` when installing Velero. This is the preferred way. + * Make MinIO server support virtual-host style address. Add the [MINIO_DOMAIN environment variable](https://min.io/docs/minio/linux/reference/minio-server/settings/core.html#id5) for MinIO server will do the magic. + + +1. Deploy the example nginx application: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Check to see that both the Velero and nginx deployments are successfully created: + + ``` + kubectl get deployments -l component=velero --namespace=velero + kubectl get deployments --namespace=nginx-example + ``` + +## Back up + +1. Create a backup for any object that matches the `app=nginx` label selector: + + ``` + velero backup create nginx-backup --selector app=nginx + ``` + + Alternatively if you want to backup all objects *except* those matching the label `backup=ignore`: + + ``` + velero backup create nginx-backup --selector 'backup notin (ignore)' + ``` + +1. (Optional) Create regularly scheduled backups based on a cron expression using the `app=nginx` label selector: + + ``` + velero schedule create nginx-daily --schedule="0 1 * * *" --selector app=nginx + ``` + + Alternatively, you can use some non-standard shorthand cron expressions: + + ``` + velero schedule create nginx-daily --schedule="@daily" --selector app=nginx + ``` + + See the [cron package's documentation][30] for more usage examples. + +1. Simulate a disaster: + + ``` + kubectl delete namespace nginx-example + ``` + +1. To check that the nginx deployment and service are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + You should get no results. + + NOTE: You might need to wait for a few minutes for the namespace to be fully cleaned up. + +## Restore + +1. Run: + + ``` + velero restore create --from-backup nginx-backup + ``` + +1. Run: + + ``` + velero restore get + ``` + + After the restore finishes, the output looks like the following: + + ``` + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20170727200524 nginx-backup Completed 0 0 2017-07-27 20:05:24 +0000 UTC + ``` + +NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + +After a successful restore, the `STATUS` column is `Completed`, and `WARNINGS` and `ERRORS` are 0. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + +If there are errors or warnings, you can look at them in detail: + +``` +velero restore describe +``` + +For more information, see [the debugging information][18]. + +## Clean up + +If you want to delete any backups you created, including data in object storage and persistent +volume snapshots, you can run: + +``` +velero backup delete BACKUP_NAME +``` + +This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do +this for each backup you want to permanently delete. A future version of Velero will allow you to +delete multiple backups by name or label selector. + +Once fully removed, the backup is no longer visible when you run: + +``` +velero backup get BACKUP_NAME +``` + +To completely uninstall Velero, minio, and the nginx example app from your Kubernetes cluster: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +kubectl delete -f examples/nginx-app/base.yaml +``` + +## Expose Minio outside your cluster with a Service + +When you run commands to get logs or describe a backup, the Velero server generates a pre-signed URL to download the requested items. To access these URLs from outside the cluster -- that is, from your Velero client -- you need to make Minio available outside the cluster. You can: + +- Change the Minio Service type from `ClusterIP` to `NodePort`. +- Set up Ingress for your cluster, keeping Minio Service type `ClusterIP`. + +You can also specify a `publicUrl` config field for the pre-signed URL in your backup storage location config. + +### Expose Minio with Service of type NodePort + +The Minio deployment by default specifies a Service of type `ClusterIP`. You can change this to `NodePort` to easily expose a cluster service externally if you can reach the node from your Velero client. + +You must also get the Minio URL, which you can then specify as the value of the `publicUrl` field in your backup storage location config. + +1. In `examples/minio/00-minio-deployment.yaml`, change the value of Service `spec.type` from `ClusterIP` to `NodePort`. + +1. Get the Minio URL: + + - if you're running Minikube: + + ```shell + minikube service minio --namespace=velero --url + ``` + + - in any other environment: + 1. Get the value of an external IP address or DNS name of any node in your cluster. You must be able to reach this address from the Velero client. + 1. Append the value of the NodePort to get a complete URL. You can get this value by running: + + ```shell + kubectl -n velero get svc/minio -o jsonpath='{.spec.ports[0].nodePort}' + ``` + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. You must include the `http://` or `https://` prefix. + +## Accessing logs with an HTTPS endpoint + +If you're using Minio with HTTPS, you may see unintelligible text in the output of `velero describe`, or `velero logs` commands. + +To fix this, you can add a public URL to the `BackupStorageLocation`. + +In a terminal, run the following: + +```shell +kubectl patch -n velero backupstoragelocation default --type merge -p '{"spec":{"config":{"publicUrl":"https://:9000"}}}' +``` + +If your certificate is self-signed, see the [documentation on self-signed certificates][32]. + +## Expose Minio outside your cluster with Kubernetes in Docker (KinD): + +Kubernetes in Docker does not have support for NodePort services (see [this issue](https://github.com/kubernetes-sigs/kind/issues/99)). In this case, you can use a port forward to access the Minio bucket. + +In a terminal, run the following: + +```shell +MINIO_POD=$(kubectl get pods -n velero -l component=minio -o jsonpath='{.items[0].metadata.name}') + +kubectl port-forward $MINIO_POD -n velero 9000:9000 +``` + +Then, in another terminal: + +```shell +kubectl edit backupstoragelocation default -n velero +``` + +Add `publicUrl: http://localhost:9000` under the `spec.config` section. + + +### Work with Ingress + +Configuring Ingress for your cluster is out of scope for the Velero documentation. If you have already set up Ingress, however, it makes sense to continue with it while you run the example Velero configuration with Minio. + +In this case: + +1. Keep the Service type as `ClusterIP`. + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. + +[1]: #expose-minio-with-service-of-type-nodeport +[3]: ../customize-installation.md +[17]: ../file-system-backup.md +[18]: ../debugging-restores.md +[26]: https://github.com/vmware-tanzu/velero/releases +[30]: https://godoc.org/github.com/robfig/cron +[32]: ../self-signed-certificates.md diff --git a/site/content/docs/v1.18/contributions/oracle-config.md b/site/content/docs/v1.18/contributions/oracle-config.md new file mode 100644 index 000000000..210f32c15 --- /dev/null +++ b/site/content/docs/v1.18/contributions/oracle-config.md @@ -0,0 +1,248 @@ +--- +title: "Use Oracle Cloud as a Backup Storage Provider for Velero" +layout: docs +--- + +## Introduction + +[Velero](https://velero.io/) is a tool used to backup and migrate Kubernetes applications. Here are the steps to use [Oracle Cloud Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Concepts/objectstorageoverview.htm) as a destination for Velero backups. + +1. [Download Velero](#download-velero) +2. [Create A Customer Secret Key](#create-a-customer-secret-key) +3. [Create An Oracle Object Storage Bucket](#create-an-oracle-object-storage-bucket) +4. [Install Velero](#install-velero) +5. [Clean Up](#clean-up) +6. [Examples](#examples) +7. [Additional Reading](#additional-reading) + +## Download Velero + +1. Download the [latest release](https://github.com/vmware-tanzu/velero/releases/) of Velero to your development environment. This includes the `velero` CLI utility and example Kubernetes manifest files. For example: + + ``` + wget https://github.com/vmware-tanzu/velero/releases/download/v1.0.0/velero-v1.0.0-linux-amd64.tar.gz + ``` + + **NOTE:** Its strongly recommend that you use an official release of Velero. The tarballs for each release contain the velero command-line client. The code in the main branch of the Velero repository is under active development and is not guaranteed to be stable! + +2. Untar the release in your `/usr/bin` directory: `tar -xzvf .tar.gz` + + You may choose to rename the directory `velero` for the sake of simplicity: `mv velero-v1.0.0-linux-amd64 velero` + +3. Add it to your PATH: `export PATH=/usr/local/bin/velero:$PATH` + +4. Run `velero` to confirm the CLI has been installed correctly. You should see an output like this: + +``` +$ velero +Velero is a tool for managing disaster recovery, specifically for Kubernetes +cluster resources. It provides a simple, configurable, and operationally robust +way to back up your application state and associated data. + +If you're familiar with kubectl, Velero supports a similar model, allowing you to +execute commands such as 'velero get backup' and 'velero create schedule'. The same +operations can also be performed as 'velero backup get' and 'velero schedule create'. + +Usage: + velero [command] +``` + + + +## Create A Customer Secret Key + +1. Oracle Object Storage provides an API to enable interoperability with Amazon S3. To use this Amazon S3 Compatibility API, you need to generate the signing key required to authenticate with Amazon S3. This special signing key is an Access Key/Secret Key pair. Follow these steps to [create a Customer Secret Key](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#To4). Refer to this link for more information about [Working with Customer Secret Keys](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#s3). + +2. Create a Velero credentials file with your Customer Secret Key: + + ``` + $ vi credentials-velero + + [default] + aws_access_key_id=bae031188893d1eb83719648790ac850b76c9441 + aws_secret_access_key=MmY9heKrWiNVCSZQ2Mf5XTJ6Ys93Bw2d2D6NMSTXZlk= + ``` + + + +## Create An Oracle Object Storage Bucket + +Create an Oracle Cloud Object Storage bucket called `velero` in the root compartment of your Oracle Cloud tenancy. Refer to this page for [more information about creating a bucket with Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Tasks/managingbuckets.htm#usingconsole). + + + +## Install Velero + +You will need the following information to install Velero into your Kubernetes cluster with Oracle Object Storage as the Backup Storage provider: + +``` +velero install \ + --provider [provider name] \ + --bucket [bucket name] \ + --prefix [tenancy name] \ + --use-volume-snapshots=false \ + --secret-file [secret file location] \ + --backup-location-config region=[region],s3ForcePathStyle="true",s3Url=[storage API endpoint] +``` + +- `--provider` This example uses the S3-compatible API, so use `aws` as the provider. +- `--bucket` The name of the bucket created in Oracle Object Storage - in our case this is named `velero`. +- ` --prefix` The name of your Oracle Cloud tenancy - in our case this is named `oracle-cloudnative`. +- `--use-volume-snapshots=false` Velero does not have a volume snapshot plugin for Oracle Cloud, so creating volume snapshots is disabled. +- `--secret-file` The path to your `credentials-velero` file. +- `--backup-location-config` The path to your Oracle Object Storage bucket. This consists of your `region` which corresponds to your Oracle Cloud region name ([List of Oracle Cloud Regions](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm?Highlight=regions)) and the `s3Url`, the S3-compatible API endpoint for Oracle Object Storage based on your region: `https://oracle-cloudnative.compat.objectstorage.[region name].oraclecloud.com` + +For example: + +``` +velero install \ + --provider aws \ + --bucket velero \ + --prefix oracle-cloudnative \ + --use-volume-snapshots=false \ + --secret-file /Users/mboxell/bin/velero/credentials-velero \ + --backup-location-config region=us-phoenix-1,s3ForcePathStyle="true",s3Url=https://oracle-cloudnative.compat.objectstorage.us-phoenix-1.oraclecloud.com +``` + +This will create a `velero` namespace in your cluster along with a number of CRDs, a ClusterRoleBinding, ServiceAccount, Secret, and Deployment for Velero. If your pod fails to successfully provision, you can troubleshoot your installation by running: `kubectl logs [velero pod name]`. + + + +## Clean Up + +To remove Velero from your environment, delete the namespace, ClusterRoleBinding, ServiceAccount, Secret, and Deployment and delete the CRDs, run: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + +This will remove all resources created by `velero install`. + + + +## Examples + +After creating the Velero server in your cluster, try this example: + +### Basic example (without PersistentVolumes) + +1. Start the sample nginx app: `kubectl apply -f examples/nginx-app/base.yaml` + + This will create an `nginx-example` namespace with a `nginx-deployment` deployment, and `my-nginx` service. + + ``` + $ kubectl apply -f examples/nginx-app/base.yaml + namespace/nginx-example created + deployment.apps/nginx-deployment created + service/my-nginx created + ``` + + You can see the created resources by running `kubectl get all` + + ``` + $ kubectl get all + NAME READY STATUS RESTARTS AGE + pod/nginx-deployment-67594d6bf6-4296p 1/1 Running 0 20s + pod/nginx-deployment-67594d6bf6-f9r5s 1/1 Running 0 20s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/my-nginx LoadBalancer 10.96.69.166 80:31859/TCP 21s + + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + deployment.apps/nginx-deployment 2 2 2 2 21s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/nginx-deployment-67594d6bf6 2 2 2 21s + ``` + +2. Create a backup: `velero backup create nginx-backup --include-namespaces nginx-example` + + ``` + $ velero backup create nginx-backup --include-namespaces nginx-example + Backup request "nginx-backup" submitted successfully. + Run `velero backup describe nginx-backup` or `velero backup logs nginx-backup` for more details. + ``` + + At this point you can navigate to appropriate bucket, called `velero`, in the Oracle Cloud Object Storage console to see the resources backed up using Velero. + +3. Simulate a disaster by deleting the `nginx-example` namespace: `kubectl delete namespaces nginx-example` + + ``` + $ kubectl delete namespaces nginx-example + namespace "nginx-example" deleted + ``` + + Wait for the namespace to be deleted. To check that the nginx deployment, service, and namespace are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + This should return: `No resources found.` + +4. Restore your lost resources: `velero restore create --from-backup nginx-backup` + + ``` + $ velero restore create --from-backup nginx-backup + Restore request "nginx-backup-20190604102710" submitted successfully. + Run `velero restore describe nginx-backup-20190604102710` or `velero restore logs nginx-backup-20190604102710` for more details. + ``` + + Running `kubectl get namespaces` will show that the `nginx-example` namespace has been restored along with its contents. + +5. Run: `velero restore get` to view the list of restored resources. After the restore finishes, the output looks like the following: + + ``` + $ velero restore get + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20190604104249 nginx-backup Completed 0 0 2019-06-04 10:42:39 -0700 PDT + ``` + + NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + + After a successful restore, the `STATUS` column shows `Completed`, and `WARNINGS` and `ERRORS` will show `0`. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + + If there are errors or warnings, for instance if the `STATUS` column displays `FAILED` instead of `InProgress`, you can look at them in detail with `velero restore describe ` + + +6. Clean up the environment with `kubectl delete -f examples/nginx-app/base.yaml` + + ``` + $ kubectl delete -f examples/nginx-app/base.yaml + namespace "nginx-example" deleted + deployment.apps "nginx-deployment" deleted + service "my-nginx" deleted + ``` + + If you want to delete any backups you created, including data in object storage, you can run: `velero backup delete BACKUP_NAME` + + ``` + $ velero backup delete nginx-backup + Are you sure you want to continue (Y/N)? Y + Request to delete backup "nginx-backup" submitted successfully. + The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed. + ``` + + This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do this for each backup you want to permanently delete. A future version of Velero will allow you to delete multiple backups by name or label selector. + + Once fully removed, the backup is no longer visible when you run: `velero backup get BACKUP_NAME` or more generally `velero backup get`: + + ``` + $ velero backup get nginx-backup + An error occurred: backups.velero.io "nginx-backup" not found + ``` + + ``` + $ velero backup get + NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR + ``` + + + +## Additional Reading + +* [Official Velero Documentation](https://velero.io/docs/v1.18.0/) +* [Oracle Cloud Infrastructure Documentation](https://docs.cloud.oracle.com/) diff --git a/site/content/docs/v1.18/contributions/tencent-config.md b/site/content/docs/v1.18/contributions/tencent-config.md new file mode 100644 index 000000000..4ad54aaa8 --- /dev/null +++ b/site/content/docs/v1.18/contributions/tencent-config.md @@ -0,0 +1,168 @@ +--- +title: "Use Tencent Cloud Object Storage as Velero's storage destination." +layout: docs +--- + + +You can deploy Velero on Tencent [TKE](https://cloud.tencent.com/document/product/457), or an other Kubernetes cluster, and use Tencent Cloud Object Store as a destination for Velero’s backups. + + +## Prerequisites + +- Registered [Tencent Cloud Account](https://cloud.tencent.com/register). +- [Tencent Cloud COS](https://console.cloud.tencent.com/cos) service, referred to as COS, has been launched +- A Kubernetes cluster has been created, cluster version v1.16 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. + +## Create a Tencent Cloud COS bucket + +Create an object bucket for Velero to store backups in the Tencent Cloud COS console. For how to create, please refer to Tencent Cloud COS [Create a bucket](https://cloud.tencent.com/document/product/436/13309) usage instructions. + +Set access to the bucket through the object storage console, the bucket needs to be **read** and **written**, so the account is granted data reading, data writing permissions. For how to configure, see the [permission access settings](https://cloud.tencent.com/document/product/436/13315) Tencent user instructions. + +## Get bucket access credentials + +Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. + +In the S3 API parameter, the "access_key_id" field is the access key ID and the "secret_access_key" field is the key. + +In the [Tencent Cloud Access Management Console](https://console.cloud.tencent.com/cam/capi), Create and acquire Tencent Cloud Keys "SecretId" and "SecretKey" for COS authorized account. **Where the "SecretId" value corresponds to the value of S3 API parameter "access_key_id" field, the "SecretKey" value corresponds to the value of S3 API parameter "secret_access_key" field**. + +Create the credential profile "credentials-velero" required by Velero in the local directory based on the above correspondence: + +```bash +[default] +aws_access_key_id= +aws_secret_access_key= +``` + +## Install Velero Resources + +You need to install the Velero CLI first, see [Install the CLI](https://velero.io/docs/v1.5/basic-install/#install-the-cli) for how to install. + +Follow the Velero installation command below to create velero and node-agent workloads and other necessary resource objects. + +```bash +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.1.0 --bucket \ +--secret-file ./credentials-velero \ +--use-node-agent \ +--default-volumes-to-fs-backup \ +--backup-location-config \ +region=ap-guangzhou,s3ForcePathStyle="true",s3Url=https://cos.ap-guangzhou.myqcloud.com +``` + +Description of the parameters: + +- `--provider`: Declares the type of plugin provided by "aws". + +- `--plugins`: Use the AWS S3 compatible API plugin "velero-plugin-for-aws". + +- `--bucket`: The bucket name created at Tencent Cloud COS. + +- `--secret-file`: Access tencent cloud COS access credential file for the "credentials-velero" credential file created above. + +- `--use-node-agent`: Enable Velero node-agent daemonset. At present, Velero File System Backup requires this daemonset, so if you are using File System Backup, it needs to be turned on. For the usage and limitation of File System Backup, See [File System Backup](../file-system-backup.md). + +- `--default-volumes-to-fs-backup`: Enable the use of File System Backup to back up all Pod volumes, provided that the `--use-node-agent`parameter needs to be turned on. + +- `--backup-location-config`: Back up the bucket access-related configuration: + + `region`: Tencent cloud COS bucket area, for example, if the created region is Guangzhou, the Region parameter value is "ap-guangzhou". + + `s3ForcePathStyle`: Use the S3 file path format. + + `s3Url`: Tencent Cloud COS-compatible S3 API access address,Note that instead of creating a COS bucket for public network access domain name, you must use a format of "https://cos.`region`.myqcloud.com" URL, for example, if the region is Guangzhou, the parameter value is "https://cos.ap-guangzhou.myqcloud.com.". + +There are other installation parameters that can be viewed using `velero install --help`, such as setting `--use-volume-snapshots-false` to close the storage volume data snapshot backup if you do not want to back up the storage volume data. + +After executing the installation commands above, the installation process looks like this: + +{{< figure src="/docs/main/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png" width="100%">}} + +After the installation command is complete, wait for the velero and node-agent workloads to be ready to see if the configured storage location is available. + +Executing the 'velero backup-location get' command to view the storage location status and display "Available" indicates that access to Tencent Cloud COS is OK, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png" width="100%">}} + +At this point, The installation using Tencent Cloud COS as Velero storage location is complete, If you need more installation information about Velero, You can see the official website [Velero documentation](https://velero.io/docs/) . + +## Velero backup and restore example + +In the cluster, use the helm tool to create a minio test service with a persistent volume, and the minio installation method can be found in the [minio installation](https://github.com/minio/charts), in which case can bound a load balancer for the minio service to access the management page using a public address in the browser. + +{{< figure src="/docs/main/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png" width="100%">}} + +Sign in to the minio web management page and upload some image data for the test, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png" width="100%">}} + +With Velero Backup, you can back up all objects in the cluster directly, or filter objects by type, namespace, and/or label. This example uses the following command to back up all resources under the 'default' namespace. + +``` +velero backup create default-backup --include-namespaces +``` + +Use the `velero backup get` command to see if the backup task is complete, and when the backup task status is "Completed," the backup task is completed without any errors, as shown in the following below: + +{{< figure src="/docs/main/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png" width="100%">}} + +At this point delete all of MinIO's resources, including its PVC persistence volume, as shown below:: + +{{< figure src="/docs/main/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png" width="100%">}} + +After deleting the MinIO resource, use your backup to restore the deleted MinIO resource, and temporarily update the backup storage location to read-only mode (this prevents the backup object from being created or deleted in the backup storage location during the restore process):: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + +``` + +Modifying access to Velero's storage location is "ReadOnly," as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png" width="100%">}} + +Now use the backup "default-backup" that Velero just created to create the restore task: + +```bash +velero restore create --from-backup +``` + +You can also use `velero restore get` to see the status of the restore task, and if the restore status is "Completed," the restore task is complete, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png" width="100%">}} + +When the restore is complete, you can see that the previously deleted minio-related resources have been restored successfully, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png" width="100%">}} + +Log in to minio's management page on your browser and you can see that the previously uploaded picture data is still there, indicating that the persistent volume's data was successfully restored, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png" width="100%">}} + +When the restore is complete, don't forget to restore the backup storage location to read and write mode so that the next backup task can be used successfully: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' +``` + + + +## Uninstall Velero Resources + +To uninstall velero resources in a cluster, you can do so using the following command: + +```bash +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + + + +## Additional Reading + +- [Official Velero Documentation](https://velero.io/docs/) +- [Tencent Cloud Documentation](https://cloud.tencent.com/document/product) diff --git a/site/content/docs/v1.18/csi-snapshot-data-movement.md b/site/content/docs/v1.18/csi-snapshot-data-movement.md new file mode 100644 index 000000000..cde862209 --- /dev/null +++ b/site/content/docs/v1.18/csi-snapshot-data-movement.md @@ -0,0 +1,431 @@ +--- +title: "CSI Snapshot Data Movement" +layout: docs +--- + +CSI Snapshot Data Movement is built according to the [Volume Snapshot Data Movement design][1] and is specifically designed to move CSI snapshot data to a backup storage location. +CSI Snapshot Data Movement takes CSI snapshots through the CSI plugin in nearly the same way as [CSI snapshot backup][2]. However, it doesn't stop after a snapshot is taken. Instead, it tries to access the snapshot data through various data movers and back up the data to a backup storage connected to the data movers. +Consequently, the volume data is backed up to a pre-defined backup storage in a consistent manner. +After the backup completes, the CSI snapshot will be removed by Velero and the snapshot data space will be released on the storage side. + +CSI Snapshot Data Movement is useful in below scenarios: +- For on-premises users, the storage usually doesn't support durable snapshots, so it is impossible/less efficient/cost ineffective to keep volume snapshots by the storage, as required by the [CSI snapshot backup][2]. This feature helps to move the snapshot data to a storage with lower cost and larger scale for long time preservation. +- For public cloud users, this feature helps users to fulfil the multiple cloud strategy. It allows users to back up volume snapshots from one cloud provider and preserve or restore the data to another cloud provider. Then users will be free to flow their business data across cloud providers based on Velero backup and restore. + +Besides, Velero [File System Backup][3] which could also back up the volume data to a pre-defined backup storage. CSI Snapshot Data Movement works together with [File System Backup][3] to satisfy different requirements for the above scenarios. And whenever available, CSI Snapshot Data Movement should be used in preference since the [File System Backup][3] reads data from the live PV, in which way the data is not captured at the same point in time, so is less consistent. +Moreover, CSI Snapshot Data Movement brings more possible ways of data access, i.e., accessing the data from the block level, either fully or incrementally. +On the other hand, there are quite some cases that CSI snapshot is not available (i.e., you need a volume snapshot plugin for your storage platform, or you're using EFS, NFS, emptyDir, local, or any other volume type that doesn't have a native snapshot), then [File System Backup][3] will be the only option. + +CSI Snapshot Data Movement supports both built-in data mover and customized data movers. For the details of how Velero works with customized data movers, check the [Volume Snapshot Data Movement design][1]. Velero provides a built-in data mover which uses Velero built-in uploaders (at present the available uploader is Kopia uploader) to read the snapshot data and write to the Unified Repository (by default implemented by Kopia repository). + +Velero built-in data mover restores both volume data and metadata, so the data mover pods need to run as root user. + +### Priority Class Configuration + +For Velero built-in data mover, data mover pods launched during CSI snapshot data movement will use the priority class name configured in the node-agent configmap. The node-agent daemonset itself gets its priority class from the `--node-agent-priority-class-name` flag during Velero installation. This can help ensure proper scheduling behavior in resource-constrained environments. For more details on configuring data mover pod resources, see [Data Movement Pod Resource Configuration][11]. + +## Setup CSI Snapshot Data Movement + +## Prerequisites + + 1. The source cluster is Kubernetes version 1.20 or greater. + 2. The source cluster is running a CSI driver capable of support volume snapshots at the [v1 API level][4]. + 3. CSI Snapshot Data Movement requires the Kubernetes [MountPropagation feature][5]. + + +### Install Velero Node Agent + +Velero Node Agent is a Kubernetes daemonset that hosts Velero data movement controllers and launches data mover pods. +If you are using Velero built-in data mover, Node Agent must be installed. To install Node Agent, use the `--use-node-agent` flag. +Velero built-in data mover doesn't require the host path for pod volumes into Node Agent pods. The installation by default creates it in order to support fs-backup. If you don't use fs-backup and want to remove it from Node Agent, you can specify the `--node-agent-disable-host-path` flag. + +``` +velero install --use-node-agent --node-agent-disable-host-path +``` + +### Configure A Backup Storage Location + +At present, Velero backup repository supports object storage as the backup storage. Velero gets the parameters from the +[BackupStorageLocation][8] to compose the URL to the backup storage. +Velero's known object storage providers are included here [supported providers][9], for which, Velero pre-defines the endpoints. If you want to use a different backup storage, make sure it is S3 compatible and you provide the correct bucket name and endpoint in BackupStorageLocation. Velero handles the creation of the backup repo prefix in the backup storage, so make sure it is specified in BackupStorageLocation correctly. + +Velero creates one backup repository per namespace. For example, if backing up 2 namespaces, namespace1 and namespace2, using kopia repository on AWS S3, the full backup repo path for namespace1 would be `https://s3-us-west-2.amazonaws.com/bucket/kopia/ns1` and for namespace2 would be `https://s3-us-west-2.amazonaws.com/bucket/kopia/ns2`. + +There may be additional installation steps depending on the cloud provider plugin you are using. You should refer to the [plugin specific documentation][9] for the must up to date information. + +**Note:** Currently, Velero creates a secret named `velero-repo-credentials` in the velero install namespace, containing a default backup repository password. +You can update the secret with your own password encoded as base64 prior to the first backup (i.e., [File System Backup][3], snapshot data movements) targeting to the backup repository. The value of the key to update is +``` +data: + repository-password: +``` +Backup repository is created during the first execution of backup targeting to it after installing Velero with node agent. If you update the secret password after the first backup which created the backup repository, then Velero will not be able to connect with the older backups. + +## Install Velero with CSI support on source cluster + +On source cluster, Velero needs to manipulate CSI snapshots through the CSI volume snapshot APIs, so you must enable the `EnableCSI` feature flag on the Velero server. + +To integrate Velero with the CSI volume snapshot APIs, you must enable the `EnableCSI` feature flag. + +From release-1.14, the `github.com/vmware-tanzu/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/vmware-tanzu/velero` repository. +The reasons to merge the CSI plugin are: +* The VolumeSnapshot data mover depends on the CSI plugin, it's reasonabe to integrate them. +* This change reduces the Velero deploying complexity. +* This makes performance tuning easier in the future. + +As a result, no need to install Velero CSI plugin anymore. + +```bash +velero install \ +--features=EnableCSI \ +--plugins= \ +... +``` + +### Configure storage class on target cluster + +For Velero built-in data movement, CSI facilities are not required necessarily in the target cluster. On the other hand, Velero built-in data movement creates a PVC with the same specification as it is in the source cluster and expects the volume to be provisioned similarly. For example, the same storage class should be working in the target cluster. +By default, Velero won't restore storage class resources from the backup since they are cluster scope resources. However, if you specify the `--include-cluster-resources` restore flag, they will be restored. For a cross provider scenario, the storage class from the source cluster is probably not usable in the target cluster. +In either of the above cases, the best practice is to create a working storage class in the target cluster with the same name as it in the source cluster. In this way, even though `--include-cluster-resources` is specified, Velero restore will skip restoring the storage class since it finds an existing one. +Otherwise, if the storage class name in the target cluster is different, you can change the PVC's storage class name during restore by the [changing PV/PVC storage class][10] method. You can also configure to skip restoring the storage class resources from the backup since they are not usable. + +### Customized Data Movers + +If you are using a customized data mover, follow the data mover's instructions for any further prerequisites. +For Velero side configurations mentioned above, the installation and configuration of node-agent may not be required. + + +## To back up + +Velero uses a new custom resource `DataUpload` to drive the data movement. The selected data mover will watch and reconcile the CRs. +Velero allows users to decide whether the CSI snapshot data should be moved per backup. +Velero also allows users to select the data mover to move the CSI snapshot data per backup. +The both selections are simply done by a parameter when running the backup. + +To take a backup with Velero's built-in data mover: + +```bash +velero backup create NAME --snapshot-move-data OPTIONS... +``` + +Or if you want to use a customized data mover: +```bash +velero backup create NAME --snapshot-move-data --data-mover DATA-MOVER-NAME OPTIONS... +``` + +When the backup starts, you will see the `VolumeSnapshot` and `VolumeSnapshotContent` objects created, but after the backup finishes, the objects will disappear. +After snapshots are created, you will see one or more `DataUpload` CRs created. +You may also see some intermediate objects (i.e., pods, PVCs, PVs) created in Velero namespace or the cluster scope, they are to help data movers to move data. And they will be removed after the backup completes. +The phase of a `DataUpload` CR changes several times during the backup process and finally goes to one of the terminal status, `Completed`, `Failed` or `Cancelled`. You can see the phase changes as well as the data upload progress by watching the `DataUpload` CRs. While the `DataUpload` is being processed, progress is shown with `BYTES DONE` representing the amount of volume data that has been processed so far and `TOTAL BYTES` representing the estimated total volume data. Upon completion, these two numbers will be the same. In addition, once the `DataUpload` is done, `INCREMENTAL BYTES` will be filled in with the amount of data which is new or changed since the last backup of this volume. Note that the actual uploaded content may be smaller than `INCREMENTAL BYTES` due to kopia deduplication, compression, etc. + +```bash +kubectl -n velero get datauploads -l velero.io/backup-name=YOUR_BACKUP_NAME -w +``` + +By default, `INCREMENTAL BYTES` is not displayed in the `kubectl get` output. To see this extended field, the `-o wide` arg is needed: +```bash +kubectl -n velero get datauploads -o wide -l velero.io/backup-name=YOUR_BACKUP_NAME -w +``` + +When the backup completes, you can view information about the backups: + +```bash +velero backup describe YOUR_BACKUP_NAME +``` +```bash +kubectl -n velero get datauploads -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml +``` + +## To restore + +You don't need to set any additional information when creating a data mover restore. The configurations are automatically retrieved from the backup, i.e., whether data movement should be involved and which data mover conducts the data movement. + +To restore from your Velero backup: + +```bash +velero restore create --from-backup BACKUP_NAME OPTIONS... +``` + +When the restore starts, you will see one or more `DataDownload` CRs created. +You may also see some intermediate objects (i.e., pods, PVCs, PVs) created in Velero namespace or the cluster scope, they are to help data movers to move data. And they will be removed after the restore completes. +The phase of a `DataDownload` CR changes several times during the restore process and finally goes to one of the terminal status, `Completed`, `Failed` or `Cancelled`. You can see the phase changes as well as the data download progress by watching the DataDownload CRs: + +```bash +kubectl -n velero get datadownloads -l velero.io/restore-name=YOUR_RESTORE_NAME -w +``` + +When the restore completes, view information about your restores: + +```bash +velero restore describe YOUR_RESTORE_NAME +``` +```bash +kubectl -n velero get datadownloads -l velero.io/restore-name=YOUR_RESTORE_NAME -o yaml +``` + +## Limitations + +- CSI and CSI snapshot support both file system volume mode and block volume mode. At present, block mode is only supported for non-Windows platforms, because the block mode code invokes some system calls that are not present in the Windows platform. +- [Velero built-in data mover] At present, Velero uses a static, common encryption key for all backup repositories it creates. **This means +that anyone who has access to your backup storage can decrypt your backup data**. Make sure that you limit access +to the backup storage appropriately. +- [Velero built-in data mover] Even though the backup data could be incrementally preserved, for a single file data, Velero built-in data mover leverages on deduplication to find the difference to be saved. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual difference is small. + +## Troubleshooting + +Run the following checks: + +Are your Velero server and daemonset pods running? + +```bash +kubectl get pods -n velero +``` + +Does your backup repository exist, and is it ready? + +```bash +velero repo get + +velero repo get REPO_NAME -o yaml +``` + +Are there any errors in your Velero backup/restore? + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME + +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +What is the status of your `DataUpload` and `DataDownload`? + +```bash +kubectl -n velero get datauploads -l velero.io/backup-name=BACKUP_NAME -o yaml + +kubectl -n velero get datadownloads -l velero.io/restore-name=RESTORE_NAME -o yaml +``` + +Is there any useful information in the Velero server or daemonset pod logs? + +```bash +kubectl -n velero logs deploy/velero +kubectl -n velero logs DAEMON_POD_NAME +``` + +**NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument to the container command in the deployment/daemonset pod template spec. + +If you are using a customized data mover, follow the data mover's instruction for additional troubleshooting methods. + + +## How backup and restore work + +CSI snapshot data movement is a combination of CSI snapshot and data movement, which is jointly executed by Velero server, CSI plugin and the data mover. +This section lists some general concept of how CSI snapshot data movement backup and restore work. For the detailed mechanisms and workflows, you can check the [Volume Snapshot Data Movement design][1] and [VGDP Micro Service For Volume Snapshot Data Movement design][18]. + +### Custom resource and controllers + +Velero has three custom resource definitions and associated controllers: + +- `DataUpload` - represents a data upload of a volume snapshot. The CSI plugin creates one `DataUpload` per CSI snapshot. Data movers need to handle these CRs to finish the data upload process. +Velero built-in data mover runs a controller for this resource on each node (in node-agent daemonset). Controllers from different nodes may handle one CR in different phases, but finally the data transfer is done by a data mover pod in one node. + +- `DataDownload` - represents a data download of a volume snapshot. The CSI plugin creates one `DataDownload` per volume to be restored. Data movers need to handle these CRs to finish the data upload process. +Velero built-in data mover runs a controller for this resource on each node (in node-agent daemonset). Controllers from different nodes may handle one CR in different phases, but finally the data transfer is done by a data mover pod in one node. + +- `BackupRepository` - represents/manages the lifecycle of Velero's backup repositories. Velero creates a backup repository per namespace when the first CSI snapshot backup/restore for a namespace is requested. You can see information about your Velero's backup repositories by running `velero repo get`. +This CR is used by Velero built-in data movers, customized data movers may or may not use it. + +For other resources or controllers involved by customized data movers, check the data mover's instructions. + +### Backup + +Velero backs up resources for CSI snapshot data movement backup in the same way as other backup types. When it encounters a PVC, particular logics will be conducted: + +- When it finds a PVC object, Velero calls CSI plugin through a Backup Item Action. +- CSI plugin first takes a CSI snapshot to the PVC by creating the `VolumeSnapshot` and `VolumeSnapshotContent`. +- CSI plugin checks if a data movement is required, if so it creates a `DataUpload` CR and then returns to Velero backup. +- Velero now is able to back up other resources, including other PVC objects. +- Velero backup controller periodically queries the data movement status from CSI plugin, the period is configurable through the Velero server parameter `--item-operation-sync-frequency`, by default it is 10s. On the call, CSI plugin turns to check the phase of the `DataUpload` CRs. +- When all the `DataUpload` CRs come to a terminal state (i.e., `Completed`, `Failed` or `Cancelled`), Velero backup persists all the necessary information and finish the backup. + +- CSI plugin expects a data mover to handle the `DataUpload` CR. If no data mover is configured for the backup, Velero built-in data mover will handle it. +- If the `DataUpload` CR does not reach to the terminal state with in the given time, the `DataUpload` CR will be cancelled. You can set the timeout value per backup through the `--item-operation-timeout` parameter, the default value is `4 hours`. + +- Velero built-in data mover creates a volume from the CSI snapshot and transfer the data to the backup storage according to the backup storage location defined by users. +- After the volume is created from the CSI snapshot, Velero built-in data mover waits for Kubernetes to provision the volume, this may take some time varying from storage providers, but if the provision cannot be finished in a given time, Velero built-in data mover will cancel this `DataUpload` CR. The timeout is configurable through a node-agent's parameter `data-mover-prepare-timeout`, the default value is 30 minutes. +- Velero built-in data mover launches a data mover pod to transfer the data from the provisioned volume to the backup storage. +- When the data transfer completes or any error happens, Velero built-in data mover sets the `DataUpload` CR to the terminal state, either `Completed` or `Failed`. +- Velero built-in data mover also monitors the cancellation request to the `DataUpload` CR, once that happens, it cancels its ongoing activities, cleans up the intermediate resources and set the `DataUpload` CR to `Cancelled`. +- Throughout the data transfer, Velero built-in data mover monitors the status of the data mover pod and deletes it after `DataUpload` CR is set to the terminal state. + +### Restore + +Velero restores resources for CSI snapshot data movement restore in the same way as other restore types. When it encounters a PVC, particular logics will be conducted: + +- When it finds a PVC object, Velero calls CSI plugin through a Restore Item Action. +- CSI plugin checks the backup information, if a data movement was involved, it creates a `DataDownload` CR and then returns to Velero restore. +- Velero is now able to restore other resources, including other PVC objects. +- Velero restore controller periodically queries the data movement status from CSI plugin, the period is configurable through the Velero server parameter `--item-operation-sync-frequency`, by default it is 10s. On the call, CSI plugin turns to check the phase of the `DataDownload` CRs. +- When all `DataDownload` CRs come to a terminal state (i.e., `Completed`, `Failed` or `Cancelled`), Velero restore will finish. + +- CSI plugin expects the same data mover for the backup to handle the `DataDownload` CR. If no data mover was configured for the backup, Velero built-in data mover will handle it. +- If the `DataDownload` CR does not reach to the terminal state with in the given time, the `DataDownload` CR will be cancelled. You can set the timeout value per backup through the same `--item-operation-timeout` parameter. + +- Velero built-in data mover creates a volume with the same specification of the source volume. +- Velero built-in data mover waits for Kubernetes to provision the volume, this may take some time varying from storage providers, but if the provision cannot be finished in a given time, Velero built-in data mover will cancel this `DataDownload` CR. The timeout is configurable through the same node-agent's parameter `data-mover-prepare-timeout`. +- After the volume is provisioned, Velero built-in data mover starts a data mover pod to transfer the data from the backup storage according to the backup storage location defined by users. +- When the data transfer completes or any error happens, Velero built-in data mover sets the `DataDownload` CR to the terminal state, either `Completed` or `Failed`. +- Velero built-in data mover also monitors the cancellation request to the `DataDownload` CR, once that happens, it cancels its ongoing activities, cleans up the intermediate resources and set the `DataDownload` CR to `Cancelled`. +- Throughout the data transfer, Velero built-in data mover monitors the status of the data mover pod and deletes it after `DataDownload` CR is set to the terminal state. + +### Backup Deletion +When a backup is created, a snapshot is saved into the repository for the volume data. The snapshot is a reference to the volume data saved in the repository. +When deleting a backup, Velero calls the repository to delete the repository snapshot. So the repository snapshot disappears immediately after the backup is deleted. Then the volume data backed up in the repository turns to orphan, but it is not deleted by this time. The repository relies on the maintenance functionalitiy to delete the orphan data. +As a result, after you delete a backup, you don't see the backup storage size reduces until some full maintenance jobs completes successfully. And for the same reason, you should check and make sure that the periodical repository maintenance job runs and completes successfully. + +Even after deleting all the backups and their backup data (by repository maintenance), the backup storage is still not empty, some repository metadata are there to keep the instance of the backup repository. +Furthermore, Velero never deletes these repository metadata, if you are sure you'll never usage the backup repository, you can empty the backup storage manually. + +For Velero built-in data mover, Kopia uploader may keep some internal snapshots which is not managed by Velero. In normal cases, the internal snapshots are deleted along with running of backups. +However, if you run a backup which aborts halfway(some internal snapshots are thereby generated) and never run new backups again, some internal snapshots may be left there. In this case, since you stop using the backup repository, you can delete the entire repository metadata from the backup storage manually. + + +### Parallelism + +Velero calls the CSI plugin concurrently for the volume, so `DataUpload`/`DataDownload` CRs are created concurrently by the CSI plugin. For more details about the call between Velero and CSI plugin, check the [Volume Snapshot Data Movement design][1]. +In which manner the `DataUpload`/`DataDownload` CRs are processed is totally decided by the data mover you select for the backup/restore. + +For Velero built-in data mover, it uses Kubernetes' scheduler to mount a snapshot volume/restore volume associated to a `DataUpload`/`DataDownload` CR into a specific node, and then the `DataUpload`/`DataDownload` controller (in node-agent daemonset) in that node will handle the `DataUpload`/`DataDownload`. +By default, a `DataUpload`/`DataDownload` controller in one node handles one request at a time. You can configure more parallelism per node by [node-agent Concurrency Configuration][14]. +That is to say, the snapshot volumes/restore volumes may spread in different nodes, then their associated `DataUpload`/`DataDownload` CRs will be processed in parallel; while for the snapshot volumes/restore volumes in the same node, by default, their associated `DataUpload`/`DataDownload` CRs are processed sequentially and can be processed concurrently according to your [node-agent Concurrency Configuration][14]. + +The prepare process of mounting the snapshot volume/restore volume may generate multiple intermediate objects, to make a control of the intermediate objects, you can configure the [node-agent Prepare Queue Length][20]. + +You can check in which node the `DataUpload`/`DataDownload` CRs are processed and their parallelism by watching the `DataUpload`/`DataDownload` CRs: + +```bash +kubectl -n velero get datauploads -l velero.io/backup-name=YOUR_BACKUP_NAME -w +``` + +```bash +kubectl -n velero get datadownloads -l velero.io/restore-name=YOUR_RESTORE_NAME -w +``` + +For each volume, the parallelism is like below: +- If it is a file system mode volume, files in the volume are processed in parallel. You can use `--parallel-files-upload` backup flag or `--parallel-files-download` restore flag to control how many files are processed in parallel. Otherwise, if they are not set, Velero by default refers to the number of CPU cores in the node (where the backup/restore is running) for the parallelism. That is to say, the parallelism is not affected by the CPU request/limit set to the data mover pods. +- If it is a block mode volume, there is no parallelism, the block data is processed sequentially. + +Notice that Golang 1.25 and later respects the CPU limit set to the pods to decide the physical threads provisioned to the pod processes (see [Container-aware GOMAXPROCS][22] for more details), so for Velero 1.18 (which consumes Golang 1.25) and later, if you set a CPU limit to the data mover pods, you may not get the expected performance (e.g., backup/restore throughput) with the default parallelism. The outcome may or may not be obvious varying on your volume data. If it is required, you could customize `--parallel-files-upload` or `--parallel-files-download` according to the CPU limit set to the data mover pods. + +### Restart and resume +When Velero server is restarted, if the resource backup/restore has completed, so the backup/restore has excceded `InProgress` status and is waiting for the completion of the data movements, Velero will recapture the status of the running data movements and resume the execution. +When node-agent is restarted, Velero tries to recapture the status of the running data movements and resume the execution; if the resume fails, the data movements are canceled. + +### Cancellation + +At present, Velero backup and restore doesn't support end to end cancellation that is launched by users. +However, Velero cancels the `DataUpload`/`DataDownload` in below scenarios automatically: +- When Velero server is restarted and the backup/restore is in `InProgress` status +- When node-agent is restarted and the resume of an existing `DataUpload`/`DataDownload` fails +- When an ongoing backup/restore is deleted +- When a backup/restore does not finish before the item operation timeout (default value is `4 hours`) + +Customized data movers that support cancellation could cancel their ongoing tasks and clean up any intermediate resources. If you are using Velero built-in data mover, the cancellation is supported. + +### Support ReadOnlyRootFilesystem setting +When the Velero server pod's SecurityContext sets the `ReadOnlyRootFileSystem` parameter to true, the Velero server pod's filesystem is running in read-only mode. Then the backup deletion may fail, because the repository needs to write some cache and configuration data into the pod's root filesystem. + +``` +Errors: /error to connect repo with storage: error to connect to repository: unable to write config file: unable to create config directory: mkdir /home/cnb/udmrepo: read-only file system +``` + +The workaround is making those directories as ephemeral k8s volumes, then those directories are not counted as pod's root filesystem. +The `user-name` is the Velero pod's running user name. The default value is `cnb`. + +``` yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: velero + namespace: velero +spec: + template: + spec: + containers: + - name: velero + ...... + volumeMounts: + ...... + - mountPath: /home//udmrepo + name: udmrepo + - mountPath: /home//.cache + name: cache + ...... + volumes: + ...... + - emptyDir: {} + name: udmrepo + - emptyDir: {} + name: cache + ...... +``` + +At present, Velero doesn't allow to set `ReadOnlyRootFileSystem` parameter to data mover pods, so the root filesystem for the data mover pods are always writable. + +### Resource Consumption + +Both the uploader and repository consume remarkable CPU/memory during the backup/restore, especially for massive small files or large backup size cases. + +For Velero built-in data mover, Velero uses [BestEffort as the QoS][13] for data mover pods (so no CPU/memory request/limit is set), so that backups/restores wouldn't fail due to resource throttling in any cases. +If you want to constraint the CPU/memory usage, you need to [Customize Data Mover Pod Resource Limits][11]. The CPU/memory consumption is always related to the scale of data to be backed up/restored, refer to [Performance Guidance][12] for more details, so it is highly recommended that you perform your own testing to find the best resource limits for your data. + +During the restore, the repository may also cache data/metadata so as to reduce the network footprint and speed up the restore. The repository uses its own policy to store and clean up the cache. +For Kopia repository, by default, the cache is stored in the data mover pod's root file system. If your root file system space is limited, the data mover pods may be evicted due to running out of the ephemeral storage, which causes the restore fails. To cope with this problem, Velero allows you: +- configure a limit of the cache size per backup repository, for more details, check [Backup Repository Configuration][17]. +- configure a dedicated volume for cache data, for more details, check [Data Movement Cache Volume][21]. + + +### Node Selection + +The node where a data movement backup/restore runs is decided by the data mover. + +For Velero built-in data mover, it uses Kubernetes' scheduler to mount a snapshot volume/restore volume associated to a `DataUpload`/`DataDownload` CR into a specific node, and then the data movement backup/restore will happen in that node. +For the backup, you can intervene this scheduling process through [Data Movement Backup Node Selection][15], so that you can decide which node(s) should/should not run the data movement backup for various purposes. +For the restore, this is not supported because sometimes the data movement restore must run in the same node where the restored workload pod is scheduled. + +### BackupPVC Configuration + +The `BackupPVC` serves as an intermediate Persistent Volume Claim (PVC) utilized during data movement backup operations, providing efficient access to data. +In complex storage environments, optimizing `BackupPVC` configurations can significantly enhance the performance of backup operations. [This document][16] outlines advanced configuration options for `BackupPVC`, allowing users to fine-tune access modes and storage class settings based on their storage provider's capabilities. + +### RestorePVC Configuration + +The `RestorePVC` serves as an intermediate Persistent Volume Claim (PVC) utilized during data movement restore operations, providing efficient access to data. +Sometimes, `RestorePVC` needs to be configured to increase the performance of restore operations. [This document][19] outlines advanced configuration options for `RestorePVC`, allowing users to fine-tune access modes and storage class settings based on their storage provider's capabilities. + + +[1]: https://github.com/vmware-tanzu/velero/pull/5968 +[2]: csi.md +[3]: file-system-backup.md +[4]: https://kubernetes.io/blog/2020/12/10/kubernetes-1.20-volume-snapshot-moves-to-ga/ +[5]: https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation +[7]: https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv +[8]: api-types/backupstoragelocation.md +[9]: supported-providers.md +[10]: restore-reference.md#changing-pv/pvc-Storage-Classes +[11]: data-movement-pod-resource-configuration.md +[12]: performance-guidance.md +[13]: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ +[14]: node-agent-concurrency.md +[15]: data-movement-node-selection.md +[16]: data-movement-backup-pvc-configuration.md +[17]: backup-repository-configuration.md +[18]: https://github.com/vmware-tanzu/velero/pull/7576 +[19]: data-movement-restore-pvc-configuration.md +[20]: node-agent-prepare-queue-length.md +[21]: data-movement-cache-volume.md +[22]: https://tip.golang.org/doc/go1.25#container-aware-gomaxprocs:~:text=Runtime%C2%B6-,Container%2Daware%20GOMAXPROCS,-%C2%B6 + + diff --git a/site/content/docs/v1.18/csi.md b/site/content/docs/v1.18/csi.md new file mode 100644 index 000000000..53843841d --- /dev/null +++ b/site/content/docs/v1.18/csi.md @@ -0,0 +1,136 @@ +--- +title: "Container Storage Interface Snapshot Support in Velero" +layout: docs +--- + +Integrating Container Storage Interface (CSI) snapshot support into Velero enables Velero to backup and restore CSI-backed volumes using the [Kubernetes CSI Snapshot APIs](https://kubernetes.io/docs/concepts/storage/volume-snapshots/). + +By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. This page gives an overview of how to add support for CSI snapshots to Velero. + +## Notice +From release-1.14, the `github.com/vmware-tanzu/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/vmware-tanzu/velero` repository. +The reasons to merge the CSI plugin are: +* The VolumeSnapshot data mover depends on the CSI plugin, it's reasonabe to integrate them. +* This change reduces the Velero deploying complexity. +* This makes performance tuning easier in the future. + +As a result, no need to install Velero CSI plugin anymore. + +## Prerequisites + + 1. Your cluster is Kubernetes version 1.20 or greater. + 1. Your cluster is running a CSI driver capable of support volume snapshots at the [v1 API level](https://kubernetes.io/blog/2020/12/10/kubernetes-1.20-volume-snapshot-moves-to-ga/). + 1. When restoring CSI VolumeSnapshots across clusters, the name of the CSI driver in the destination cluster is the same as that on the source cluster to ensure cross cluster portability of CSI VolumeSnapshots + +**NOTE:** Not all cloud provider's CSI drivers guarantee snapshot durability, meaning that the VolumeSnapshot and VolumeSnapshotContent objects may be stored in the same object storage system location as the original PersistentVolume and may be vulnerable to data loss. You should refer to your cloud provider's documentation for more information on configuring snapshot durability. Since v0.3.0 the velero team will provide official support for CSI plugin when they are used with AWS and Azure drivers. + +## Installing Velero with CSI support + +To integrate Velero with the CSI volume snapshot APIs, you must enable the `EnableCSI` feature flag. + +```bash +velero install \ +--features=EnableCSI \ +--plugins= \ +... +``` + +To include the status of CSI objects associated with a Velero backup in `velero backup describe` output, run `velero client config set features=EnableCSI`. +See [Enabling Features][1] for more information about managing client-side feature flags. + +## Implementation Choices + +This section documents some of the choices made during implementing the CSI snapshot. + + 1. VolumeSnapshots created by the Velero CSI plugins are retained only for the lifetime of the backup even if the `DeletionPolicy` on the VolumeSnapshotClass is set to `Retain`. To accomplish this, during deletion of the backup the prior to deleting the VolumeSnapshot, VolumeSnapshotContent object is patched to set its `DeletionPolicy` to `Delete`. Deleting the VolumeSnapshot object will result in cascade delete of the VolumeSnapshotContent and the snapshot in the storage provider. + 2. VolumeSnapshotContent objects created during a `velero backup` that are dangling, unbound to a VolumeSnapshot object, will be discovered, using labels, and deleted on backup deletion. + 3. The Velero CSI plugins, to backup CSI backed PVCs, will choose the VolumeSnapshotClass in the cluster based on the following logic: + 1. **Default Behavior Based On Annotation:** + You can specify a default VolumeSnapshotClass for VolumeSnapshots that don't request any particular class to bind to by adding the snapshot.storage.kubernetes.io/is-default-class: "true" annotation. + For example, if you want to create a VolumeSnapshotClass for the CSI driver `disk.csi.cloud.com` for taking snapshots of disks created with `disk.csi.cloud.com` based storage classes, you can create a VolumeSnapshotClass like this: + ```yaml + apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + name: test-snapclass-by-annotation + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + driver: disk.csi.cloud.com + ``` + Note: If multiple CSI drivers exist, a default VolumeSnapshotClass can be specified for each of them. + 2. **Default Behavior Based On Label:** + You can simply create a VolumeSnapshotClass for a particular driver and put a label on it to indicate that it is the default VolumeSnapshotClass for that driver. For example, if you want to create a VolumeSnapshotClass for the CSI driver `disk.csi.cloud.com` for taking snapshots of disks created with `disk.csi.cloud.com` based storage classes, you can create a VolumeSnapshotClass like this: + ```yaml + apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + name: test-snapclass-by-label + labels: + velero.io/csi-volumesnapshot-class: "true" + driver: disk.csi.cloud.com + ``` + Note: For each driver type, there should only be 1 VolumeSnapshotClass with the label `velero.io/csi-volumesnapshot-class: "true"`. + + 2. **Choose VolumeSnapshotClass for a particular Backup Or Schedule:** + If you want to use a particular VolumeSnapshotClass for a particular backup or schedule, you can add a annotation to the backup or schedule to indicate which VolumeSnapshotClass to use. For example, if you want to use the VolumeSnapshotClass `test-snapclass` for a particular backup for snapshotting PVCs of `disk.csi.cloud.com`, you can create a backup like this: + ```yaml + apiVersion: velero.io/v1 + kind: Backup + metadata: + name: test-backup + annotations: + velero.io/csi-volumesnapshot-class_disk.csi.cloud.com: "test-snapclass" + spec: + includedNamespaces: + - default + ``` + Note: Please ensure all your annotations are in lowercase. And follow the following format: `velero.io/csi-volumesnapshot-class_ = ` + + 3. **Choosing VolumeSnapshotClass for a particular PVC:** + If you want to use a particular VolumeSnapshotClass for a particular PVC, you can add a annotation to the PVC to indicate which VolumeSnapshotClass to use. This overrides any annotation added to backup or schedule. For example, if you want to use the VolumeSnapshotClass `test-snapclass` for a particular PVC, you can create a PVC like this: + ```yaml + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: test-pvc + annotations: + velero.io/csi-volumesnapshot-class: "test-snapclass" + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: disk.csi.cloud.com + ``` + 4. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete`. + +## How it Works - Overview + +Velero's CSI support does not rely on the Velero VolumeSnapshotter plugin interface. + +Instead, Velero uses a collection of BackupItemAction plugins that act first against PersistentVolumeClaims. + +When this BackupItemAction sees PersistentVolumeClaims pointing to a PersistentVolume backed by a CSI driver, it will choose the VolumeSnapshotClass with the same driver name that has the `velero.io/csi-volumesnapshot-class` label to create a CSI VolumeSnapshot object with the PersistentVolumeClaim as a source. +This VolumeSnapshot object resides in the same namespace as the PersistentVolumeClaim that was used as a source. + +From there, the CSI external-snapshotter controller will see the VolumeSnapshot and create a VolumeSnapshotContent object, a cluster-scoped resource that will point to the actual, disk-based snapshot in the storage system. +The external-snapshotter plugin will call the CSI driver's snapshot method, and the driver will call the storage system's APIs to generate the snapshot. +Once an ID is generated and the storage system marks the snapshot as usable for restore, the VolumeSnapshotContent object will be updated with a `status.snapshotHandle` and the `status.readyToUse` field will be set. + +Velero will include the generated VolumeSnapshot and VolumeSnapshotContent objects in the backup tarball, as well as +upload all VolumeSnapshots and VolumeSnapshotContents objects in a JSON file to the object storage system. **Note that +only Kubernetes objects are uploaded to the object storage, not the data in snapshots.** + +From v1.16, when Velero synchronizes backups into a new cluster, the VolumeSnapshotClass that is chosen to take +snapshot will be synced into the cluster, so that Velero can manage backup expiration appropriately. + + +The `DeletionPolicy` on the VolumeSnapshotContent will be the same as the `DeletionPolicy` on the VolumeSnapshotClass that was used to create the VolumeSnapshot. Setting a `DeletionPolicy` of `Retain` on the VolumeSnapshotClass will preserve the volume snapshot in the storage system for the lifetime of the Velero backup and will prevent the deletion of the volume snapshot, in the storage system, in the event of a disaster where the namespace with the VolumeSnapshot object may be lost. + +When the Velero backup expires, the VolumeSnapshot objects will be deleted and the VolumeSnapshotContent objects will be updated to have a `DeletionPolicy` of `Delete`, to free space on the storage system. + +**Note:** The AWS, Microsoft Azure, and Google Cloud Platform (GCP) Velero plugins version 1.4 and later are able to snapshot and restore persistent volumes provisioned by a CSI driver via the APIs of the cloud provider, without having to install Velero CSI plugins. See the [AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws), [Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure), and [Google Cloud Platform (GCP)](https://github.com/vmware-tanzu/velero-plugin-for-gcp) Velero plugin repo for more information on supported CSI drivers. +From v1.14, no need to install the CSI plugin, because it is integrated into the Velero code base. + +[1]: customize-installation.md#enable-server-side-features diff --git a/site/content/docs/v1.18/custom-plugins.md b/site/content/docs/v1.18/custom-plugins.md new file mode 100644 index 000000000..33c6c1ba8 --- /dev/null +++ b/site/content/docs/v1.18/custom-plugins.md @@ -0,0 +1,121 @@ +--- +title: "Plugins" +layout: docs +--- + +Velero has a plugin architecture that allows users to add their own custom functionality to Velero backups & restores without having to modify/recompile the core Velero binary. To add custom functionality, users simply create their own binary containing implementations of Velero's plugin kinds (described below), plus a small amount of boilerplate code to expose the plugin implementations to Velero. This binary is added to a container image that serves as an init container for the Velero server pod and copies the binary into a shared emptyDir volume for the Velero server to access. + +Multiple plugins, of any type, can be implemented in this binary. + +A fully-functional [sample plugin repository][1] is provided to serve as a convenient starting point for plugin authors. + +## Plugin Naming + +A plugin is identified by a prefix + name. + +**Note: Please don't use `velero.io` as the prefix for a plugin not supported by the Velero team.** The prefix should help users identify the entity developing the plugin, so please use a prefix that identify yourself. + +Whenever you define a Backup Storage Location or Volume Snapshot Location, this full name will be the value for the `provider` specification. + +For example: `oracle.io/oracle`. + +``` +apiVersion: velero.io/v1 +kind: BackupStorageLocation +spec: + provider: oracle.io/oracle +``` + +``` +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +spec: + provider: oracle.io/oracle +``` + +When naming your plugin, keep in mind that the full name needs to conform to these rules: +- have two parts, prefix + name, separated by '/' +- none of the above parts can be empty +- the prefix is a valid DNS subdomain name +- a plugin with the same prefix + name cannot already exist + +### Some examples: + +``` +- example.io/azure +- 1.2.3.4/5678 +- example-with-dash.io/azure +``` + +You will need to give your plugin(s) the full name when registering them by calling the appropriate `RegisterX` function: + +## Plugin Kinds + +Velero supports the following kinds of plugins: + +- **Object Store** - persists and retrieves backups, backup logs and restore logs +- **Volume Snapshotter** - creates volume snapshots (during backup) and restores volumes from snapshots (during restore) +- **Backup Item Action** - executes arbitrary logic for individual items prior to storing them in a backup file +- **Restore Item Action** - executes arbitrary logic for individual items prior to restoring them into a cluster +- **Delete Item Action** - executes arbitrary logic based on individual items within a backup prior to deleting the backup +- **Item Block Action** - executes arbitrary logic for individual items to determine which items should be backed up together + +Plugin binaries are discovered by recursively reading a directory in no particular order. Hence no guarantee is provided for the +order in which item action plugins are invoked. However, if a single binary implements multiple item action plugins, +they may be invoked in the order in which they are registered but it is best to not depend on this +implementation. This is not guaranteed officially and the implementation can change at any time. + +## Plugin Logging + +Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or +per-backup/restore logs. It also passes a `--log-level` flag to each plugin binary, whose value is the value of the same +flag from the main Velero process. This means that if you turn on debug logging for the Velero server via `--log-level=debug`, +plugins will also emit debug-level logs. See the [sample repository][1] for an example of how to use the logger within your plugin. + +## Plugin Configuration + +Velero uses a ConfigMap-based convention for providing configuration to plugins. If your plugin needs to be configured at runtime, +define a ConfigMap like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: my-plugin-config + + # must be in the namespace where the velero deployment + # is running + namespace: velero + + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (the built-in change storageclass + # restore item action plugin) + velero.io/plugin-config: "" + + # add a label whose key corresponds to the fully-qualified + # plugin name (for example mydomain.io/my-plugin-name), and whose + # value is the plugin type (BackupItemAction, RestoreItemAction, + # ObjectStore, or VolumeSnapshotter) + : + +data: + # add your configuration data here as key-value pairs +``` + +Then, in your plugin's implementation, you can read this ConfigMap to fetch the necessary configuration. + +## Feature Flags + +Velero will pass any known features flags as a comma-separated list of strings to the `--features` argument. + +Once parsed into a `[]string`, the features can then be registered using the `NewFeatureFlagSet` function and queried with `features.Enabled()`. + +## Environment Variables + +Velero adds the `LD_LIBRARY_PATH` into the list of environment variables to provide the convenience for plugins that requires C libraries/extensions in the runtime. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example +[2]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/pkg/plugin/logger.go diff --git a/site/content/docs/v1.18/customize-installation.md b/site/content/docs/v1.18/customize-installation.md new file mode 100644 index 000000000..3aed69a74 --- /dev/null +++ b/site/content/docs/v1.18/customize-installation.md @@ -0,0 +1,535 @@ +--- +title: "Customize Velero Install" +layout: docs +--- + +## Plugins + +During install, Velero requires that at least one plugin is added (with the `--plugins` flag). Please see the documentation under [Plugins](overview-plugins.md) + +## Install in any namespace + +Velero is installed in the `velero` namespace by default. However, you can install Velero in any namespace. See [run in custom namespace][2] for details. + +## Use non-file-based identity mechanisms + +By default, `velero install` expects a credentials file for your `velero` IAM account to be provided via the `--secret-file` flag. + +If you are using an alternate identity mechanism, such as kube2iam/kiam on AWS, Workload Identity on GKE, etc., that does not require a credentials file, you can specify the `--no-secret` flag instead of `--secret-file`. + +## Enable file system backup + +By default, `velero install` does not install Velero's [File System Backup][3]. To enable it, specify the `--use-node-agent` flag. + +If you've already run `velero install` without the `--use-node-agent` flag, you can run the same command again, including the `--use-node-agent` flag, to add the file system backup to your existing install. + +Note that for some use cases (including installation on OpenShift clusters) the fs-backup pods must run in a Privileged security context. This is configured through the node-agent configmap (see below) by setting `privilegedFsBackup` to `true` in the configmap. + +## CSI Snapshot Data Movement + +Velero node-agent is required by [CSI Snapshot Data Movement][12] when Velero built-in data mover is used. By default, `velero install` does not install Velero's node-agent. To enable it, specify the `--use-node-agent` flag. + +For some use cases, Velero node-agent requires to run under privileged mode. For example, when backing up block volumes, it is required to allow the node-agent to access the block device. To enable it set velero install flags `--privileged-node-agent`. + +If you've already run `velero install` without the `--use-node-agent` or `--privileged-node-agent` flag, you can run the same command again, including the `--use-node-agent` or `--privileged-node-agent` flag, to add CSI snapshot data movement to your existing install. + +## Customize the kubelet root path of the node-agent +When installing with the `--use-node-agent` flag, the node-agent will mount the default kubelet paths `/var/lib/kubelet/pods` and `/var/lib/kubelet/plugins` (hostPath). To customize these kubelet mount paths, use the `--kubelet-root-dir` flag. + +## Default Pod Volume backup to file system backup + +By default, `velero install` does not enable the use of File System Backup (FSB) to take backups of all pod volumes. You must apply an [annotation](file-system-backup.md/#using-opt-in-pod-volume-backup) to every pod which contains volumes for Velero to use FSB for the backup. + +If you are planning to only use FSB for volume backups, you can run the `velero install` command with the `--default-volumes-to-fs-backup` flag. This will default all pod volumes backups to use FSB without having to apply annotations to pods. Note that when this flag is set during install, Velero will always try to use FSB to perform the backup, even want an individual backup to use volume snapshots, by setting the `--snapshot-volumes` flag in the `backup create` command. Alternatively, you can set the `--default-volumes-to-fs-backup` on an individual backup to to make sure Velero uses FSB for each volume being backed up. + +## Update an existing installation + +By default, the `velero install` command creates new resources in your cluster. If you're updating an existing Velero installation, you can use the `--apply` flag to apply changes to existing resources instead of attempting to create new ones: + +```bash +velero install --apply +``` + +When the `--apply` flag is specified, Velero uses server-side apply to update existing resources. This is particularly useful when updating Velero to a new version or when modifying your installation configuration. While this can be used as part of an upgrade process, please note that for version upgrades, additional steps may be required depending on the specific changes between versions. Also ensure when using this flag that you are setting any additional flags previously used for your existing configuration so that you don't introduce unexpected changes. + +## Enable features + +New features in Velero will be released as beta features behind feature flags which are not enabled by default. A full listing of Velero feature flags can be found [here][11]. + +### Enable server side features + +Features on the Velero server can be enabled using the `--features` flag to the `velero install` command. This flag takes as value a comma separated list of feature flags to enable. As an example [CSI snapshotting of PVCs][10] can be enabled using `EnableCSI` feature flag in the `velero install` command as shown below: + +```bash +velero install --features=EnableCSI +``` + +Another example is enabling the support of multiple API group versions, as documented at [- -features=EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +Feature flags, passed to `velero install` will be passed to the Velero deployment and also to the `node-agent` daemon set, if `--use-node-agent` flag is used. + +Similarly, features may be disabled by removing the corresponding feature flags from the `--features` flag. + +Enabling and disabling feature flags will require modifying the Velero deployment and also the node-agent daemonset. This may be done from the CLI by uninstalling and re-installing Velero, or by editing the `deploy/velero` and `daemonset/node-agent` resources in-cluster. + +```bash +$ kubectl -n velero edit deploy/velero +$ kubectl -n velero edit daemonset/node-agent +``` + +### Enable client side features + +For some features it may be necessary to use the `--features` flag to the Velero client. This may be done by passing the `--features` on every command run using the Velero CLI or the by setting the features in the velero client config file using the `velero client config set` command as shown below: + +```bash +velero client config set features=feature1,feature2... +``` + +This stores the config in a file at `$HOME/.config/velero/config.json`. + +All client side feature flags may be disabled using the below command + +```bash +velero client config set features= +``` + +### Colored CLI output + +Velero CLI uses colored output for some commands, such as `velero describe`. If +the environment in which Velero is run doesn't support colored output, the +colored output will be automatically disabled. However, you can manually disable +colors with config file: + +```bash +velero client config set colorized=false +``` + +Note that if you specify `--colorized=true` as a CLI option it will override +the config file setting. + + +## Set priority class names for Velero components + +You can set priority class names for different Velero components during installation. This allows you to influence the scheduling and eviction behavior of Velero pods, which can be useful in clusters where resource contention is high. + +### Priority class configuration options: + +1. **Velero server deployment**: Use the `--server-priority-class-name` flag +2. **Node agent daemonset**: Use the `--node-agent-priority-class-name` flag +3. **Data mover pods**: Configure through the node-agent configmap (see below) +4. **Maintenance jobs**: Configure through the repository maintenance job configmap (see below) + +```bash +velero install \ + --server-priority-class-name= \ + --node-agent-priority-class-name= +``` + +### Configuring priority classes for data mover pods and maintenance jobs + +For data mover pods and maintenance jobs, priority classes are configured through ConfigMaps that must be created before installation: + +**Data mover pods** (via node-agent configmap): +```bash +kubectl create configmap node-agent-config -n velero --from-file=config.json=/dev/stdin <}} +|Setting|Velero pod defaults|node-agent pod defaults| +|--- |--- |--- | +|CPU request|500m|N/A| +|Memory requests|128Mi|N/A| +|CPU limit|1000m (1 CPU)|N/A| +|Memory limit|512Mi|N/A| +{{< /table >}} + +For Velero pod, through testing, the Velero maintainers have found these defaults work well when backing up and restoring 1000 or less resources. If you are enabling concurrent backups and your backups tend to be large, you may need to increase these limits. +For node-agent pod, by default it doesn't have CPU/memory request/limit, so that the backups/restores won't break due to resource throttling. The Velero maintainers have also done some [Performance Tests][13] to show the relationship of CPU/memory usage and the scale of data being backed up/restored. + +For repository maintenance job, it's no limit on resources by default. You could configure the job resource limitation based on target data to be backed up, some further settings please refer to [repository maintenance job][14]. + +You don't have to change the defaults all the time, but if you need, it's recommended that you perform your own testing to find the best resource limits for your clusters and resources. + +### Install with custom resource requests and limits + +You can customize these resource requests and limit when you first install using the [velero install][6] CLI command. + +``` +velero install \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + [--use-node-agent] \ + [--default-volumes-to-fs-backup] \ + [--node-agent-pod-cpu-request ] \ + [--node-agent-pod-mem-request ] \ + [--node-agent-pod-cpu-limit ] \ + [--node-agent-pod-mem-limit ] \ + [--maintenance-job-cpu-request ] \ + [--maintenance-job-mem-request ] \ + [--maintenance-job-cpu-limit ] \ + [--maintenance-job-mem-limit ] +``` + +### Update resource requests and limits after install + +After installation you can adjust the resource requests and limits in the Velero Deployment spec or node-agent DaemonSet spec, if you are using the File System Backup. + +**Velero pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the Velero deployment. + +```bash +kubectl patch deployment velero -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "velero", "resources": {"limits":{"cpu": "1", "memory": "512Mi"}, "requests": {"cpu": "1", "memory": "128Mi"}}}]}}}}' +``` + +**node-agent pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the node-agent DaemonSet spec. + +```bash +kubectl patch daemonset node-agent -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "node-agent", "resources": {"limits":{"cpu": "1", "memory": "1024Mi"}, "requests": {"cpu": "1", "memory": "512Mi"}}}]}}}}' +``` + +Additionally, you may want to update the the default File System Backup operation timeout (default 240 minutes) to allow larger backups more time to complete. You can adjust this timeout by adding the `- --fs-backup-timeout` argument to the Velero Deployment spec. + +**NOTE:** Changes made to this timeout value will revert back to the default value if you re-run the Velero install command. + +1. Open the Velero Deployment spec. + + ``` + kubectl edit deploy velero -n velero + ``` + +1. Add `- --fs-backup-timeout` to `spec.template.spec.containers`. + + ```yaml + spec: + template: + spec: + containers: + - args: + - --fs-backup-timeout=240m + ``` + +### Ephemeral-storage Requests and Limits + +Velero does not set ephemeral-storage limits during installation. Limits and requests can be edited after install for clusters that monitor and restrict ephemeral-storage usage. + +Plugins will use ephemeral-storage. There needs to be a sufficient requests and limit set to account for plugins and the additional ephemeral-storage used to maintain credentials and cache space for datamovers. Object storage plugins will fit comfortably into an allocation of 100MB of ephemeral-storage. + +## Configure more than one storage location for backups or volume snapshots + +Velero supports any number of backup storage locations and volume snapshot locations. For more details, see [about locations](locations.md). + +However, `velero install` only supports configuring at most one backup storage location and one volume snapshot location. + +To configure additional locations after running `velero install`, use the `velero backup-location create` and/or `velero snapshot-location create` commands along with provider-specific configuration. Use the `--help` flag on each of these commands for more details. + +### Set default backup storage location or volume snapshot locations + +When performing backups, Velero needs to know where to backup your data. This means that if you configure multiple locations, you must specify the location Velero should use each time you run `velero backup create`, or you can set a default backup storage location or default volume snapshot locations. If you only have one backup storage location or volume snapshot location set for a provider, Velero will automatically use that location as the default. + +#### Set default backup storage location +currently, Velero could set the default backup storage location as below: +- First way: Set a default backup storage location by passing a `--default` flag when running `velero backup-location create`. + + ``` + velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-east-1 \ + --default + ``` +- Second way: Set a default backup storage location by passing a `--default` flag when running `velero backup-location set`. + ```bash + velero backup-location set backups-primary --default + ``` + We also could remove the default backup storage location by this command, below is one example + ```bash + velero backup-location set backups-primary --default=false + ``` +- Third way: Set a default backup storage location by passing `--default-backup-storage-location` flag on the `velero server` command. + ```bash + velero server --default-backup-storage-location backups-primary + ``` +Note: Only could have one default backup storage location, which means it's not allowed to set two default backup storage locations at the same time, the priorities among these three are as follows: +- if velero server side has specified one default backup storage location, suppose it's `A` + - if `A` backup storage location exists, it's not allowed to set a new default backup storage location + - if `A` does not exist + - if using `velero backup-location set` or `velero backup-location create --default` command + - it could be successful if no default backup storage location exists. + - it would fail if already exist one default backup storage location. (So it need to remove other default backup storage location at first) +- if velero server side has not specified one default backup storage location + - if using `velero backup-location set` or `velero backup-location create --default` command + - it could be successful if no default backup storage location exists. + - it would fail if already exist one default backup storage location. (So it need to remove other default backup storage location at first) +#### Set default volume snapshot location +You can set a default volume snapshot location for each of your volume snapshot providers using the `--default-volume-snapshot-locations` flag on the `velero server` command. + +``` +velero server --default-volume-snapshot-locations=":,:" +``` + +## Do not configure a backup storage location during install + +If you need to install Velero without a default backup storage location (without specifying `--bucket` or `--provider`), the `--no-default-backup-location` flag is required for confirmation. + +## Install an additional volume snapshot provider + +Velero supports using different providers for volume snapshots than for object storage -- for example, you can use AWS S3 for object storage, and Portworx for block volume snapshots. + +However, `velero install` only supports configuring a single matching provider for both object storage and volume snapshots. + +To use a different volume snapshot provider: + +1. Install the Velero server components by following the instructions for your **object storage** provider + +1. Add your volume snapshot provider's plugin to Velero (look in [your provider][0]'s documentation for the image name): + + ```bash + velero plugin add + ``` + +1. Add a volume snapshot location for your provider, following [your provider][0]'s documentation for configuration: + + ```bash + velero snapshot-location create \ + --provider \ + [--config ] + ``` + +## Generate YAML only + +By default, `velero install` generates and applies a customized set of Kubernetes configuration (YAML) to your cluster. + +To generate the YAML without applying it to your cluster, use the `--dry-run -o yaml` flags. + +This is useful for applying bespoke customizations, integrating with a GitOps workflow, etc. + +If you are installing Velero in Kubernetes 1.14.x or earlier, you need to use `kubectl apply`'s `--validate=false` option when applying the generated configuration to your cluster. See [issue 2077][7] and [issue 2311][8] for more context. + +## Use a storage provider secured by a self-signed certificate + +If you intend to use Velero with a storage provider that is secured by a self-signed certificate, +you may need to instruct Velero to trust that certificate. See [use Velero with a storage provider secured by a self-signed certificate][9] for details. + +## Additional options + +Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options. + +## Optional Velero CLI configurations + +### Enabling shell autocompletion + +**Velero CLI** provides autocompletion support for `Bash` and `Zsh`, which can save you a lot of typing. + +Below are the procedures to set up autocompletion for `Bash` (including the difference between `Linux` and `macOS`) and `Zsh`. + +#### Bash on Linux + +The **Velero CLI** completion script for `Bash` can be generated with the command `velero completion bash`. Sourcing the completion script in your shell enables velero autocompletion. + +However, the completion script depends on [**bash-completion**](https://github.com/scop/bash-completion), which means that you have to install this software first (you can test if you have bash-completion already installed by running `type _init_completion`). + +##### Install bash-completion + +`bash-completion` is provided by many package managers (see [here](https://github.com/scop/bash-completion#installation)). You can install it with `apt-get install bash-completion` or `yum install bash-completion`, etc. + +The above commands create `/usr/share/bash-completion/bash_completion`, which is the main script of bash-completion. Depending on your package manager, you have to manually source this file in your `~/.bashrc` file. + +To find out, reload your shell and run `type _init_completion`. If the command succeeds, you're already set, otherwise add the following to your `~/.bashrc` file: + +```shell +source /usr/share/bash-completion/bash_completion +``` + +Reload your shell and verify that bash-completion is correctly installed by typing `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on Linux + +You now need to ensure that the **Velero CLI** completion script gets sourced in all your shell sessions. There are two ways in which you can do this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + ``` + +- Add the completion script to the `/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +> `bash-completion` sources all completion scripts in `/etc/bash_completion.d`. + +Both approaches are equivalent. After reloading your shell, velero autocompletion should be working. + +#### Bash on macOS + +The **Velero CLI** completion script for Bash can be generated with `velero completion bash`. Sourcing this script in your shell enables velero completion. + +However, the velero completion script depends on [**bash-completion**](https://github.com/scop/bash-completion) which you thus have to previously install. + + +> There are two versions of bash-completion, v1 and v2. V1 is for Bash 3.2 (which is the default on macOS), and v2 is for Bash 4.1+. The velero completion script **doesn't work** correctly with bash-completion v1 and Bash 3.2. It requires **bash-completion v2** and **Bash 4.1+**. Thus, to be able to correctly use velero completion on macOS, you have to install and use Bash 4.1+ ([*instructions*](https://itnext.io/upgrading-bash-on-macos-7138bd1066ba)). The following instructions assume that you use Bash 4.1+ (that is, any Bash version of 4.1 or newer). + + +##### Install bash-completion + +> As mentioned, these instructions assume you use Bash 4.1+, which means you will install bash-completion v2 (in contrast to Bash 3.2 and bash-completion v1, in which case kubectl completion won't work). + +You can test if you have bash-completion v2 already installed with `type _init_completion`. If not, you can install it with Homebrew: + + ```shell + brew install bash-completion@2 + ``` + +As stated in the output of this command, add the following to your `~/.bashrc` file: + + ```shell + export BASH_COMPLETION_COMPAT_DIR="/usr/local/etc/bash_completion.d" + [[ -r "/usr/local/etc/profile.d/bash_completion.sh" ]] && . "/usr/local/etc/profile.d/bash_completion.sh" + ``` + +Reload your shell and verify that bash-completion v2 is correctly installed with `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on macOS + +You now have to ensure that the velero completion script gets sourced in all your shell sessions. There are multiple ways to achieve this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + + ``` + +- Add the completion script to the `/usr/local/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/usr/local/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +- If you installed velero with Homebrew (as explained [above](#install-with-homebrew-on-macos)), then the velero completion script should already be in `/usr/local/etc/bash_completion.d/velero`. In that case, you don't need to do anything. + +> The Homebrew installation of bash-completion v2 sources all the files in the `BASH_COMPLETION_COMPAT_DIR` directory, that's why the latter two methods work. + +In any case, after reloading your shell, velero completion should be working. + +#### Autocompletion on Zsh + +The velero completion script for Zsh can be generated with the command `velero completion zsh`. Sourcing the completion script in your shell enables velero autocompletion. + +To do so in all your shell sessions, add the following to your `~/.zshrc` file: + + ```shell + source <(velero completion zsh) + ``` + +If you have an alias for kubectl, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.zshrc + echo 'complete -F __start_velero v' >>~/.zshrc + ``` + +After reloading your shell, kubectl autocompletion should be working. + +If you get an error like `complete:13: command not found: compdef`, then add the following to the beginning of your `~/.zshrc` file: + + ```shell + autoload -Uz compinit + compinit + ``` + +## Advanced configuration through external ConfigMaps + +Velero supports to configure its some advanced behaviors by external ConfigMaps. +Velero itself isn't responsible for creating and maintaining these ConfigMaps, instead the users should do that. + +By far, `velero install` supports the following parameters to specify the external ConfigMap names: +* --backup-repository-configmap: [backup repository configuration document][15] +* --node-agent-configmap: [node-agent concurrency configuration document][16], and there are some other documents specify other parts of node-agent-config. +* --repo-maintenance-job-configmap: [repository maintenance configuration document][17] + +From v1.17, Velero adds verification for the ConfigMaps in CLI and server side, which means `velero install` CLI will fail and velero server and node-agent pod will exit if the specified ConfigMaps don't exist or are invalid. + +The change's aim is validating the ConfigMaps and fail early instead of finding the ConfigMaps are not valid during running data mover pod or repository maintenance job. + +However, there means the user cannot just running `velero install` CLI then get a working environment, when the external ConfigMaps are involved. + +The new workflow is: +* Create the needed namespace: `kubectl create ns velero` +* Add PSA labels to the namespace: `kubectl label ns velero pod-security.velero.io/enforce=privileged` +* Create the needed ConfigMaps. +* Run the `velero install` CLI: + ``` bash + velero install \ + --provider aws \ + ...... + --backup-repository-configmap=... \ + --node-agent-configmap=... \ + --repo-maintenance-job-configmap=... + ``` + + +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: namespace.md +[3]: file-system-backup.md +[4]: on-premises.md +[6]: velero-install.md#usage +[7]: https://github.com/vmware-tanzu/velero/issues/2077 +[8]: https://github.com/vmware-tanzu/velero/issues/2311 +[9]: self-signed-certificates.md +[10]: csi.md +[11]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/pkg/apis/velero/v1/constants.go +[12]: csi-snapshot-data-movement.md +[13]: performance-guidance.md +[14]: repository-maintenance.md +[15]: backup-repository-configuration.md +[16]: node-agent-concurrency.md +[17]: repository-maintenance.md diff --git a/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md new file mode 100644 index 000000000..9d48b0a5f --- /dev/null +++ b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md @@ -0,0 +1,87 @@ +--- +title: "BackupPVC Configuration for Data Movement Backup" +layout: docs +--- + +`BackupPVC` is an intermediate PVC to access data from during the data movement backup operation. + +In some scenarios users may need to configure some advanced options of the backupPVC so that the data movement backup +operation could perform better. Specifically: +- For some storage providers, when creating a read-only volume from a snapshot, it is very fast; whereas, if a writable volume + is created from the snapshot, they need to clone the entire disk data, which is time consuming. If the `backupPVC`'s `accessModes` is + set as `ReadOnlyMany`, the volume driver is able to tell the storage to create a read-only volume, which may dramatically shorten the + snapshot expose time. On the other hand, `ReadOnlyMany` is not supported by all volumes. Therefore, users should be allowed to configure + the `accessModes` for the `backupPVC`. +- Some storage providers create one or more replicas when creating a volume, the number of replicas is defined in the storage class. + However, it doesn't make any sense to keep replicas when an intermediate volume used by the backup. Therefore, users should be allowed + to configure another storage class specifically used by the `backupPVC`. +- In SELinux-enabled clusters, such as OpenShift, when using the above-mentioned readOnly access mode setting, SELinux relabeling of the + volume is not possible. Therefore for these clusters, when setting `readOnly` for a storage class, users must also disable relabeling. + Note that this option is not consistent with the Restricted pod security policy, so if Velero pods must run with a restricted policy, + disabling relabeling (and therefore readOnly volume mounting) is not possible. + +Velero introduces a new section in the node agent configuration ConfigMap (the name of this ConfigMap is passed using `--node-agent-configmap` velero server argument) +called `backupPVC`, through which you can specify the following +configurations: + +- `storageClass`: This specifies the storage class to be used for the backupPVC. If this value does not exist or is empty then by +default the source PVC's storage class will be used. + +- `readOnly`: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise +`ReadWriteOnce` value will be used. + +- `spcNoRelabeling`: This is a boolean value. If set to `true`, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From + the SELinux point of view, this will be considered a "Super Privileged Container" which means that selinux enforcement will be disabled and + volume relabeling will not occur. This field is ignored if `readOnly` is `false`. + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --node-agent-configmap=` + +- `annotations`: permits to set annotations on the backupPVC itself. typically useful for some CSI provider which cannot mount + a VolumeSnapshot without a custom annotation. + +A sample of `backupPVC` config as part of the ConfigMap would look like: +```json +{ + "backupPVC": { + "storage-class-1": { + "storageClass": "backupPVC-storage-class", + "readOnly": true + }, + "storage-class-2": { + "storageClass": "backupPVC-storage-class" + }, + "storage-class-3": { + "readOnly": true, + "annotations": { + "some-csi.provider.io/readOnlyClone": true + } + }, + "storage-class-4": { + "readOnly": true, + "spcNoRelabeling": true + } + } +} +``` + +**Note:** +- Users should make sure that the storage class specified in `backupPVC` config should exist in the cluster and can be used by the +`backupPVC`, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout (data movement prepare timeout value is 30m by default). +- If the users are setting `readOnly` value as `true` in the `backupPVC` config then they must also make sure that the storage class that is being used for +`backupPVC` should support creation of `ReadOnlyMany` PVC from a snapshot, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until +timeout (data movement prepare timeout value is 30m by default). +- In an SELinux-enabled cluster, any time users set `readOnly=true` they must also set `spcNoRelabeling=true`. There is no need to set `spcNoRelabeling=true` +if the volume is not readOnly. +- If any of the above problems occur, then the DataUpload CR is `canceled` after timeout, and the backupPod and backupPVC will be deleted, and the backup +will be marked as `PartiallyFailed`. + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage \ No newline at end of file diff --git a/site/content/docs/v1.18/data-movement-cache-volume.md b/site/content/docs/v1.18/data-movement-cache-volume.md new file mode 100644 index 000000000..c41d1609b --- /dev/null +++ b/site/content/docs/v1.18/data-movement-cache-volume.md @@ -0,0 +1,55 @@ +--- +title: "Cache PVC Configuration for Data Movement Restore" +layout: docs +--- + +Velero data movement restore (i.e., for CSI snapshot data movement and fs-backup) may request the backup repository to cache data locally so as to reduce the data request from the remote backup storage. +The cache behavior is decided by the specific backup repository, and Velero allows you to configure a cache limit for the backup repositories who support it (i.e., kopia repository). For more details, see [Backup Repository Configuration][1]. +The size of cache may significantly impact on the performance. Specifically, if the cache size is too small, the restore throughput will be severely reduced and much more data would be downloaded from the backup storage. +By default, the cache data location is in the data mover pods' root disk. In some environments, the pods' root disk size is very limited, so a large cache size would cause the data mover pods evicted because of running out of ephemeral disk. + +To cope with the problems and guarantee the data mover pods always run with a fine tuned local cache, Velero supports dedicated cache PVCs for data movement restore, for CSI snapshot data movement and fs-backup. + +By default, Velero data mover pods run without cache PVCs. To enable cache PVC, you need to fill the cache PVC configurations in the node-agent configMap. + +A sample of cache PVC configuration as part of the ConfigMap would look like: +```json +{ + "cachePVC": { + "residentThresholdInMB": 1024, + "storageClass": "sc-wffc" + } +} +``` + +To create the configMap, save something like the above sample to a file and then run below commands: +```shell +kubectl create cm node-agent-config -n velero --from-file= +``` + +A must-have field in the configuration is `storageClass` which tells Velero which storage class is used to provision the cache PVC. Velero relies on Kubernetes dynamic provision process to provision the PVC, static provision is not supported. + +The cache PVC behavior could be further fine tuned through `residentThresholdInMB`. Its value is compared to the size of the backup, if the size is smaller than this value, no cache PVC would be created when restoring from the backup. This ensures that cache PVCs are not created in vain when the backup size is too small and can be accommodated in the data mover pods' root disk. + +This configuration decides whether and how to provision cache PVCs, but it doesn't decide their size. Instead, the size is decided by the specific backup repository. Specifically, Velero asks a cache limit from the backup repository and uses this limit to calculate the cache PVC size. +The cache limit is decided by the backup repository itself, for Kopia repository, if `cacheLimitMB` is specified in the backup repository configuration, its value will be used; otherwise, a default limit (5 GB) is used. +Then Velero inflates the limit by 20% by considering the non-payload overheads and delay cache cleanup behavior varying on backup repositories. + +Take Kopia repository and the above cache PVC configuration for example: +- When `cacheLimitMB` is not available for the repository, a 6GB cache PVC is created for the backup that is larger than 1GB; otherwise, no cache volume is created +- When `cacheLimitMB` is specified as `10240` for the repository, a 12GB cache PVC is created for the backup that is larger than 1GB; otherwise, no cache volume is created + +To enable both the node-agent configMap and backup repository configMap, specify the flags in velero installation by CLI: +`velero install --node-agent-configmap= --backup-repository-configmap=` + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage + +[1]: backup-repository-configuration.md diff --git a/site/content/docs/v1.18/data-movement-node-selection.md b/site/content/docs/v1.18/data-movement-node-selection.md new file mode 100644 index 000000000..f25b02ff9 --- /dev/null +++ b/site/content/docs/v1.18/data-movement-node-selection.md @@ -0,0 +1,270 @@ +--- +title: "Node Selection for Data Movement" +layout: docs +--- + +Velero node-agent is a DaemonSet hosting the data movement modules to complete the concrete work of backups/restores. +Varying from the data size, data complexity, resource availability, the data movement may take a long time and remarkable resources (CPU, memory, network bandwidth, etc.) during the backup and restore. + +Velero data movement backup and restore support to constrain the nodes where it runs. This is helpful in below scenarios: +- Prevent the data movement from running in specific nodes because users have more critical workloads in the nodes +- Constrain the data movement to run in specific nodes because these nodes have more resources than others +- Constrain the data movement to run in specific nodes because the storage allows volume/snapshot provisions in these nodes only + +Velero introduces a new section in the node-agent ConfigMap, called ```loadAffinity```, through which users can specify the nodes to/not to run data movement, in the affinity and anti-affinity flavors. + +**Important**: Currently, only the first element in the `loadAffinity` array is used. Any additional elements after the first one will be ignored. If you need to specify multiple conditions, combine them within a single `loadAffinity` element using both `matchLabels` and `matchExpressions`. + +### Example of Incorrect Usage (Multiple Array Elements) + +The following configuration will NOT work as intended: + +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "environment": "production" + } + } + }, + { + "nodeSelector": { + "matchLabels": { + "disk-type": "ssd" + } + } + } + ] +} +``` + +In this example, you might expect data movement to run only on nodes that are BOTH in production AND have SSD disks. However, **only the first condition (environment=production) will be applied**. The second array element will be completely ignored. + +### Correct Usage (Combined Conditions) + +To achieve the intended behavior, combine all conditions into a single array element: + +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "environment": "production", + "disk-type": "ssd" + } + } + } + ] +} +``` + +If it is not there, a ConfigMap should be created manually. The ConfigMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one ConfigMap in each namespace which applies to node-agent in that namespace only. The name of the ConfigMap should be specified in the node-agent server parameter ```--node-agent-configmap```. +The node-agent server checks these configurations at startup time. Therefore, users could edit this ConfigMap any time, but in order to make the changes effective, node-agent server needs to be restarted. + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --node-agent-configmap=` + +## Node Selection manner + +### Affinity +Affinity configuration means allowing the data movement to run in the nodes specified. There are two ways to define it: +- It could be defined by `MatchLabels`. The labels defined in `MatchLabels` means a `LabelSelectorOpIn` operation by default, so in the current context, they will be treated as affinity rules. In the above sample, it defines to run data movement in nodes with label `beta.kubernetes.io/instance-type` of value `Standard_B4ms` (Run data movement in `Standard_B4ms` nodes only). +- It could be defined by `MatchExpressions`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpIn` or `LabelSelectorOpExists`. In the above sample, it defines to run data movement in nodes with label `kubernetes.io/hostname` of values `node-1`, `node-2` and `node-3` (Run data movement in `node-1`, `node-2` and `node-3` only). + +### Anti-affinity +Anti-affinity configuration means preventing the data movement from running in the nodes specified. Below is the way to define it: +- It could be defined by `MatchExpressions`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpNotIn` or `LabelSelectorOpDoesNotExist`. In the above sample, it disallows data movement to run in nodes with label `xxx/critial-workload`. + +## How to create the LoadAffinity ConfigMap and apply to the NodeAgent + +To create the ConfigMap, save something like the above sample to a json file and then run below command: +``` +kubectl create cm -n velero --from-file= +``` + +To provide the ConfigMap to node-agent, edit the node-agent DaemonSet and add the ```- --node-agent-configmap``` argument to the spec: +1. Open the node-agent daemonset spec +``` +kubectl edit ds node-agent -n velero +``` +2. Add ```- --node-agent-configmap``` to ```spec.template.spec.containers``` +``` +spec: + template: + spec: + containers: + - args: + - --node-agent-configmap= +``` + +## Examples + +### LoadAffinity +Here is a sample of the ConfigMap with ```loadAffinity```: +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + }, + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "values": [ + "node-1", + "node-2", + "node-3" + ], + "operator": "In" + }, + { + "key": "xxx/critial-workload", + "operator": "DoesNotExist" + } + ] + } + } + ] +} +``` + +This example demonstrates how to use both `matchLabels` and `matchExpressions` in the same single LoadAffinity element. + +### LoadAffinity with StorageClass (Note: Only First Element Used) + +``` json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + } + } + }, + { + "nodeSelector": { + "matchExpressions": [ + { + "key": "kubernetes.io/os", + "values": [ + "linux" + ], + "operator": "In" + } + ] + }, + "storageClass": "kibishii-storage-class" + } + ] +} +``` + +This sample demonstrates how the `loadAffinity` elements with `StorageClass` field and without `StorageClass` field setting work together. If the VGDP mounting volume is created from StorageClass `kibishii-storage-class`, its pod will run Linux nodes. + +The other VGDP instances will run on nodes, which instance type is `Standard_B4ms`. + +### LoadAffinity interacts with BackupPVC + +``` json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + } + }, + "storageClass": "kibishii-storage-class" + }, + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B2ms" + } + }, + "storageClass": "worker-storagepolicy" + } + ], + "backupPVC": { + "kibishii-storage-class": { + "storageClass": "worker-storagepolicy" + } + } +} +``` + +Velero data mover supports to use different StorageClass to create backupPVC by [design](https://github.com/vmware-tanzu/velero/pull/7982). + +In this example, if the backup target PVC's StorageClass is `kibishii-storage-class`, its backupPVC should use StorageClass `worker-storagepolicy`. Because the final StorageClass is `worker-storagepolicy`, the backupPod uses the loadAffinity specified by `loadAffinity`'s elements with `StorageClass` field set to `worker-storagepolicy`. backupPod will be assigned to nodes, which instance type is `Standard_B2ms`. + +### LoadAffinity interacts with RestorePVC + +``` json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + } + }, + "storageClass": "kibishii-storage-class" + } + ], + "restorePVC": { + "ignoreDelayBinding": false + } +} +``` + +#### StorageClass's bind mode is WaitForFirstConsumer + +``` yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kibishii-storage-class +parameters: + svStorageClass: worker-storagepolicy +provisioner: csi.vsphere.vmware.com +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +``` + +If restorePVC should be created from StorageClass `kibishii-storage-class`, and it's volumeBindingMode is `WaitForFirstConsumer`. +Although `loadAffinityPerStorageClass` has a section matches the StorageClass, the `ignoreDelayBinding` is set `false`, the Velero exposer will wait until the target Pod scheduled to a node, and returns the node as SelectedNode for the restorePVC. +As a result, the `loadAffinityPerStorageClass` will not take affect. + +#### StorageClass's bind mode is Immediate + +``` yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kibishii-storage-class +parameters: + svStorageClass: worker-storagepolicy +provisioner: csi.vsphere.vmware.com +reclaimPolicy: Delete +volumeBindingMode: Immediate +``` + +Because the StorageClass volumeBindingMode is `Immediate`, although `ignoreDelayBinding` is set to `false`, restorePVC will not be created according to the target Pod. + +The restorePod will be assigned to nodes, which instance type is `Standard_B4ms`. + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage diff --git a/site/content/docs/v1.18/data-movement-pod-resource-configuration.md b/site/content/docs/v1.18/data-movement-pod-resource-configuration.md new file mode 100644 index 000000000..3440aa931 --- /dev/null +++ b/site/content/docs/v1.18/data-movement-pod-resource-configuration.md @@ -0,0 +1,173 @@ +--- +title: "Data Movement Pod Resource Configuration" +layout: docs +--- + +During [CSI Snapshot Data Movement][1], Velero built-in data mover launches data mover pods to run the data transfer. +During [fs-backup][2], Velero also launches data mover pods to run the data transfer. +The data transfer is a time and resource consuming activity. + +Velero by default uses the [BestEffort QoS][2] for the data mover pods, which guarantees the best performance of the data movement activities. On the other hand, it may take lots of cluster resource, i.e., CPU, memory, and how many resources are taken is decided by the concurrency and the scale of data to be moved. + +If the cluster nodes don't have sufficient resource, Velero also allows you to customize the resources for the data mover pods. +Note: If less resources are assigned to data mover pods, the data movement activities may take longer time; or the data mover pods may be OOM killed if the assigned memory resource doesn't meet the requirements. Consequently, the dataUpload/dataDownload may run longer or fail. + +Refer to [Performance Guidance][3] for a guidance of performance vs. resource usage, and it is highly recommended that you perform your own testing to find the best resource limits for your data. + +Velero introduces a new section in the node-agent configMap, called ```podResources```, through which you can set customized resources configurations for data mover pods. +If it is not there, a configMap should be created manually. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. The name of the configMap should be specified in the node-agent server parameter ```--node-agent-configmap```. +Node-agent server checks these configurations at startup time. Therefore, you could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. + +### Pod Resources +Here is a sample of the configMap with ```podResources```: +```json +{ + "podResources": { + "cpuRequest": "1000m", + "cpuLimit": "1000m", + "memoryRequest": "512Mi", + "memoryLimit": "1Gi" + } +} +``` + +The string values in ```podResources``` must match Kubernetes Quantity expressions; for each resource, the "request" value must not be larger than the "limit" value. Otherwise, if any one of the values fail, the entire ```podResources``` configuration will be ignored (so the default policy will be used). + +To create the configMap, save something like the above sample to a json file and then run below command: +``` +kubectl create cm node-agent-config -n velero --from-file= +``` + +To provide the configMap to node-agent, edit the node-agent daemonset and add the ```- --node-agent-configmap``` argument to the spec: +1. Open the node-agent daemonset spec +``` +kubectl edit ds node-agent -n velero +``` +2. Add ```- --node-agent-configmap``` to ```spec.template.spec.containers``` +``` +spec: + template: + spec: + containers: + - args: + - --node-agent-configmap= +``` + +### Priority Class + +Data mover pods will use the priorityClassName configured in the node-agent configmap. The priorityClassName for data mover pods is configured through the node-agent configmap (specified via the `--node-agent-configmap` flag), while the node-agent daemonset itself uses the priority class set by the `--node-agent-priority-class-name` flag during Velero installation. + +#### When to Use Priority Classes + +**Higher Priority Classes** (e.g., `system-cluster-critical`, `system-node-critical`, or custom high-priority): +- When you have dedicated nodes for backup operations +- When backup/restore operations are time-critical +- When you want to ensure data mover pods are scheduled even during high cluster utilization +- For disaster recovery scenarios where restore speed is critical + +**Lower Priority Classes** (e.g., `low-priority` or negative values): +- When you want to protect production workload performance +- When backup operations can be delayed during peak hours +- When cluster resources are limited and production workloads take precedence +- For non-critical backup operations that can tolerate delays + +#### Consequences of Priority Class Settings + +**High Priority**: +- ✅ Data mover pods are more likely to be scheduled quickly +- ✅ Less likely to be preempted by other workloads +- ❌ May cause resource pressure on production workloads +- ❌ Could lead to production pod evictions in extreme cases + +**Low Priority**: +- ✅ Production workloads are protected from resource competition +- ✅ Cluster stability is maintained during backup operations +- ❌ Backup/restore operations may take longer to start +- ❌ Data mover pods may be preempted, causing backup failures +- ❌ In resource-constrained clusters, backups might not run at all + +#### Example Configuration + +To configure priority class for data mover pods, include it in your node-agent configmap: + +```json +{ + "priorityClassName": "backup-priority" +} +``` + +First, create the priority class in your cluster: + +```yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: backup-priority +value: 1000 +globalDefault: false +description: "Priority class for Velero data mover pods" +``` + +Then create or update the node-agent configmap: + +```bash +kubectl create cm node-agent-config -n velero --from-file=node-agent-config.json +``` + +**Note**: If the specified priority class doesn't exist in the cluster when data mover pods are created, the pods will fail to schedule. Velero validates the priority class at startup and logs a warning if it doesn't exist, but the pods will still attempt to use it. + +### Pod Labels +Add customized labels for data mover pods to support third-party integrations and environment-specific requirements. + +If `podLabels` is configured, it supersedes Velero's [in-tree third-party labels](https://github.com/vmware-tanzu/velero/blob/94f64639cee09c5caaa65b65ab5f42175f41c101/pkg/util/third_party.go#L19-L21). +If `podLabels` is not configured, Velero uses the in-tree third-party labels for compatibility with common cloud providers and networking solutions. + +The configurations work for DataUpload, DataDownload, PodVolumeBackup, and PodVolumeRestore pods. + +#### Configuration Example +```json +{ + "podLabels": { + "spectrocloud.com/connection": "proxy", + "gnp/k8s-api-access": "", + "gnp/monitoring-client": "", + "np/s3-backup-backend": "", + "cp/inject-truststore": "extended" + } +} +``` + +### Pod Annotations +Add customized annotations for data mover pods to support third-party integrations and pod-level configuration. + +If `podAnnotations` is configured, it supersedes Velero's [in-tree third-party annotations](https://github.com/vmware-tanzu/velero/blob/94f64639cee09c5caaa65b65ab5f42175f41c101/pkg/util/third_party.go#L23-L25). +If `podAnnotations` is not configured, Velero uses the in-tree third-party annotations for compatibility with common cloud providers and networking solutions. + +The configurations work for DataUpload, DataDownload, PodVolumeBackup, and PodVolumeRestore pods. + +#### Configuration Example +```json +{ + "podAnnotations": { + "iam.amazonaws.com/role": "velero-backup-role", + "vault.hashicorp.com/agent-inject": "true", + "prometheus.io/scrape": "true", + "custom.company.com/environment": "production" + } +} +``` + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage + +[1]: csi-snapshot-data-movement.md +[2]: file-system-backup.md +[3]: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ +[4]: performance-guidance.md diff --git a/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md new file mode 100644 index 000000000..1cb8fa14d --- /dev/null +++ b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md @@ -0,0 +1,40 @@ +--- +title: "RestorePVC Configuration for Data Movement Restore" +layout: docs +--- + +`RestorePVC` is an intermediate PVC to write data during the data movement restore operation. + +In some scenarios users may need to configure some advanced options of the `restorePVC` so that the data movement restore operation could perform better. Specifically: +- For a volume with `WaitForFirstConsumer` mode, theoretically, the data mover pod should not be created until the restored is scheduled to a node; and the data movement should happen in that node only (because the pod may not run in every node because of topology constraints). This significantly degrades the parallelism of data movement restores; and this also prevents Velero from restoring a volume without a pod mounted. On the other hand, users must know their topology constrains if they have, or they must know in which nodes their restored workload pods can be scheduled. Therefore, in the backup/restore context, it is fine not to strictly follow the rule of `WaitForFirstConsumer` mode, instead, users should be allowed to configure to ignore the rule if they are aware that there is no topology constraints in their environments or they know how to select the nodes for restore pods to run appropriately. + +Velero introduces a new section in the node agent configuration ConfigMap (the name of this ConfigMap is passed using `--node-agent-configmap` velero server argument) called `restorePVC`, through which you can specify the following configurations: + +- `ignoreDelayBinding`: If this flag is set, the data movement restore will ignore the delay binding requirements from `WaitForFirstConsumer` mode, create the restore pod and provision the volume associated to an arbitrary node. When multiple volume restores happen in parallel, the restore pods will be spread evenly to all the nodes. + + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --node-agent-configmap=` + +A sample of `restorePVC` config as part of the ConfigMap would look like: +```json +{ + "restorePVC": { + "ignoreDelayBinding": true + } +} +``` + +**Note:** +- If `ignoreDelayBinding` is set, the restored volume is provisioned in the storage areas associated to an arbitrary node, if the restored pod cannot be scheduled to that node, e.g., because of topology constraints, the data mover restore still completes, but the workload is not usable since the restored pod cannot mount the restored volume +- At present, node selection is not supported for data mover restore, so the restored volume may be attached to any node in the cluster; once node selection is supported and enabled, the restored volume will be attached to one of the selected nodes only. In this way, node selection and `ignoreDelayBinding` can work together even though the environment is with topology constraints + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage diff --git a/site/content/docs/v1.18/debugging-install.md b/site/content/docs/v1.18/debugging-install.md new file mode 100644 index 000000000..e6a75e525 --- /dev/null +++ b/site/content/docs/v1.18/debugging-install.md @@ -0,0 +1,74 @@ +--- +title: "Debugging Installation Issues" +layout: docs +--- + +## General + +### `invalid configuration: no configuration has been provided` +This typically means that no `kubeconfig` file can be found for the Velero client to use. Velero looks for a kubeconfig in the +following locations: +* the path specified by the `--kubeconfig` flag, if any +* the path specified by the `$KUBECONFIG` environment variable, if any +* `~/.kube/config` + +### Backups or restores stuck in `New` phase +This means that the Velero controllers are not processing the backups/restores, which usually happens because the Velero server is not running. Check the pod description and logs for errors: +``` +kubectl -n velero describe pods +kubectl -n velero logs deployment/velero +``` + + +## AWS + +### `NoCredentialProviders: no valid providers in chain` + +#### Using credentials +This means that the secret containing the AWS IAM user credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `credentials-velero` file is formatted properly and has the correct values: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +#### Using kube2iam +This means that Velero can't read the content of the S3 bucket. Ensure the following: + +* A Trust Policy document exists that allows the role used by kube2iam to assume Velero's role, as stated in the AWS config documentation. +* The new Velero role has all the permissions listed in the documentation regarding S3. + + +## Azure + +### `Failed to refresh the Token` or `adal: Refresh request failed` +This means that the secrets containing the Azure service principal credentials for Velero has not been created/mounted +properly into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has all of the expected keys and each one has the correct value (see [setup instructions][0]) +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + + +## GCE/GKE + +### `open credentials/cloud: no such file or directory` +This means that the secret containing the GCE service account credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +[0]: azure-config.md#create-service-principal diff --git a/site/content/docs/v1.18/debugging-restores.md b/site/content/docs/v1.18/debugging-restores.md new file mode 100644 index 000000000..3320349cd --- /dev/null +++ b/site/content/docs/v1.18/debugging-restores.md @@ -0,0 +1,105 @@ +--- +title: "Debugging Restores" +layout: docs +--- + +## Example + +When Velero finishes a Restore, its status changes to "Completed" regardless of whether or not there are issues during the process. The number of warnings and errors are indicated in the output columns from `velero restore get`: + +``` +NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR +backup-test-20170726180512 backup-test Completed 155 76 2017-07-26 11:41:14 -0400 EDT +backup-test-20170726180513 backup-test Completed 121 14 2017-07-26 11:48:24 -0400 EDT +backup-test-2-20170726180514 backup-test-2 Completed 0 0 2017-07-26 13:31:21 -0400 EDT +backup-test-2-20170726180515 backup-test-2 Completed 0 1 2017-07-26 13:32:59 -0400 EDT +``` + +To delve into the warnings and errors into more detail, you can use `velero restore describe`: + +```bash +velero restore describe backup-test-20170726180512 +``` + +The output looks like this: + +``` +Name: backup-test-20170726180512 +Namespace: velero +Labels: +Annotations: + +Backup: backup-test + +Namespaces: + Included: * + Excluded: + +Resources: + Included: serviceaccounts + Excluded: nodes, events, events.events.k8s.io + Cluster-scoped: auto + +Namespace mappings: + +Label selector: + +Restore PVs: auto + +Preserve Service NodePorts: auto + +Phase: Completed + +Validation errors: + +Warnings: + Velero: + Cluster: + Namespaces: + velero: serviceaccounts "velero" already exists + serviceaccounts "default" already exists + kube-public: serviceaccounts "default" already exists + kube-system: serviceaccounts "attachdetach-controller" already exists + serviceaccounts "certificate-controller" already exists + serviceaccounts "cronjob-controller" already exists + serviceaccounts "daemon-set-controller" already exists + serviceaccounts "default" already exists + serviceaccounts "deployment-controller" already exists + serviceaccounts "disruption-controller" already exists + serviceaccounts "endpoint-controller" already exists + serviceaccounts "generic-garbage-collector" already exists + serviceaccounts "horizontal-pod-autoscaler" already exists + serviceaccounts "job-controller" already exists + serviceaccounts "kube-dns" already exists + serviceaccounts "namespace-controller" already exists + serviceaccounts "node-controller" already exists + serviceaccounts "persistent-volume-binder" already exists + serviceaccounts "pod-garbage-collector" already exists + serviceaccounts "replicaset-controller" already exists + serviceaccounts "replication-controller" already exists + serviceaccounts "resourcequota-controller" already exists + serviceaccounts "service-account-controller" already exists + serviceaccounts "service-controller" already exists + serviceaccounts "statefulset-controller" already exists + serviceaccounts "ttl-controller" already exists + default: serviceaccounts "default" already exists + +Errors: + Velero: + Cluster: + Namespaces: +``` + +## Structure + +Errors appear for incomplete or partial restores. Warnings appear for non-blocking issues, for example, the +restore looks "normal" and all resources referenced in the backup exist in some form, although some +of them may have been pre-existing. + +Both errors and warnings are structured in the same way: + +* `Velero`: A list of system-related issues encountered by the Velero server. For example, Velero couldn't read a directory. + +* `Cluster`: A list of issues related to the restore of cluster-scoped resources. + +* `Namespaces`: A map of namespaces to the list of issues related to the restore of their respective resources. diff --git a/site/content/docs/v1.18/development.md b/site/content/docs/v1.18/development.md new file mode 100644 index 000000000..6d0fa5227 --- /dev/null +++ b/site/content/docs/v1.18/development.md @@ -0,0 +1,49 @@ +--- +title: "Development " +layout: docs +--- + +## Update generated files + +Run `make update` to regenerate files if you make the following changes: + +* Add/edit/remove command line flags and/or their help text +* Add/edit/remove commands or subcommands +* Add new API types +* Add/edit/remove plugin protobuf message or service definitions + +The following files are automatically generated from the source code: + +* CRDs +* Documentation +* Protobuf/gRPC types + +You can run `make verify` to ensure that all generated files (CRDs, docs) are up to date. + +## Linting + +You can run `make lint` which executes golangci-lint inside the build image, or `make local-lint` which executes outside of the build image. +Both `make lint` and `make local-lint` will only run the linter against changes. + +Use `lint-all` to run the linter against the entire code base. + +The default linters are defined in the `Makefile` via the `LINTERS` variable. + +You can also override the default list of linters by running the command + +`$ make lint LINTERS=gosec` + +## Test + +To run unit tests, use `make test`. + +## Using the main branch + +If you are developing or using the main branch, note that you may need to update the Velero CRDs to get new changes as other development work is completed. + +```bash +velero install --crds-only --dry-run -o yaml | kubectl apply -f - +``` + +**NOTE:** You could change the default CRD API version (v1beta1 _or_ v1) if Velero CLI can't discover the Kubernetes preferred CRD API version. The Kubernetes version < 1.16 preferred CRD API version is v1beta1; the Kubernetes version >= 1.16 preferred CRD API version is v1. + diff --git a/site/content/docs/v1.18/disaster-case.md b/site/content/docs/v1.18/disaster-case.md new file mode 100644 index 000000000..6b73e6716 --- /dev/null +++ b/site/content/docs/v1.18/disaster-case.md @@ -0,0 +1,44 @@ +--- +title: "Disaster recovery" +layout: docs +--- + +*Using Schedules and Read-Only Backup Storage Locations* + +If you periodically back up your cluster's resources, you are able to return to a previous state in case of some unexpected mishap, such as a service outage. Doing so with Velero looks like the following: + +1. After you first run the Velero server on your cluster, set up a daily backup (replacing `` in the command as desired): + + ``` + velero schedule create --schedule "0 7 * * *" + ``` + + This creates a Backup object with the name `-`. The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works][1] for more information about backup expiry. + +1. A disaster happens and you need to recreate your resources. + +1. Update your backup storage location to read-only mode (this prevents backup objects from being created or deleted in the backup storage location during the restore process): + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + ``` + +1. Create a restore with your most recent Velero Backup: + + ``` + velero restore create --from-backup - + ``` + +1. When ready, revert your backup storage location to read-write mode: + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' + ``` + +[1]: how-velero-works.md#set-a-backup-to-expire diff --git a/site/content/docs/v1.18/enable-api-group-versions-feature.md b/site/content/docs/v1.18/enable-api-group-versions-feature.md new file mode 100644 index 000000000..20ae9bf66 --- /dev/null +++ b/site/content/docs/v1.18/enable-api-group-versions-feature.md @@ -0,0 +1,115 @@ +--- +title: "Enable API Group Versions Feature" +layout: docs +--- + +## Background + +Velero serves to both restore and migrate Kubernetes applications. Typically, backup and restore does not involve upgrading Kubernetes API group versions. However, when migrating from a source cluster to a destination cluster, it is not unusual to see the API group versions differing between clusters. + +**NOTE:** Kubernetes applications are made up of various resources. Common resources are pods, jobs, and deployments. Custom resources are created via custom resource definitions (CRDs). Every resource, whether custom or not, is part of a group, and each group has a version called the API group version. + +Kubernetes by default allows changing API group versions between clusters as long as the upgrade is a single version, for example, v1 -> v2beta1. Jumping multiple versions, for example, v1 -> v3, is not supported out of the box. This is where the Velero Enable API Group Version feature can help you during an upgrade. + +Currently, the Enable API Group Version feature is in beta and can be enabled by installing Velero with a [feature flag](customize-installation.md/#enable-server-side-features), `--features=EnableAPIGroupVersions`. + +For the most up-to-date information on Kubernetes API version compatibility, you should always review the [Kubernetes release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG) for the source and destination cluster version to before starting an upgrade, migration, or restore. If there is a difference between Kubernetes API versions, use the Enable API Group Version feature to help mitigate compatibility issues. + +## How the Enable API Group Versions Feature Works + +When the Enable API Group Versions feature is enabled on the source cluster, Velero will not only back up Kubernetes preferred API group versions, but it will also back up all supported versions on the cluster. As an example, consider the resource `horizontalpodautoscalers` which falls under the `autoscaling` group. Without the feature flag enabled, only the preferred API group version for autoscaling, `v2` will be backed up. With the feature enabled, the remaining supported versions, `v1` will also be backed up. Once the versions are stored in the backup tarball file, they will be available to be restored on the destination cluster. + +When the Enable API Group Versions feature is enabled on the destination cluster, Velero restore will choose the version to restore based on an API group version priority order. + +The version priorities are listed from highest to lowest priority below: + +- Priority 1: destination cluster preferred version +- Priority 2: source cluster preferred version +- Priority 3: non-preferred common supported version with the highest [Kubernetes version priority](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority) + +The highest priority (Priority 1) will be the destination cluster's preferred API group version. If the destination preferred version is found in the backup tarball, it will be the API group version chosen for restoration for that resource. However, if the destination preferred version is not found in the backup tarball, the next version in the list will be selected: the source cluster preferred version (Priority 2). + +If the source cluster preferred version is found to be supported by the destination cluster, it will be chosen as the API group version to restore. However, if the source preferred version is not supported by the destination cluster, then the next version in the list will be considered: a non-preferred common supported version (Priority 3). + +In the case that there are more than one non-preferred common supported version, which version will be chosen? The answer requires understanding the [Kubernetes version priority order](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority). Kubernetes prioritizes group versions by making the latest, most stable version the highest priority. The highest priority version is the Kubernetes preferred version. Here is a sorted version list example from the Kubernetes.io documentation: + +- v10 +- v2 +- v1 +- v11beta2 +- v10beta3 +- v3beta1 +- v12alpha1 +- v11alpha2 +- foo1 +- foo10 + +Of the non-preferred common versions, the version that has the highest Kubernetes version priority will be chosen. See the example for Priority 3 below. + +To better understand which API group version will be chosen, the following provides some concrete examples. The examples use the term "target cluster" which is synonymous to "destination cluster". + +![Priority 1 Case A example](/docs/main/img/gv_priority1-caseA.png) + +![Priority 1 Case B example](/docs/main/img/gv_priority1-caseB.png) + +![Priority 2 Case C example](/docs/main/img/gv_priority2-caseC.png) + +![Priority 3 Case D example](/docs/main/img/gv_priority3-caseD.png) + +## Procedure for Using the Enable API Group Versions Feature + +1. [Install Velero](basic-install.md) on source cluster with the [feature flag enabled](customize-installation.md/#enable-server-side-features). The flag is `--features=EnableAPIGroupVersions`. For the enable API group versions feature to work, the feature flag needs to be used for Velero installations on both the source and destination clusters. +2. Back up and restore following the [migration case instructions](migration-case.md). Note that "Cluster 1" in the instructions refers to the source cluster, and "Cluster 2" refers to the destination cluster. + +## Advanced Procedure for Customizing the Version Prioritization + +Optionally, users can create a config map to override the default API group prioritization for some or all of the resources being migrated. For each resource that is specified by the user, Velero will search for the version in both the backup tarball and the destination cluster. If there is a match, the user-specified API group version will be restored. If the backup tarball and the destination cluster does not have or support any of the user-specified versions, then the default version prioritization will be used. + +Here are the steps for creating a config map that allows users to override the default version prioritization. These steps must happen on the destination cluster before a Velero restore is initiated. + +1. Create a file called `restoreResourcesVersionPriority`. The file name will become a key in the `data` field of the config map. + - In the file, write a line for each resource group you'd like to override. Make sure each line follows the format `.=,` + - Note that the resource group and versions are separated by a single equal (=) sign. Each version is listed in order of user's priority separated by commas. + - Here is an example of the contents of a config map file: + + ```cm + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + ``` + +2. Apply config map with + + ```bash + kubectl create configmap enableapigroupversions --from-file=/restoreResourcesVersionPriority -n velero + ``` + +3. See the config map with + + ```bash + kubectl describe configmap enableapigroupversions -n velero + ``` + + The config map should look something like + + ```bash + Name: enableapigroupversions + Namespace: velero + Labels: + Annotations: + + Data + ==== + restoreResourcesVersionPriority: + ---- + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + Events: + ``` + +## Troubleshooting + +1. Refer to the [troubleshooting section](troubleshooting.md) of the docs as the techniques generally apply here as well. +2. The [debug logs](troubleshooting.md/#getting-velero-debug-logs) will contain information on which version was chosen to restore. +3. If no API group version could be found that both exists in the backup tarball file and is supported by the destination cluster, then the following error will be recorded (no need to activate debug level logging): `"error restoring rockbands.music.example.io/rockstars/beatles: the server could not find the requested resource"`. diff --git a/site/content/docs/v1.18/examples.md b/site/content/docs/v1.18/examples.md new file mode 100644 index 000000000..d774b387d --- /dev/null +++ b/site/content/docs/v1.18/examples.md @@ -0,0 +1,70 @@ +--- +title: "Examples" +layout: docs +--- + +After you set up the Velero server, you can clone the examples used in the following sections by running the following: +``` +git clone https://github.com/vmware-tanzu/velero.git +cd velero +``` + +## Basic example (without PersistentVolumes) + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Create a backup: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Wait for the namespace to be deleted. + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +## Snapshot example (with PersistentVolumes) + +> NOTE: For Azure, you must run Kubernetes version 1.7.2 or later to support PV snapshotting of managed disks. + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/with-pv.yaml + ``` + +1. Create a backup with PV snapshotting. `--csi-snapshot-timeout` is used to setup time to wait before CSI snapshot creation timeout. The default value is 10 minutes: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example --csi-snapshot-timeout=20m + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Because the default [reclaim policy][1] for dynamically-provisioned PVs is "Delete", these commands should trigger your cloud provider to delete the disk that backs the PV. Deletion is asynchronous, so this may take some time. **Before continuing to the next step, check your cloud provider to confirm that the disk no longer exists.** + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +[1]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming diff --git a/site/content/docs/v1.18/file-system-backup.md b/site/content/docs/v1.18/file-system-backup.md new file mode 100644 index 000000000..b47225799 --- /dev/null +++ b/site/content/docs/v1.18/file-system-backup.md @@ -0,0 +1,767 @@ +--- +title: "File System Backup" +layout: docs +--- + +Velero supports backing up and restoring Kubernetes volumes attached to pods from the file system of the volumes, called +File System Backup (FSB shortly) or Pod Volume Backup. The data movement is fulfilled by using modules from free open-source +backup tools [restic][1] and [kopia][2]. This support is considered beta quality. Please see the list of [limitations](#limitations) +to understand if it fits your use case. + +Velero allows you to take snapshots of persistent volumes as part of your backups if you’re using one of +the supported cloud providers’ block storage offerings (Amazon EBS Volumes, Azure Managed Disks, Google Persistent Disks). +It also provides a plugin model that enables anyone to implement additional object and block storage backends, outside the +main Velero repository. + +If your storage supports CSI (Container Storage Interface) snapshots, Velero also allows you to take snapshots through CSI and then optionally move the snapshot data to a different storage location. + +Velero's File System Backup is an addition to the aforementioned snapshot approaches. Its pros and cons are listed below: +Pros: +- It is capable of backing up and restoring almost any type of Kubernetes volume. Therefore, if you need a volume snapshot +plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, local, or any other volume type that doesn't +have a native snapshot concept, FSB might be for you. +- It is not tied to a specific storage platform, so you could save the backup data to a different storage platform from +the one backing Kubernetes volumes, for example, a durable storage. + +Cons: +- It backs up data from the live file system, in which way the data is not captured at the same point in time, so is less consistent than the snapshot approaches. +- It access the file system from the mounted hostpath directory, so Velero Node Agent pods need to run as root user and even under privileged mode in some environments. + +## Relationship with Volume Snapshots + +It's important to understand that File System Backup (FSB) and volume snapshots (native/CSI) are **mutually exclusive** for the same volume: + +- When FSB is performed on a volume, Velero will **skip** taking a snapshot of that volume +- When FSB is opted out for a volume, Velero will **attempt** to take a snapshot (if configured) +- This prevents duplicate backups of the same data + +This behavior is automatic and ensures optimal backup performance and storage usage. + +**NOTE:** hostPath volumes are not supported, but the [local volume type][5] is supported. +**NOTE:** restic is under the deprecation process by following [Velero Deprecation Policy][17], for more details, see the Restic Deprecation section. + +## Setup File System Backup + +### Prerequisites + +- Understand how Velero performs [file system backup](#how-backup-and-restore-work). +- [Download][4] the latest Velero release. +- Kubernetes v1.16.0 or later are required. Velero's File System Backup requires the Kubernetes [MountPropagation feature][6]. + +### Install Velero Node Agent + +Velero Node Agent is a Kubernetes daemonset that hosts controllers for File System Backup. +To install Node Agent, use the `--use-node-agent` flag in the `velero install` command. See the [install overview][3] for more +details on other flags for the install command. + +``` +velero install --use-node-agent +``` + +When using FSB on a storage that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an +unused `VolumeSnapshotLocation` from being created on installation. + +At present, Velero FSB supports object storage as the backup storage only. Velero gets the parameters from the +[BackupStorageLocation `config`](api-types/backupstoragelocation.md) to compose the URL to the backup storage. Velero's known object +storage providers are include here [supported providers](supported-providers.md), for which, Velero pre-defines the endpoints; if you +want to use a different backup storage, make sure it is S3 compatible and you provide the correct bucket name and endpoint in +BackupStorageLocation. Velero handles the creation of the backup repo prefix in the backup storage, so make sure it is specified in BackupStorageLocation correctly. + +Velero creates one backup repo per namespace. For example, if backing up 2 namespaces, namespace1 and namespace2, using kopia +repository on AWS S3, the full backup repo path for namespace1 would be `https://s3-us-west-2.amazonaws.com/bucket/kopia/ns1` and +for namespace2 would be `https://s3-us-west-2.amazonaws.com/bucket/kopia/ns2`. + +There may be additional installation steps depending on the cloud provider plugin you are using. You should refer to the +[plugin specific documentation](supported-providers.md) for the most up to date information. + +**Note:** Currently, Velero creates a secret named `velero-repo-credentials` in the velero install namespace, containing a default backup repository password. +You can update the secret with your own password encoded as base64 prior to the first backup (i.e., FS Backup, data mover) targeting to the backup repository. The value of the key to update is +``` +data: + repository-password: +``` +Backup repository is created during the first execution of backup targeting to it after installing Velero with node agent. If you update the secret password after the first +backup which created the backup repository, then Velero will not be able to connect with the older backups. + +### Configure Node Agent DaemonSet spec + +After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the node-agent DaemonSet spec. +The steps in this section are only needed if you are installing on RancherOS, Nutanix, OpenShift, VMware Tanzu Kubernetes Grid +Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. + + +**RancherOS** + + +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/opt/rke/var/lib/kubelet/pods`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /opt/rke/var/lib/kubelet/pods +``` + +**Nutanix** + + +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/var/nutanix/var/lib/kubelet`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /var/nutanix/var/lib/kubelet +``` + +**OpenShift** + + +To mount the correct hostpath to pods volumes, run the node-agent pod in `privileged` mode. + +1. Add the `velero` ServiceAccount to the `privileged` SCC: + + ``` + oc adm policy add-scc-to-user privileged -z velero -n velero + ``` + +2. Install Velero with the '--privileged-node-agent' option to request a privileged mode: + + ``` + velero install --use-node-agent --privileged-node-agent + ``` + + +If node-agent is not running in a privileged mode, it will not be able to access pods volumes within the mounted +hostpath directory because of the default enforced SELinux mode configured in the host system level. You can +[create a custom SCC](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) to relax the +security in your cluster so that node-agent pods are allowed to use the hostPath volume plugin without granting +them access to the `privileged` SCC. + +By default a userland openshift namespace will not schedule pods on all nodes in the cluster. + +To schedule on all nodes the namespace needs an annotation: + +``` +oc annotate namespace openshift.io/node-selector="" +``` + +This should be done before velero installation. + +Or the ds needs to be deleted and recreated: + +``` +oc get ds node-agent -o yaml -n > ds.yaml +oc annotate namespace openshift.io/node-selector="" +oc create -n -f ds.yaml +``` + +**VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS)** + +You need to enable the `Allow Privileged` option in your plan configuration so that Velero is able to mount the hostpath. + +The hostPath should be changed from `/var/lib/kubelet/pods` to `/var/vcap/data/kubelet/pods` + +```yaml +hostPath: + path: /var/vcap/data/kubelet/pods +``` + +## To back up + +Velero supports two approaches of discovering pod volumes that need to be backed up using FSB: + +- Opt-in approach: Where every pod containing a volume to be backed up using FSB must be annotated +with the volume's name. +- Opt-out approach: Where all pod volumes are backed up using FSB, with the ability to opt-out any +volumes that should not be backed up. + +The following sections provide more details on the two approaches. + +### Using the opt-out approach + +In this approach, Velero will back up all pod volumes using FSB with the exception of: + +- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps +- Hostpath volumes +- **Volumes explicitly excluded using annotations** (see below) + +**Important:** When you exclude a volume from FSB using annotations, Velero will attempt to back it up using volume snapshots instead (if CSI snapshots are enabled and the volume is a CSI volume or if properly configured with a compatible VolumeSnapshotLocation). + +It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` +annotation on the pod. + +Instructions to back up using this approach are as follows: + +1. Run the following command on each pod that contains volumes that should **not** be backed up using FSB + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes-excludes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + where the volume names are the names of the volumes in the pod spec. + + For example, in the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: app1 + namespace: sample + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc1-vm + mountPath: /volume-1 + - name: pvc2-vm + mountPath: /volume-2 + volumes: + - name: pvc1-vm + persistentVolumeClaim: + claimName: pvc1 + - name: pvc2-vm + claimName: pvc2 + ``` + to exclude FSB of volume `pvc1-vm`, you would run: + + ```bash + kubectl -n sample annotate pod/app1 backup.velero.io/backup-volumes-excludes=pvc1-vm + ``` + +2. Take a Velero backup: + + ```bash + velero backup create BACKUP_NAME --default-volumes-to-fs-backup OTHER_OPTIONS + ``` + + The above steps uses the opt-out approach on a per backup basis. + + Alternatively, this behavior may be enabled on all velero backups running the `velero install` command with + the `--default-volumes-to-fs-backup` flag. Refer [install overview][10] for details. + +3. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +### Using opt-in pod volume backup + +Velero, by default, uses this approach to discover pod volumes that need to be backed up using FSB. Every pod +containing a volume to be backed up using FSB must be annotated with the volume's name using the +`backup.velero.io/backup-volumes` annotation. + +**Note:** Volumes not annotated for FSB will be considered for volume snapshots if: +- `--snapshot-volumes` is not set to `false` +- The volume supports snapshots (either CSI or native) +- Either the volume is a CSI volume and CSI snapshots are enabled or there is a compatible VolumeSnapshotLocation configured + +Instructions to back up using this approach are as follows: + +1. Run the following for each pod that contains a volume to back up: + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + + where the volume names are the names of the volumes in the pod spec. + + For example, for the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: sample + namespace: foo + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc-volume + mountPath: /volume-1 + - name: emptydir-volume + mountPath: /volume-2 + volumes: + - name: pvc-volume + persistentVolumeClaim: + claimName: test-volume-claim + - name: emptydir-volume + emptyDir: {} + ``` + + You'd run: + + ```bash + kubectl -n foo annotate pod/sample backup.velero.io/backup-volumes=pvc-volume,emptydir-volume + ``` + + This annotation can also be provided in a pod template spec if you use a controller to manage your pods. + +1. Take a Velero backup: + + ```bash + velero backup create NAME OPTIONS... + ``` + +1. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +## To restore + +Regardless of how volumes are discovered for backup using FSB, the process of restoring remains the same. + +1. Restore from your Velero backup: + + ```bash + velero restore create --from-backup BACKUP_NAME OPTIONS... + ``` + +1. When the restore completes, view information about your pod volume restores: + + ```bash + velero restore describe YOUR_RESTORE_NAME + ``` + ```bash + kubectl -n velero get podvolumerestores -l velero.io/restore-name=YOUR_RESTORE_NAME -o yaml + ``` + +## Limitations + +- `hostPath` volumes are not supported. [Local persistent volumes][5] are supported. +- At present, Velero uses a static, common encryption key for all backup repositories it creates. **This means +that anyone who has access to your backup storage can decrypt your backup data**. Make sure that you limit access +to the backup storage appropriately. +- An incremental backup chain will be maintained across pod reschedules for PVCs. However, for pod volumes that +are *not* PVCs, such as `emptyDir` volumes, when a pod is deleted/recreated (for example, by a ReplicaSet/Deployment), +the next backup of those volumes will be full rather than incremental, because the pod volume's lifecycle is assumed +to be defined by its pod. +- Even though the backup data could be incrementally preserved, for a single file data, FSB leverages on deduplication +to find the difference to be saved. This means that large files (such as ones storing a database) will take a long time +to scan for data deduplication, even if the actual difference is small. +- Velero's File System Backup reads/writes data from volumes by accessing the node's filesystem, on which the pod is running. +For this reason, FSB can only backup volumes that are mounted by a pod and not directly from the PVC. For orphan PVC/PV pairs +(without running pods), some Velero users overcame this limitation running a staging pod (i.e. a busybox or alpine container +with an infinite sleep) to mount these PVC/PV pairs prior taking a Velero backup. +- Velero File System Backup expects volumes to be mounted under `/` (`hostPath` is configurable as mentioned in [Configure Node Agent DaemonSet spec](#configure-node-agent-daemonset-spec)). Some Kubernetes systems (i.e., [vCluster][11]) don't mount volumes under the `` sub-dir, Velero File System Backup is not working with them. +- File system restores of the same pod won't start until all the volumes of the pod get bound, even though some of the volumes have been bound and ready for restore. An a result, if a pod has multiple volumes, while only part of the volumes are restored by file system restore, these file system restores won't start until the other volumes are restored completely by other restore types (i.e., [CSI Snapshot Restore][12], [CSI Snapshot Data Movement][13]), the file system restores won't happen concurrently with those other types of restores. + +## Customize Restore Helper Container + +Velero uses a helper init container when performing a FSB restore. By default, the image for this container is same with the Velero server container. +You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with the alternate image. + +In addition, you can customize the resource requirements for the init container, should you need. + +The ConfigMap must look like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: fs-restore-action-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore + # item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/pod-volume-restore: RestoreItemAction +data: + # The value for "image" can either include a tag or not; + # if the tag is *not* included, the tag from the main Velero + # image will automatically be used. + image: myregistry.io/my-custom-helper-image[:OPTIONAL_TAG] + + # "cpuRequest" sets the request.cpu value on the restore init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuRequest: 200m + + # "memRequest" sets the request.memory value on the restore init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memRequest: 128Mi + + # "cpuLimit" sets the request.cpu value on the restore init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuLimit: 200m + + # "memLimit" sets the request.memory value on the restore init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memLimit: 128Mi + + # "secCtxRunAsUser" sets the securityContext.runAsUser value on the restore init containers during restore. + secCtxRunAsUser: 1001 + + # "secCtxRunAsGroup" sets the securityContext.runAsGroup value on the restore init containers during restore. + secCtxRunAsGroup: 999 + + # "secCtxAllowPrivilegeEscalation" sets the securityContext.allowPrivilegeEscalation value on the restore init containers during restore. + secCtxAllowPrivilegeEscalation: false + + # "secCtx" sets the securityContext object value on the restore init containers during restore. + # This key override `secCtxRunAsUser`, `secCtxRunAsGroup`, `secCtxAllowPrivilegeEscalation` if `secCtx.runAsUser`, `secCtx.runAsGroup` or `secCtx.allowPrivilegeEscalation` are set. + secCtx: | + capabilities: + drop: + - ALL + add: [] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 999 + +``` + +## Troubleshooting + +Run the following checks: + +Are your Velero server and daemonset pods running? + +```bash +kubectl get pods -n velero +``` + +Does your backup repository exist, and is it ready? + +```bash +velero repo get + +velero repo get REPO_NAME -o yaml +``` + +Are there any errors in your Velero backup/restore? + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME + +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +What is the status of your pod volume backups/restores? + +```bash +kubectl -n velero get podvolumebackups -l velero.io/backup-name=BACKUP_NAME -o yaml + +kubectl -n velero get podvolumerestores -l velero.io/restore-name=RESTORE_NAME -o yaml +``` + +Is there any useful information in the Velero server or daemon pod logs? + +```bash +kubectl -n velero logs deploy/velero +kubectl -n velero logs DAEMON_POD_NAME +``` + +**NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument +to the container command in the deployment/daemonset pod template spec. + +### Verifying backup methods used + +To understand which backup method was used for your volumes: + +1. Check if volumes were skipped for FSB: + ```bash + velero backup logs BACKUP_NAME | grep "skipped PVs" + ``` + This will show volumes opted out of FSB, which may still be backed up via snapshots. + +2. Verify volume snapshots were created: + ```bash + velero backup describe BACKUP_NAME --details + ``` + Look for the "Velero-Native Snapshots" or "CSI Snapshots" sections. + +3. Check PodVolumeBackups for FSB: + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=BACKUP_NAME + ``` + +**Note:** A volume appearing in the "skipped PVs" summary doesn't mean it wasn't backed up - it may have been backed up via volume snapshot instead. + +## Backup Method Decision Flow + +When Velero encounters a volume during backup, it follows this decision flow: + +1. **Is the volume opted out of FSB?** (via `backup.velero.io/backup-volumes-excludes`) + - Yes → Skip FSB, attempt volume snapshot (if configured) + - No → Continue to step 2 + +2. **Is the volume opted in for FSB?** (via `backup.velero.io/backup-volumes` or `--default-volumes-to-fs-backup`) + - Yes → Perform FSB, skip volume snapshot + - No → Attempt volume snapshot (if configured) + +3. **For volume snapshots to succeed:** + - CSI snapshots must be enabled for CSI volumes or compatible VolumeSnapshotLocation must be configured + - Volume type must be supported by the snapshot provider + - `--snapshot-volumes` must not be `false` + +## How backup and restore work + +### How Velero integrates with Kopia +Velero integrate Kopia modules into Velero's code, primarily two modules: +- Kopia Uploader: Velero makes some wrap and isolation around it to create a generic file system uploader, +which is used to backup pod volume data +- Kopia Repository: Velero integrates it with Velero's Unified Repository Interface, it is used to preserve the backup data and manage +the backup storage + +For more details, refer to [kopia architecture](https://kopia.io/docs/advanced/architecture/) and +Velero's [Unified Repository & Kopia Integration Design](https://github.com/vmware-tanzu/velero/blob/v1.18.0/design/Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md) + +### Custom resource and controllers +Velero has three custom resource definitions and associated controllers: + +- `BackupRepository` - represents/manages the lifecycle of Velero's backup repositories. Velero creates +a backup repository per namespace when the first FSB backup/restore for a namespace is requested. The backup +repository is backed by kopia, the `BackupRepository` controller invokes kopia internally, +refer to [kopia integration](#how-velero-integrates-with-kopia) for details. + +You can see information about your Velero's backup repositories by running `velero repo get`. + +- `PodVolumeBackup` - represents a FSB backup of a volume in a pod. The main Velero backup process creates +one or more of these when it finds an annotated pod. Each node in the cluster runs a controller for this +resource (in a daemonset) that handles the `PodVolumeBackups` for pods on that node. `PodVolumeBackup` is backed by kopia, +the data mover pod invokes kopia internally, refer to [kopia integration](#how-velero-integrates-with-kopia) for details. + +- `PodVolumeRestore` - represents a FSB restore of a pod volume. The main Velero restore process creates one +or more of these when it encounters a pod that has associated FSB backups. Each node in the cluster runs a +controller for this resource (in the same daemonset as above) that handles the `PodVolumeRestores` for pods +on that node. `PodVolumeRestore` is backed by kopia, the controller or data mover pod invokes kopia internally, +refer to [kopia integration](#how-velero-integrates-with-kopia) for details. + +### Backup + +1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod +that it's backing up for the volumes to be backed up using FSB. +2. When found, Velero first ensures a backup repository exists for the pod's namespace, by: + - checking if a `BackupRepository` custom resource already exists + - if not, creating a new one, and waiting for the `BackupRepository` controller to init/connect it +3. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation +4. The main Velero process now waits for the `PodVolumeBackup` resources to complete or fail +5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - finds the pod volume's subdirectory within the above volume + - creates a data mover pod which mounts the pod volume's subdirectory as a host path + - waits the data mover pod until it reaches to a terminal state + - updates the status of the custom resource to `Completed` or `Failed` +6. Kopia modules are launched inside the data mover pod and back up data from the host path mount +7. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named +`-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. +It will be used for restores, as seen in the next section. + +### Restore + +1. The main Velero restore process checks each existing `PodVolumeBackup` custom resource in the cluster to backup from. +2. For each `PodVolumeBackup` found, Velero first ensures a backup repository exists for the pod's namespace, by: + - checking if a `BackupRepository` custom resource already exists + - if not, creating a new one, and waiting for the `BackupRepository` controller to connect it (note that + in this case, the actual repository should already exist in backup storage, so the Velero controller will simply + check it for integrity and make a location connection) +3. Velero adds an init container to the pod, whose job is to wait for all FSB restores for the pod to complete (more +on this shortly) +4. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes +scheduler schedules this pod to a worker node. If the pod fails to be scheduled for +some reason (i.e. lack of cluster resources), the FSB restore will not be done. +5. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod +6. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail +7. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - waits for the pod to be running the init container + - finds the pod volume's subdirectory within the above volume + - launches kopia modules inside the node-agent pod to run the restore + - creates a data mover pod which mounts the pod volume's subdirectory as a host path and wait until it reaches to a terminal state + - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero + restore that this pod volume restore is for + - updates the status of the custom resource to `Completed` or `Failed` +8. Kopia modules are launched inside the data mover pod and restore data to the host path mount +9. The init container that was added to the pod is running a process that waits until it finds a file +within each restored volume, under `.velero`, whose name is the UID of the Velero restore being run +10. Once all such files are found, the init container's process terminates successfully and the pod moves +on to running other init containers/the main containers. + +Velero won't restore a resource if a that resource is scaled to 0 and already exists in the cluster. If Velero restored the +requested pods in this scenario, the Kubernetes reconciliation loops that manage resources would delete the running pods +because its scaled to be 0. Velero will be able to restore once the resources is scaled up, and the pods are created and remain running. + +### Backup Deletion +When a backup is created, a snapshot is saved into the repository for the volume data under the both path. The snapshot is a reference to the volume data saved in the repository. +When deleting a backup, Velero calls the repository to delete the repository snapshot. So the repository snapshot disappears immediately after the backup is deleted. Then the volume data backed up in the repository turns to orphan, but it is not deleted by this time. The repository relies on the maintenance functionalitiy to delete the orphan data. +As a result, after you delete a backup, you don't see the backup storage size reduces until some full maintenance jobs completes successfully. And for the same reason, you should check and make sure that the periodical repository maintenance job runs and completes successfully. + +Even after deleting all the backups and their backup data (by repository maintenance), the backup storage is still not empty, some repository metadata are there to keep the instance of the backup repository. +Furthermore, Velero never deletes these repository metadata, if you are sure you'll never usage the backup repository, you can empty the backup storage manually. + +Kopia uploader may keep some internal snapshots which is not managed by Velero. In normal cases, the internal snapshots are deleted along with running of backups. +However, if you run a backup which aborts halfway(some internal snapshots are thereby generated) and never run new backups again, some internal snapshots may be left there. In this case, since you stop using the backup repository, you can delete the entire repository metadata from the backup storage manually. + +### Parallelism +By default, one `PodVolumeBackup`/`PodVolumeRestore` request is handled in a node at a time. You can configure more parallelism per node by [node-agent Concurrency Configuration][19]. +By the meantime, one data mover pod is created for each volume to be backed up/restored, if there is no available concurrency quota, the data mover pod has to wait there. To make a control of the data mover pods, you can configure the [node-agent Prepare Queue Length][20]. + +For each volume, files in the volume are processed in parallel. You can use `--parallel-files-upload` backup flag or `--parallel-files-download` restore flag to control how many files are processed in parallel. Otherwise, if they are not set, Velero by default refers to the number of CPU cores in the node (where the backup/restore is running) for the parallelism. That is to say, the parallelism is not affected by the CPU request/limit set to the data mover pods. + +Notice that Golang 1.25 and later respects the CPU limit set to the pods to decide the physical threads provisioned to the pod processes (see [Container-aware GOMAXPROCS][23] for more details), so for Velero 1.18 (which consumes Golang 1.25) and later, if you set a CPU limit to the data mover pods, you may not get the expected performance (e.g., backup/restore throughput) with the default parallelism. The outcome may or may not be obvious varying on your volume data. If it is required, you could customize `--parallel-files-upload` or `--parallel-files-download` according to the CPU limit set to the data mover pods. + +### Restart and resume +When Velero server is restarted, the running backups/restores will be marked as `Failed`. The corresponding `PodVolumeBackup`/`PodVolumeRestore` will be canceled. +When node-agent is restarted, the controller will try to recapture and resume the `PodVolumeBackup`/`PodVolumeRestore`. If the resume fails, the `PodVolumeBackup`/`PodVolumeRestore` will be canceled. + +### Cancellation + +At present, Velero backup and restore doesn't support end to end cancellation that is launched by users. +However, Velero cancels the `PodVolumeBackup`/`PodVolumeRestore` in below scenarios automatically: +- When Velero server is restarted +- When node-agent is restarted and the resume fails +- When an ongoing backup/restore is deleted +- When a backup/restore does not finish before the timeout (specified by Velero server parameter `fs-backup-timeout`, default value is `4 hours`) + +## 3rd party controllers + +### Monitor backup annotation + +Velero does not provide a mechanism to detect persistent volume claims that are missing the File System Backup annotation. + +To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watcher][7] + +## Support ReadOnlyRootFilesystem setting +When the Velero server/node-agent pod's SecurityContext sets the `ReadOnlyRootFileSystem` parameter to true, the Velero server/node-agent pod's filesystem is running in read-only mode. +If the user creates a backup with Kopia as the uploader, the backup will fail, because the Kopia needs to write some cache and configuration data into the pod filesystem. + +``` +Errors: Velero: name: /mongodb-0 message: /Error backing up item error: /failed to wait BackupRepository: backup repository is not ready: error to connect to backup repo: error to connect repo with storage: error to connect to repository: unable to write config file: unable to create config directory: mkdir /home/cnb/udmrepo: read-only file system name: /mongodb-1 message: /Error backing up item error: /failed to wait BackupRepository: backup repository is not ready: error to connect to backup repo: error to connect repo with storage: error to connect to repository: unable to write config file: unable to create config directory: mkdir /home/cnb/udmrepo: read-only file system name: /mongodb-2 message: /Error backing up item error: /failed to wait BackupRepository: backup repository is not ready: error to connect to backup repo: error to connect repo with storage: error to connect to repository: unable to write config file: unable to create config directory: mkdir /home/cnb/udmrepo: read-only file system Cluster: +``` + +The workaround is making those directories as ephemeral k8s volumes, then those directories are not counted as pod's root filesystem. +The `user-name` is the Velero pod's running user name. The default value is `cnb`. + +``` yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: velero + namespace: velero +spec: + template: + spec: + containers: + - name: velero + ...... + volumeMounts: + ...... + - mountPath: /home//udmrepo + name: udmrepo + - mountPath: /home//.cache + name: cache + ...... + volumes: + ...... + - emptyDir: {} + name: udmrepo + - emptyDir: {} + name: cache + ...... +``` + +## Priority Class Configuration + +For Velero built-in data mover, data mover pods launched during file system backup will use the priority class name configured in the node-agent configmap. The node-agent daemonset itself gets its priority class from the `--node-agent-priority-class-name` flag during Velero installation. This can help ensure proper scheduling behavior in resource-constrained environments. For more details on configuring data mover pod resources, see [Data Movement Pod Resource Configuration][21]. + +## Resource Consumption + +Both the uploader and repository consume remarkable CPU/memory during the backup/restore, especially for massive small files or large backup size cases. +Velero node-agent uses [BestEffort as the QoS][14] for node-agent pods (so no CPU/memory request/limit is set), so that backups/restores wouldn't fail due to resource throttling in any cases. +If you want to constraint the CPU/memory usage, you need to [customize the resource limits][15]. The CPU/memory consumption is always related to the scale of data to be backed up/restored, refer to [Performance Guidance][16] for more details, so it is highly recommended that you perform your own testing to find the best resource limits for your data. + +During the restore, the repository may also cache data/metadata so as to reduce the network footprint and speed up the restore. The repository uses its own policy to store and clean up the cache. +For Kopia repository, by default, the cache is stored in the data mover pod's root file system. If your root file system space is limited, the data mover pods may be evicted due to running out of the ephemeral storage, which causes the restore fails. To cope with this problem, Velero allows you: +- configure a limit of the cache size per backup repository, for more details, check [Backup Repository Configuration][18]. +- configure a dedicated volume for cache data, for more details, check [Data Movement Cache Volume][22]. + +## Restic Deprecation + +According to the [Velero Deprecation Policy][17], restic path is being deprecated starting from v1.15, specifically: +- For 1.15 and 1.16, if restic path is used by a backup, the backup still creates and succeeds but you will see warnings +- For 1.17 and 1.18, backups with restic path are disabled, but you are still allowed to restore from your previous restic backups +- From 1.19, both backups and restores with restic path will be disabled, you are not able to use 1.19 or higher to restore your restic backup data + +From 1.17, backup from restic path is not allowed, though you can still restore from the existing backups created by restic path. +Velero could automatically identify the legacy backups and switch to restic path without user intervention. + +### How Velero integrates with Restic +Velero integrate Restic binary directly, so the operations are done by calling Restic commands: +- Run `restic init` command to initialize the [restic repository](https://restic.readthedocs.io/en/latest/100_references.html#terminology) +- Run `restic prune` command periodically to prune restic repository +- Run `restic restore` commands to restore pod volume data + +For a restore from restic path, restic commands are called by the node-agent itself; whereas, for kopia path backup/restore, the data path runs in the data mover pods. +Restore from restic path is handled by the legacy `PodVolumeRestore` controller, so Resume and Cancellation are not supported: +- When Velero server is restarted, the legacy `PodVolumeRestore` is left as orphan and contineue running, though the restore has already marked as `Failed` +- When node-agent is restarted, the `PodVolumeRestore` is marked as `Failed` directly + +### Restic Repository +To support restic repository, the BackupRepository CR should be specially configured: + - You need to set the `resticRepoPrefix` value in BackupStorageLocation. For example, on AWS, `resticRepoPrefix` is something like + `s3:s3-us-west-2.amazonaws.com/bucket` (note that `resticRepoPrefix` doesn't work for Kopia). + +Velero still effectively manage restic repository, though you cannot write any new backup to it: +- When you delete a backup, the restic repository snapshots (if any) could be deleted from restic repository +- Velero backup repository controller periodically runs mainteance jobs for BackupRepository CRs representing restic repositories + + + +[1]: https://github.com/restic/restic +[2]: https://github.com/kopia/kopia +[3]: customize-installation.md#enable-file-system-backup +[4]: https://github.com/vmware-tanzu/velero/releases/ +[5]: https://kubernetes.io/docs/concepts/storage/volumes/#local +[6]: https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation +[7]: https://github.com/bitsbeats/velero-pvc-watcher +[8]: https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv +[9]: https://github.com/restic/restic/issues/1800 +[10]: customize-installation.md#default-pod-volume-backup-to-file-system-backup +[11]: https://www.vcluster.com/ +[12]: csi.md +[13]: csi-snapshot-data-movement.md +[14]: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ +[15]: customize-installation.md#customize-resource-requests-and-limits +[16]: performance-guidance.md +[17]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/GOVERNANCE.md#deprecation-policy +[18]: backup-repository-configuration.md +[19]: node-agent-concurrency.md +[20]: node-agent-prepare-queue-length.md +[21]: data-movement-pod-resource-configuration.md +[22]: data-movement-cache-volume.md +[23]: https://tip.golang.org/doc/go1.25#container-aware-gomaxprocs:~:text=Runtime%C2%B6-,Container%2Daware%20GOMAXPROCS,-%C2%B6 diff --git a/site/content/docs/v1.18/how-velero-works.md b/site/content/docs/v1.18/how-velero-works.md new file mode 100644 index 000000000..0f2fa10b9 --- /dev/null +++ b/site/content/docs/v1.18/how-velero-works.md @@ -0,0 +1,117 @@ +--- +title: "How Velero Works" +layout: docs +--- + +Each Velero operation -- on-demand backup, scheduled backup, restore -- is a custom resource, defined with a Kubernetes [Custom Resource Definition (CRD)][20] and stored in [etcd][22]. Velero also includes controllers that process the custom resources to perform backups, restores, and all related operations. + +You can back up or restore all objects in your cluster, or you can filter objects by type, namespace, and/or label. + +Velero is ideal for the disaster recovery use case, as well as for snapshotting your application state, prior to performing system operations on your cluster, like upgrades. + +## On-demand backups + +The **backup** operation: + +1. Uploads a tarball of copied Kubernetes objects into cloud object storage. + +1. Calls the cloud provider API to make disk snapshots of persistent volumes, if specified. + +You can optionally specify backup hooks to be executed during the backup. For example, you might +need to tell a database to flush its in-memory buffers to disk before taking a snapshot. [More about backup hooks][10]. + +Note that cluster backups are not strictly atomic. If Kubernetes objects are being created or edited at the time of backup, they might not be included in the backup. The odds of capturing inconsistent information are low, but it is possible. + +## Scheduled backups + +The **schedule** operation allows you to back up your data at recurring intervals. You can create a scheduled backup at any time, and the first backup is then performed at the schedule's specified interval. These intervals are specified by a Cron expression. + +Velero saves backups created from a schedule with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For more information see the [Backup Reference documentation](backup-reference.md). + + +## Backup workflow + +When you run `velero backup create test-backup`: + +1. The Velero client makes a call to the Kubernetes API server to create a `Backup` object. + +1. The `BackupController` notices the new `Backup` object and performs validation. + +1. The `BackupController` begins the backup process. It collects the data to back up by querying the API server for resources. + +1. The `BackupController` makes a call to the object storage service -- for example, AWS S3 -- to upload the backup file. + +By default, `velero backup create` makes disk snapshots of any persistent volumes. You can adjust the snapshots by specifying additional flags. Run `velero backup create --help` to see available flags. Snapshots can be disabled with the option `--snapshot-volumes=false`. + +![19] + +## Restores + +The **restore** operation allows you to restore all of the objects and persistent volumes from a previously created backup. You can also restore only a [filtered](resource-filtering.md) subset of objects and persistent volumes. Velero supports multiple namespace remapping--for example, in a single restore, objects in namespace "abc" can be recreated under namespace "def", and the objects in namespace "123" under "456". + +The default name of a restore is `-`, where `` is formatted as *YYYYMMDDhhmmss*. You can also specify a custom name. A restored object also includes a label with key `velero.io/restore-name` and value ``. + +By default, backup storage locations are created in read-write mode. However, during a restore, you can configure a backup storage location to be in read-only mode, which disables backup creation and deletion for the storage location. This is useful to ensure that no backups are inadvertently created or deleted during a restore scenario. + +You can optionally specify [restore hooks][11] to be executed during a restore or after resources are restored. For example, you might need to perform a custom database restore operation before the database application containers start. + +### Restore workflow + +When you run `velero restore create`: + +1. The Velero client makes a call to the Kubernetes API server to create a [`Restore`](api-types/restore.md) object. + +1. The `RestoreController` notices the new Restore object and performs validation. + +1. The `RestoreController` fetches the backup information from the object storage service. It then runs some preprocessing on the backed up resources to make sure the resources will work on the new cluster. For example, using the [backed-up API versions](#backed-up-api-versions) to verify that the restore resource will work on the target cluster. + +1. The `RestoreController` starts the restore process, restoring each eligible resource one at a time. + +By default, Velero performs a non-destructive restore, meaning that it won't delete any data on the target cluster. If a resource in the backup already exists in the target cluster, Velero will skip that resource. You can configure Velero to use an update policy instead using the [`--existing-resource-policy`](restore-reference.md#restore-existing-resource-policy) restore flag. When this flag is set to `update`, Velero will attempt to update an existing resource in the target cluster to match the resource from the backup. + +For more details about the Velero restore process, see the [Restore Reference](restore-reference.md) page. + +## Backed-up API versions + +Velero backs up resources using the Kubernetes API server's *preferred version* for each group/resource. When restoring a resource, this same API group/version must exist in the target cluster in order for the restore to be successful. + +For example, if the cluster being backed up has a `gizmos` resource in the `things` API group, with group/versions `things/v1alpha1`, `things/v1beta1`, and `things/v1`, and the server's preferred group/version is `things/v1`, then all `gizmos` will be backed up from the `things/v1` API endpoint. When backups from this cluster are restored, the target cluster **must** have the `things/v1` endpoint in order for `gizmos` to be restored. Note that `things/v1` **does not** need to be the preferred version in the target cluster; it just needs to exist. + +## Set a backup to expire + +When you create a backup, you can specify a TTL (time to live) by adding the flag `--ttl `. If Velero sees that an existing backup resource is expired, it removes: + +* The backup resource +* The backup file from cloud object storage +* All PersistentVolume snapshots +* All associated Restores + +The TTL flag allows the user to specify the backup retention period with the value specified in hours, minutes and seconds in the form `--ttl 24h0m0s`. If not specified, a default TTL value of 30 days will be applied. + +The effects of expiration are not applied immediately, they are applied when the gc-controller runs its reconciliation loop every hour by default. If needed, you can adjust the frequency of the reconciliation loop using the `--garbage-collection-frequency +` flag. + +If backup fails to delete, a label `velero.io/gc-failure=` will be added to the backup custom resource. + +You can use this label to filter and select backups that failed to delete. + +Implemented reasons are: +- BSLNotFound: Backup storage location not found +- BSLCannotGet: Backup storage location cannot be retrieved from the API server for reasons other than not found +- BSLReadOnly: Backup storage location is read-only + +## Object storage sync + +Velero treats object storage as the source of truth. It continuously checks to see that the correct backup resources are always present. If there is a properly formatted backup file in the storage bucket, but no corresponding backup resource in the Kubernetes API, Velero synchronizes the information from object storage to Kubernetes. + +This allows restore functionality to work in a cluster migration scenario, where the original backup objects do not exist in the new cluster. + +Likewise, if a `Completed` backup object exists in Kubernetes but not in object storage, it will be deleted from Kubernetes since the backup tarball no longer exists. +`Failed` or `PartiallyFailed` backup will not be removed by object storage sync. + +[10]: backup-hooks.md +[11]: restore-hooks.md +[19]: /docs/main/img/backup-process.png +[20]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#customresourcedefinitions +[21]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#custom-controllers +[22]: https://github.com/coreos/etcd diff --git a/site/content/docs/v1.18/image-tagging.md b/site/content/docs/v1.18/image-tagging.md new file mode 100644 index 000000000..64e9310b5 --- /dev/null +++ b/site/content/docs/v1.18/image-tagging.md @@ -0,0 +1,24 @@ +--- +title: "Image tagging policy" +layout: docs +--- + +This document describes Velero's image tagging policy. + +## Released versions + +`velero/velero:` + +Velero follows the [Semantic Versioning](http://semver.org/) standard for releases. Each tag in the `github.com/vmware-tanzu/velero` repository has a matching image, `velero/velero:v1.0.0`. + +### Latest + +`velero/velero:latest` + +The `latest` tag follows the most recently released version of Velero. + +## Development + +`velero/velero:main` + +The `main` tag follows the latest commit to land on the `main` branch. diff --git a/site/content/docs/v1.18/img/README.md b/site/content/docs/v1.18/img/README.md new file mode 100644 index 000000000..85c071c63 --- /dev/null +++ b/site/content/docs/v1.18/img/README.md @@ -0,0 +1 @@ +Some of these diagrams (for instance backup-process.png), have been created on [draw.io](https://www.draw.io), using the "Include a copy of my diagram" option. If you want to make changes to these diagrams, try importing them into draw.io, and you should have access to the original shapes/text that went into the originals. diff --git a/site/content/docs/v1.18/img/backup-process.png b/site/content/docs/v1.18/img/backup-process.png new file mode 100644 index 000000000..7d4f10d56 Binary files /dev/null and b/site/content/docs/v1.18/img/backup-process.png differ diff --git a/site/content/docs/v1.18/img/gv_priority1-caseA.png b/site/content/docs/v1.18/img/gv_priority1-caseA.png new file mode 100644 index 000000000..a1940453f Binary files /dev/null and b/site/content/docs/v1.18/img/gv_priority1-caseA.png differ diff --git a/site/content/docs/v1.18/img/gv_priority1-caseB.png b/site/content/docs/v1.18/img/gv_priority1-caseB.png new file mode 100644 index 000000000..6623f7f81 Binary files /dev/null and b/site/content/docs/v1.18/img/gv_priority1-caseB.png differ diff --git a/site/content/docs/v1.18/img/gv_priority2-caseC.png b/site/content/docs/v1.18/img/gv_priority2-caseC.png new file mode 100644 index 000000000..980c0d5f2 Binary files /dev/null and b/site/content/docs/v1.18/img/gv_priority2-caseC.png differ diff --git a/site/content/docs/v1.18/img/gv_priority3-caseD.png b/site/content/docs/v1.18/img/gv_priority3-caseD.png new file mode 100644 index 000000000..11c9a11ce Binary files /dev/null and b/site/content/docs/v1.18/img/gv_priority3-caseD.png differ diff --git a/site/content/docs/v1.18/img/velero.png b/site/content/docs/v1.18/img/velero.png new file mode 100644 index 000000000..a1937a1dc Binary files /dev/null and b/site/content/docs/v1.18/img/velero.png differ diff --git a/site/content/docs/v1.18/locations.md b/site/content/docs/v1.18/locations.md new file mode 100644 index 000000000..db347801b --- /dev/null +++ b/site/content/docs/v1.18/locations.md @@ -0,0 +1,316 @@ +--- +title: "Backup Storage Locations and Volume Snapshot Locations" +layout: docs +--- + +## Overview + +Velero has two custom resources, `BackupStorageLocation` and `VolumeSnapshotLocation`, that are used to configure where Velero backups and their associated persistent volume snapshots are stored. + +A `BackupStorageLocation` is defined as a bucket or a prefix within a bucket under which all Velero data is stored and a set of additional provider-specific fields (AWS region, Azure storage account, etc.). Velero assumes it has control over the location you provide so you should use a dedicated bucket or prefix. If you provide a prefix, then the rest of the bucket is safe to use for multiple purposes. The [API documentation][1] captures the configurable parameters for each in-tree provider. + +A `VolumeSnapshotLocation` is defined entirely by provider-specific fields (AWS region, Azure resource group, Portworx snapshot type, etc.) The [API documentation][2] captures the configurable parameters for each in-tree provider. + +The user can pre-configure one or more possible `BackupStorageLocations` and one or more `VolumeSnapshotLocations`, and can select *at backup creation time* the location in which the backup and associated snapshots should be stored. + +This configuration design enables a number of different use cases, including: + +- Take snapshots of more than one kind of persistent volume in a single Velero backup. For example, in a cluster with both EBS volumes and Portworx volumes +- Have some Velero backups go to a bucket in an eastern USA region, and others go to a bucket in a western USA region, or to a different storage provider +- For volume providers that support it, like Portworx, you can have some snapshots stored locally on the cluster and have others stored in the cloud + +## Limitations / Caveats + +- Velero supports multiple credentials for `BackupStorageLocations`, allowing you to specify the credentials to use with any `BackupStorageLocation`. + However, use of this feature requires support within the plugin for the object storage provider you wish to use. + All [plugins maintained by the Velero team][5] support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this feature is supported. + +- Velero supports multiple credentials for `VolumeSnapshotLocations`, allowing you to specify the credentials to use with any `VolumeSnapshotLocation`. + However, use of this feature requires support within the plugin for the object storage provider you wish to use. + All [plugins maintained by the Velero team][5] support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this feature is supported. + +- Volume snapshots are still limited by where your provider allows you to create snapshots. For example, AWS and Azure do not allow you to create a volume snapshot in a different region than where the volume is. If you try to take a Velero backup using a volume snapshot location with a different region than where your cluster's volumes are, the backup will fail. + +- Each Velero backup has one `BackupStorageLocation`, and one `VolumeSnapshotLocation` per volume provider. It is not possible (yet) to send a single Velero backup to multiple backup storage locations simultaneously, or a single volume snapshot to multiple locations simultaneously. However, you can always set up multiple scheduled backups that differ only in the storage locations used if redundancy of backups across locations is important. + +- Cross-provider snapshots are not supported. If you have a cluster with more than one type of volume, like EBS and Portworx, but you only have a `VolumeSnapshotLocation` configured for EBS, then Velero will **only** snapshot the EBS volumes. + +- File System Backup data is stored under a prefix/subdirectory of the main Velero bucket, and will go into the bucket corresponding to the `BackupStorageLocation` selected by the user at backup creation time. + +- Velero's backups are split into 2 pieces - the metadata stored in object storage, and snapshots/backups of the persistent volume data. Right now, Velero *itself* does not encrypt either of them, instead it relies on the native mechanisms in the object and snapshot systems. A special case is File System Backup, which backs up the persistent volume data at the filesystem level and send it to Velero's object storage. + +- Velero's compression for object metadata is limited, using Golang's tar implementation. In most instances, Kubernetes objects are limited to 1.5MB in size, but many don't approach that, meaning that compression may not be necessary. Note that File System Backup has not yet implemented compression, but does have de-deduplication capabilities. + +- If you have [multiple](customize-installation.md/#configure-more-than-one-storage-location-for-backups-or-volume-snapshots) `VolumeSnapshotLocations` configured for a provider, you must always specify a valid `VolumeSnapshotLocation` when creating a backup, even if you are using [File System Backup](file-system-backup.md) for volume backups. You can optionally decide to set the [`--default-volume-snapshot-locations`](customize-locations.md#set-default-backup-storage-location-or-volume-snapshot-locations) flag using the `velero server`, which lists the default `VolumeSnapshotLocation` Velero should use if a `VolumeSnapshotLocation` is not specified when creating a backup. If you only have one `VolumeSnapshotLocation` for a provider, Velero will automatically use that location as the default. + +## Examples + +Let's look at some examples of how you can use this configuration mechanism to address some common use cases: + +### Take snapshots of more than one kind of persistent volume in a single Velero backup + +During server configuration: + +```shell +velero snapshot-location create ebs-us-east-1 \ + --provider aws \ + --config region=us-east-1 + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +velero backup create full-cluster-backup \ + --volume-snapshot-locations ebs-us-east-1,portworx-cloud +``` + +Alternately, since in this example there's only one possible volume snapshot location configured for each of our two providers (`ebs-us-east-1` for `aws`, and `portworx-cloud` for `portworx`), Velero doesn't require them to be explicitly specified when creating the backup: + +```shell +velero backup create full-cluster-backup +``` + +### Have some Velero backups go to a bucket in an eastern USA region (default), and others go to a bucket in a western USA region + +In this example, two `BackupStorageLocations` will be created within the same account but in different regions. +They will both use the credentials provided at install time and stored in the `cloud-credentials` secret. +If you need to configure unique credentials for each `BackupStorageLocation`, please refer to the [later example][8]. + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-east-1 \ + --default + +velero backup-location create backups-secondary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 +``` + +A "default" backup storage location (BSL) is where backups get saved to when no BSL is specified at backup creation time. + +You can change the default backup storage location at any time by setting the `--default` flag using the +`velero backup-location set` command and configure a different location to be the default. + +Examples: + +```shell +velero backup-location set backups-secondary --default +``` + + + +During backup creation: + +```shell +velero backup create full-cluster-backup +``` + +Or: + +```shell +velero backup create full-cluster-alternate-location-backup \ + --storage-location backups-secondary +``` + +### For volume providers that support it (like Portworx), have some snapshots be stored locally on the cluster and have others be stored in the cloud + +During server configuration: + +```shell +velero snapshot-location create portworx-local \ + --provider portworx \ + --config type=local + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +# Note that since in this example you have two possible volume snapshot locations for the Portworx +# provider, you need to explicitly specify which one to use when creating a backup. Alternately, +# you can set the --default-volume-snapshot-locations flag on the `velero server` command (run by +# the Velero deployment) to specify which location should be used for each provider by default, in +# which case you don't need to specify it when creating a backup. +velero backup create local-snapshot-backup \ + --volume-snapshot-locations portworx-local +``` + +Or: + +```shell +velero backup create cloud-snapshot-backup \ + --volume-snapshot-locations portworx-cloud +``` + +### Use a single location + +If you don't have a use case for more than one location, it's still easy to use Velero. Let's assume you're running on AWS, in the `us-west-1` region: + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 \ + --default + +velero snapshot-location create ebs-us-west-1 \ + --provider aws \ + --config region=us-west-1 +``` + +During backup creation: + +```shell +# Velero will automatically use your configured backup storage location and volume snapshot location. +# Nothing needs to be specified when creating a backup. +velero backup create full-cluster-backup +``` + +### Create a storage location that uses unique credentials + +It is possible to create additional `BackupStorageLocations` that use their own credentials. +This enables you to save backups to another storage provider or to another account with the storage provider you are already using. + +If you create additional `BackupStorageLocations` without specifying the credentials to use, Velero will use the credentials provided at install time and stored in the `cloud-credentials` secret. +Please see the [earlier example][9] for details on how to create multiple `BackupStorageLocations` that use the same credentials. + +#### Prerequisites +- This feature requires support from the [object storage provider plugin][5] you wish to use. + All plugins maintained by the Velero team support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this is supported. +- The [plugin for the object storage provider][5] you wish to use must be [installed][6]. +- You must create a file with the object storage credentials. Follow the instructions provided by your object storage provider plugin to create this file. + +Once you have installed the necessary plugin and created the credentials file, create a [Kubernetes Secret][7] in the Velero namespace that contains these credentials: + +```shell +kubectl create secret generic -n velero credentials --from-file=bsl= +``` + +This will create a secret named `credentials` with a single key (`bsl`) which contains the contents of your credentials file. +Next, create a `BackupStorageLocation` that uses this Secret by passing the Secret name and key in the `--credential` flag. +When interacting with this `BackupStorageLocation` in the future, Velero will fetch the data from the key within the Secret you provide. + +For example, a new `BackupStorageLocation` with a Secret would be configured as follows: + +```bash +velero backup-location create \ + --provider \ + --bucket \ + --config region= \ + --credential== +``` + +The `BackupStorageLocation` is ready to use when it has the phase `Available`. +You can check the status with the following command: + +```bash +velero backup-location get +``` + +To use this new `BackupStorageLocation` when performing a backup, use the flag `--storage-location ` when running `velero backup create`. +You may also set this new `BackupStorageLocation` as the default with the command `velero backup-location set --default `. + +### Modify the credentials used by an existing storage location + +By default, `BackupStorageLocations` will use the credentials provided at install time and stored in the `cloud-credentials` secret in the Velero namespace. +You can modify these existing credentials by [editing the `cloud-credentials` secret][10], however, these changes will apply to all locations using this secret. +This may be the desired outcome, for example, in the case where you wish to rotate the credentials used for a particular account. + +You can also opt to modify an existing `BackupStorageLocation` such that it uses its own credentials by using the `backup-location set` command. + +If you have a credentials file that you wish to use for a `BackupStorageLocation`, follow the instructions above to create the Secret with that file in the Velero namespace. + +Once you have created the Secret, or have an existing Secret which contains the credentials you wish to use for your `BackupStorageLocation`, set the credential to use as follows: + +```bash +velero backup-location set \ + --credential== +``` + +### Create a volume snapshot location that uses unique credentials + +It is possible to create additional `VolumeSnapshotLocations` that use their own credentials. +This may be necessary if you already have default credentials which don't match the account used by the cloud volumes being backed up. + +If you create additional `VolumeSnapshotLocations` without specifying the credentials to use, Velero will use the credentials provided at install time and stored in the `cloud-credentials` secret. + +#### Prerequisites +- This feature requires support from the [volume snapshotter plugin][5] you wish to use. + All plugins maintained by the Velero team support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this is supported. +- The [plugin for the volume snapshotter provider][5] you wish to use must be [installed][6]. +- You must create a file with the object storage credentials. Follow the instructions provided by your object storage provider plugin to create this file. + +Once you have installed the necessary plugin and created the credentials file, create a [Kubernetes Secret][7] in the Velero namespace that contains these credentials: + +```shell +kubectl create secret generic -n velero credentials --from-file=vsl= +``` + +This will create a secret named `credentials` with a single key (`vsl`) which contains the contents of your credentials file. +Next, create a `VolumeSnapshotLocation` that uses this Secret by passing the Secret name and key in the `--credential` flag. +When interacting with this `VolumeSnapshotLocation` in the future, Velero will fetch the data from the key within the Secret you provide. + +For example, a new `VolumeSnapshotLocation` with a Secret would be configured as follows: + +```bash +velero snapshot-location create \ + --provider \ + --config region= \ + --credential== +``` + +To use this new `VolumeSnapshotLocation` when performing a backup, use the flag `--volume-snapshot-locations [, \ + --credential== +``` + +## Additional Use Cases + +1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details. + +1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details. + + + +[1]: api-types/backupstoragelocation.md +[2]: api-types/volumesnapshotlocation.md +[3]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/volumesnapshotlocation.md +[4]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md +[5]: /plugins +[6]: overview-plugins.md +[7]: https://kubernetes.io/docs/concepts/configuration/secret/ +[8]: #create-a-storage-location-that-uses-unique-credentials +[9]: #have-some-velero-backups-go-to-a-bucket-in-an-eastern-usa-region-default-and-others-go-to-a-bucket-in-a-western-usa-region +[10]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret diff --git a/site/content/docs/v1.18/maintainers.md b/site/content/docs/v1.18/maintainers.md new file mode 100644 index 000000000..e249880ab --- /dev/null +++ b/site/content/docs/v1.18/maintainers.md @@ -0,0 +1,37 @@ +--- +title: "Instructions for Maintainers" +layout: docs +toc: "true" +--- + +There are some guidelines maintainers need to follow. We list them here for quick reference, especially for new maintainers. These guidelines apply to all projects in the Velero org, including the main project, the Velero Helm chart, and all other [related repositories](https://github.com/vmware-tanzu/velero/blob/v1.18.0/GOVERNANCE.md#code-repositories). + +Please be sure to also go through the guidance under the entire [Contribute](start-contributing/) section. + +## Reviewing PRs +- PRs require 2 approvals before it is mergeable. +- The second reviewer usually merges the PR (if you notice a PR open for a while and with 2 approvals, go ahead and merge it!) +- As you review a PR that is not yet ready to merge, please check if the "request review" needs to be refreshed for any reviewer (this is better than @mention at them) +- Refrain from @mention other maintainers to review the PR unless it is an immediate need. All maintainers already get notified through the automated add to the "request review". If it is an urgent need, please add a helpful message as to why it is so people can properly prioritize work. +- There is no need to manually request reviewers: after the PR is created, all maintainers will be automatically added to the list (note: feel free to remove people if they are on PTO, etc). +- Be familiar with the [lazy consensus](https://github.com/vmware-tanzu/velero/blob/v1.18.0/GOVERNANCE.md#lazy-consensus) policy for the project. + +Some tips for doing reviews: +- There are some [code standards and general guidelines](https://velero.io/docs/v1.18.0/code-standards) we aim for +- We have [guidelines for writing and reviewing documentation](https://velero.io/docs/v1.18.0/style-guide/) +- When reviewing a design document, ensure it follows [our format and guidelines]( https://github.com/vmware-tanzu/velero/blob/v1.18.0/design/_template.md). Also, when reviewing a PR that implements a previously accepted design, ensure the associated design doc is moved to the [design/implemented](https://github.com/vmware-tanzu/velero/tree/main/design/implemented) folder. + + +## Creating a release +Maintainers are expected to create releases for the project. We have parts of the process automated, and full [instructions](release-instructions). +We are working towards automating more the Velero testing, but there is still a need for manual testing as part of the release process. +The manual test cases for release testing are documented [here](./manual-testing). + +## Community support +Maintainers are expected to participate in the community support rotation. We have guidelines for how we handle the [support](support-process). + +## Community engagement +Maintainers for the Velero project are highly involved with the open source community. All the online community meetings for the project are listed in our [community](community) page. + +## How do I become a maintainer? +The Velero project welcomes contributors of all kinds. We are also always on the look out for a high level of engagement from contributors and opportunities to bring in new maintainers. If this is of interest, take a look at how [adding a maintainer](https://github.com/vmware-tanzu/velero/blob/v1.18.0/GOVERNANCE.md#maintainers) is decided. diff --git a/site/content/docs/v1.18/manual-testing.md b/site/content/docs/v1.18/manual-testing.md new file mode 100644 index 000000000..411508738 --- /dev/null +++ b/site/content/docs/v1.18/manual-testing.md @@ -0,0 +1,98 @@ +--- +title: "Manual Testing Requirements for Velero" +layout: docs +--- + +Although we have automated unit and end-to-end tests, there is still a need for Velero to undergo manual tests during a release. +This document outlines the manual test operations that Velero needs to correctly perform in order to be considered ready for release. + +## Current test cases + +The following are test cases that are currently performed as part of a Velero release. + +### Install + +- Verify that Velero CRDs are compatible with the earliest and latest versions of Kubernetes that we support: + - Kubernetes v1.12 + - Kubernetes v1.20 + +### Upgrade + +- Verify that Velero upgrade instructions work + +### Basic functionality + +The "Backup and Restore" test cases below describe general backup and restore functionality that needs to run successfully on all the following providers that we maintain plugins for: +- AWS +- GCP +- Microsoft Azure +- VMware vSphere + +#### Backup and Restore + +- Verify that a backup and restore using Volume Snapshots can be performed +- Verify that a backup and restore using File System Backup can be performed +- Verify that a backup of a cluster workload can be restored in a new cluster +- Verify that an installation using the latest version can be used to restore from backups created with the last 3 versions. + - e.g. Install Velero 1.6 and use it to restore backups from Velero v1.3, v1.4, v1.5. + +### Working with Multiple Providers + +The following are test cases that exercise Velero behaviour when interacting with multiple providers: + +- Verify that a backup and restore to multiple BackupStorageLocations using the same provider with unique credentials can be performed +- Verify that a backup and restore to multiple BackupStorageLocations using different providers with unique credentials can be performed +- Verify that a backup and restore that includes volume snapshots using different providers for the snapshots and object storage can be performed + - e.g. perform a backup and restore using AWS for the VolumeSnapshotLocation and Azure Blob Storage as the BackupStorageLocation + +## Future test cases + +The following are test cases that are not currently performed as part of a Velero release but cases that we will want to cover with future releases. + +### Schedules + +- Verify that schedules create a backup upon creation and create Backup resources at the correct frequency + +### Resource management + +- Verify that deleted backups are successfully removed from object storage +- Verify that backups that have been removed from object storage can still be deleted with `velero delete backup` +- Verify that Volume Snapshots associated with a deleted backup are removed +- Verify that backups that exceed their TTL are deleted +- Verify that existing backups in object storage are synced to Velero + +### Backup repository test cases + +- Verify that backup repository maintenance is performed as the specified interval + +### Backup Hooks + +- Verify that a pre backup hook provided via pod annotation is performed during backup +- Verify that a pre backup hook provided via Backup spec is performed during backup +- Verify that a post backup hook provided via pod annotation is performed during backup +- Verify that a post backup hook provided via Backup spec is performed during backup + +### Restore Hooks + +- Verify that an InitContainer restore hook provided via pod annotation is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore that includes restoring File System Backup volumes +- Verify that an Exec restore hook provided via pod annotation is performed during restore +- Verify that an Exec restore hook provided via Restore spec is performed during restore + + +#### Resource filtering + +- Verify that backups and restores correctly apply the following resource filters: + - `--include-namespaces` + - `--include-resources` + - `--include-cluster-resources` + - `--exclude-namespaces` + - `--exclude-resources` + - `velero.io/exclude-from-backup=true` label + +- Since v1.11, new resource filters are added. The new filters only work for backup, and cannot work with old filters (`--include-resources`, `--exclude-resources` and `--include-cluster-resources`). Need to verify backups correctly apply the following new resource filters: + - `--exclude-cluster-scoped-resources` + - `--include-cluster-scoped-resources` + - `--exclude-namespace-scoped-resources` + - `--include-namespace-scoped-resources` diff --git a/site/content/docs/v1.18/migration-case.md b/site/content/docs/v1.18/migration-case.md new file mode 100644 index 000000000..7492a159b --- /dev/null +++ b/site/content/docs/v1.18/migration-case.md @@ -0,0 +1,108 @@ +--- +title: "Cluster migration" +layout: docs +--- + +Velero's backup and restore capabilities make it a valuable tool for migrating your data between clusters. Cluster migration with Velero is based on Velero's [object storage sync](how-velero-works.md#object-storage-sync) functionality, which is responsible for syncing Velero resources from your designated object storage to your cluster. This means that to perform cluster migration with Velero you must point each Velero instance running on clusters involved with the migration to the same cloud object storage location. + +This page outlines a cluster migration scenario and some common configurations you will need to start using Velero to begin migrating data. + +## Before migrating your cluster + +Before migrating you should consider the following, + +* Velero does not natively support the migration of persistent volumes snapshots across cloud providers. If you would like to migrate volume data between cloud platforms, enable [File System Backup](file-system-backup.md), which will backup volume contents at the filesystem level. +* Velero doesn't support restoring into a cluster with a lower Kubernetes version than where the backup was taken. +* Migrating workloads across clusters that are not running the same version of Kubernetes might be possible, but some factors need to be considered before migration, including the compatibility of API groups between clusters for each custom resource. If a Kubernetes version upgrade breaks the compatibility of core/native API groups, migrating with Velero will not be possible without first updating the impacted custom resources. For more information about API group versions, please see [EnableAPIGroupVersions](enable-api-group-versions-feature.md). +* The Velero plugin for AWS and Azure does not support migrating data between regions. If you need to do this, you must use [File System Backup](file-system-backup.md). + + +## Migration Scenario + +This scenario steps through the migration of resources from Cluster 1 to Cluster 2. In this scenario, both clusters are using the same cloud provider, AWS, and Velero's [AWS plugin](https://github.com/vmware-tanzu/velero-plugin-for-aws). + +1. On Cluster 1, make sure Velero is installed and points to an object storage location using the `--bucket` flag. + + ``` + velero install --provider aws --image velero/velero:v1.8.0 --plugins velero/velero-plugin-for-aws:v1.4.0 --bucket velero-migration-demo --secret-file xxxx/aws-credentials-cluster1 --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 + ``` + + During installation, Velero creates a Backup Storage Location called `default` inside the `--bucket` your provided in the install command, in this case `velero-migration-demo`. This is the location that Velero will use to store backups. Running `velero backup-location get` will show the backup location of Cluster 1. + + + ``` + velero backup-location get + NAME PROVIDER BUCKET/PREFIX PHASE LAST VALIDATED ACCESS MODE DEFAULT + default aws velero-migration-demo Available 2022-05-13 13:41:30 +0800 CST ReadWrite true + ``` + +1. Still on Cluster 1, make sure you have a backup of your cluster. Replace `` with a name for your backup. + + ``` + velero backup create + ``` + + Alternatively, you can create a [scheduled backup](https://velero.io/docs/v1.18.0/backup-reference/#schedule-a-backup) of your data with the Velero `schedule` operation. This is the recommended way to make sure your data is automatically backed up according to the schedule you define. + + The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works](how-velero-works.md#set-a-backup-to-expire) for more information about backup expiry. + +1. On Cluster 2, make sure that Velero is installed. Note that the install command below has the same `region` and `--bucket` location as the install command for Cluster 1. The Velero plugin for AWS does not support migrating data between regions. + + ``` + velero install --provider aws --image velero/velero:v1.8.0 --plugins velero/velero-plugin-for-aws:v1.4.0 --bucket velero-migration-demo --secret-file xxxx/aws-credentials-cluster2 --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 + ``` + + Alternatively you could configure `BackupStorageLocations` and `VolumeSnapshotLocations` after installing Velero on Cluster 2, pointing to the `--bucket` location and `region` used by Cluster 1. To do this you can use to `velero backup-location create` and `velero snapshot-location create` commands. + + ``` + velero backup-location create bsl --provider aws --bucket velero-migration-demo --config region=us-east-2 --access-mode=ReadOnly + ``` + + Its recommended that you configure the `BackupStorageLocations` as read-only + by using the `--access-mode=ReadOnly` flag for `velero backup-location create`. This will make sure that the backup is not deleted from the object store by mistake during the restore. See `velero backup-location –help` for more information about the available flags for this command. + + ``` + velero snapshot-location create vsl --provider aws --config region=us-east-2 + ``` + See `velero snapshot-location –help` for more information about the available flags for this command. + + +1. Continuing on Cluster 2, make sure that the Velero Backup object created on Cluster 1 is available. `` should be the same name used to create your backup of Cluster 1. + + ``` + velero backup describe + ``` + + Velero resources are [synchronized](how-velero-works.md#object-storage-sync) with the backup files in object storage. This means that the Velero resources created by Cluster 1's backup will be synced to Cluster 2 through the shared Backup Storage Location. Once the sync occurs, you will be able to access the backup from Cluster 1 on Cluster 2 using Velero commands. The default sync interval is 1 minute, so you may need to wait before checking for the backup's availability on Cluster 2. You can configure this interval with the `--backup-sync-period` flag to the Velero server on Cluster 2. + +1. On Cluster 2, once you have confirmed that the right backup is available, you can restore everything to Cluster 2. + + ``` + velero restore create --from-backup + ``` + + Make sure `` is the same backup name from Cluster 1. + +## Verify Both Clusters + +Check that the Cluster 2 is behaving as expected: + +1. On Cluster 2, run: + + ``` + velero restore get + ``` + +1. Then run: + + ``` + velero restore describe + ``` + + Your data that was backed up from Cluster 1 should now be available on Cluster 2. + +If you encounter issues, make sure that Velero is running in the same namespace in both clusters. + + +## Notice +If the two clusters couldn't share the snapshots generated by backup, for example migration from EKS to AKS, then please consider using [the file system backup](file-system-backup.md) or [the snapshot data mover](csi-snapshot-data-movement.md). \ No newline at end of file diff --git a/site/content/docs/v1.18/namespace-glob-patterns.md b/site/content/docs/v1.18/namespace-glob-patterns.md new file mode 100644 index 000000000..6a2f9f0e3 --- /dev/null +++ b/site/content/docs/v1.18/namespace-glob-patterns.md @@ -0,0 +1,71 @@ +--- +title: "Namespace Glob Patterns" +layout: docs +--- + +When using `--include-namespaces` and `--exclude-namespaces` flags with backup and restore commands, you can use glob patterns to match multiple namespaces. + +## Supported Patterns + +Velero supports the following glob pattern characters: + +- `*` - Matches any sequence of characters + ```bash + velero backup create my-backup --include-namespaces "app-*" + # Matches: app-prod, app-staging, app-dev, etc. + ``` + +- `?` - Matches exactly one character + ```bash + velero backup create my-backup --include-namespaces "ns?" + # Matches: ns1, ns2, nsa, but NOT ns10 + ``` + +- `[abc]` - Matches any single character in the brackets + ```bash + velero backup create my-backup --include-namespaces "ns[123]" + # Matches: ns1, ns2, ns3 + ``` + +- `[a-z]` - Matches any single character in the range + ```bash + velero backup create my-backup --include-namespaces "ns[a-c]" + # Matches: nsa, nsb, nsc + ``` + +## Unsupported Patterns + +The following patterns are **not supported** and will cause validation errors: + +- `**` - Consecutive asterisks +- `|` - Alternation (regex operator) +- `()` - Grouping (regex operators) +- `!` - Negation +- `{}` - Brace expansion +- `,` - Comma (used in brace expansion) + +## Special Cases + +- `*` alone means "all namespaces" and is not expanded +- Empty brackets `[]` are invalid +- Unmatched or unclosed brackets will cause validation errors + +## Examples + +Combine patterns with include and exclude flags: + +```bash +# Backup all production namespaces except test +velero backup create prod-backup \ + --include-namespaces "*-prod" \ + --exclude-namespaces "test-*" + +# Backup specific numbered namespaces +velero backup create numbered-backup \ + --include-namespaces "app-[0-9]" + +# Restore namespaces matching multiple patterns +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces "frontend-*,backend-*" +``` diff --git a/site/content/docs/v1.18/namespace.md b/site/content/docs/v1.18/namespace.md new file mode 100644 index 000000000..68561e720 --- /dev/null +++ b/site/content/docs/v1.18/namespace.md @@ -0,0 +1,22 @@ +--- +title: "Run in a non-default namespace" +layout: docs +--- + +The Velero installation and backups by default are run in the `velero` namespace. However, it is possible to use a different namespace. + +## Customize the namespace during install + +Use the `--namespace` flag, in conjunction with the other flags in the `velero install` command (as shown in the [the Velero install instructions][0]). This will inform Velero where to install. + +## Customize the namespace for operational commands + +To have namespace consistency, specify the namespace for all Velero operational commands to be the same as the namespace used to install Velero: + +```bash +velero client config set namespace= +``` + +Alternatively, you may use the global `--namespace` flag with any operational command to tell Velero where to run. + +[0]: basic-install.md#install-the-cli diff --git a/site/content/docs/v1.18/node-agent-concurrency.md b/site/content/docs/v1.18/node-agent-concurrency.md new file mode 100644 index 000000000..4d640240d --- /dev/null +++ b/site/content/docs/v1.18/node-agent-concurrency.md @@ -0,0 +1,91 @@ +--- +title: "Node-agent Concurrency" +layout: docs +--- + +Velero node-agent is a daemonset hosting modules to complete the concrete tasks of backups/restores, i.e., file system backup/restore, CSI snapshot data movement. +Varying from the data size, data complexity, resource availability, the tasks may take a long time and remarkable resources (CPU, memory, network bandwidth, etc.). These tasks make the loads of node-agent. + +Node-agent concurrency configurations allow you to configure the concurrent number of node-agent loads per node. When the resources are sufficient in nodes, you can set a large concurrent number, so as to reduce the backup/restore time; otherwise, the concurrency should be reduced, otherwise, the backup/restore may encounter problems, i.e., time lagging, hang or OOM kill. + +To set Node-agent concurrency configurations, a configMap should be created manually. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. The name of the configMap should be specified in the node-agent server parameter ```--node-agent-configmap```. +Node-agent server checks these configurations at startup time. Therefore, you could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --node-agent-configmap=` + +### Global concurrent number +You can specify a concurrent number that will be applied to all nodes if the per-node number is not specified. This number is set through ```globalConfig``` field in ```loadConcurrency```. +The number starts from 1 which means there is no concurrency, only one load is allowed. There is no roof limit. If this number is not specified or not valid, a hard-coded default value will be used, the value is set to 1. + +### Per-node concurrent number +You can specify different concurrent number per node, for example, you can set 3 concurrent instances in Node-1, 2 instances in Node-2 and 1 instance in Node-3. +The range of Per-node concurrent number is the same with Global concurrent number. Per-node concurrent number is preferable to Global concurrent number, so it will overwrite the Global concurrent number for that node. + +Per-node concurrent number is implemented through ```perNodeConfig``` field in ```loadConcurrency```. +```perNodeConfig``` is a list of ```RuledConfigs``` each item of which matches one or more nodes by label selectors and specify the concurrent number for the matched nodes. +Here is an example of the ```perNodeConfig``: +``` +"nodeSelector: kubernetes.io/hostname=node1; number: 3" +"nodeSelector: beta.kubernetes.io/instance-type=Standard_B4ms; number: 5" +``` +The first element means the node with host name ```node1``` gets the Per-node concurrent number of 3. +The second element means all the nodes with label ```beta.kubernetes.io/instance-type``` of value ```Standard_B4ms``` get the Per-node concurrent number of 5. +At least one node is expected to have a label with the specified ```RuledConfigs``` element (rule). If no node is with this label, the Per-node rule makes no effect. +If one node falls into more than one rules, e.g., if node1 also has the label ```beta.kubernetes.io/instance-type=Standard_B4ms```, the smallest number (3) will be used. + +### Sample +A sample of the complete ConfigMap is as below: +```json +{ + "loadConcurrency": { + "globalConfig": 2, + "perNodeConfig": [ + { + "nodeSelector": { + "matchLabels": { + "kubernetes.io/hostname": "node1" + } + }, + "number": 3 + }, + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + } + }, + "number": 5 + } + ] + } +} +``` +To create the ConfigMap, save something like the above sample to a json file and then run below command: +``` +kubectl create cm -n velero --from-file= +``` +To provide the ConfigMap to node-agent, edit the node-agent daemonset and add the ```- --node-agent-configmap``` argument to the spec: +1. Open the node-agent daemonset spec +``` +kubectl edit ds node-agent -n velero +``` +2. Add ```- --node-agent-configmap``` to ```spec.template.spec.containers``` +``` +spec: + template: + spec: + containers: + - args: + - --node-agent-configmap= +``` + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage diff --git a/site/content/docs/v1.18/node-agent-prepare-queue-length.md b/site/content/docs/v1.18/node-agent-prepare-queue-length.md new file mode 100644 index 000000000..b80c36cf5 --- /dev/null +++ b/site/content/docs/v1.18/node-agent-prepare-queue-length.md @@ -0,0 +1,58 @@ +--- +title: "Node-agent Prepare Queue Length" +layout: docs +--- + +During [CSI Snapshot Data Movement][1], Velero built-in data mover launches data mover pods to run the data transfer. +During [fs-backup][2], Velero also launches data mover pods to run the data transfer. +Other intermediate resources may also be created along with the data mover pods, i.e., PVCs, VolumeSnapshots, VolumeSnapshotContents, etc. + +Velero uses [node-agent Concurrency Configuration][3] to control the number of concurrent data transfer activities across the nodes, by default, the concurrency is 1 per node. + +when the parallelism across the available nodes are much lower than the total number of volumes to be backed up/restored, the intermediate objects may exist for much longer time unnecessarily, which takes unnecessary resources from the cluster. +The available nodes are decided by various factors, e.g., node OS type (linux or Windows), [Node Selection][4] (for CSI Snapshot Data Movement only), etc. + +Velero allows you to configure the `prepareQueueLength` in node-agent Configuration, which defines the maximum number of `DataUpload`/`DataDownload`/`PodVolumeBackup`/`PodVolumeRestore` CRs under the preparation statuses but are not yet processed by any node (e.g., in phases of `Accepted`, `Prepared`). In this way, the number of intermediate objects are constrained. + +### Sample +Here is a sample of the configMap with ```prepareQueueLength```: +```json +{ + "prepareQueueLength": 10 +} +``` + +To create the configMap, save something like the above sample to a json file and then run below command: +``` +kubectl create cm node-agent-config -n velero --from-file= +``` + +To provide the configMap to node-agent, edit the node-agent daemonset and add the ```- --node-agent-configmap`` argument to the spec: +1. Open the node-agent daemonset spec +``` +kubectl edit ds node-agent -n velero +``` +2. Add ```- --node-agent-configmap``` to ```spec.template.spec.containers``` +``` +spec: + template: + spec: + containers: + - args: + - --node-agent-configmap= +``` + +## Related Documentation + +- [Node-agent Configuration](supported-configmaps/node-agent-configmap.md) - Complete reference for all configuration options +- [Node-agent Concurrency](node-agent-concurrency.md) - Configure concurrent operations per node +- [Node Selection for Data Movement](data-movement-node-selection.md) - Configure which nodes run data movement +- [Data Movement Pod Resource Configuration](data-movement-pod-resource-configuration.md) - Configure pod resources +- [BackupPVC Configuration](data-movement-backup-pvc-configuration.md) - Configure backup storage +- [RestorePVC Configuration](data-movement-restore-pvc-configuration.md) - Configure restore storage +- [Cache PVC Configuration](data-movement-cache-volume.md) - Configure restore data mover storage + +[1]: csi-snapshot-data-movement.md +[2]: file-system-backup.md +[3]: node-agent-concurrency.md +[4]: data-movement-node-selection.md diff --git a/site/content/docs/v1.18/on-premises.md b/site/content/docs/v1.18/on-premises.md new file mode 100644 index 000000000..88e547973 --- /dev/null +++ b/site/content/docs/v1.18/on-premises.md @@ -0,0 +1,95 @@ +--- +title: "On-Premises Environments" +layout: docs +--- + +You can run Velero in an on-premises cluster in different ways depending on your requirements. + +### Selecting an object storage provider + +You must select an object storage backend that Velero can use to store backup data. [Supported providers][0] contains information on various +options that are supported or have been reported to work by users. + +If you do not already have an object storage system, [MinIO][2] is an open-source S3-compatible object storage system that can be installed on-premises and is compatible with Velero. The details of configuring it for production usage are out of scope for Velero's documentation, but an [evaluation install guide][3] using MinIO is provided for convenience. + +### (Optional) Selecting volume snapshot providers + +If you need to back up persistent volume data, you must select a volume backup solution. [Supported providers][0] contains information on the supported options. + +For example, if you use [Portworx][4] for persistent storage, you can install their Velero plugin to get native Portworx snapshots as part of your Velero backups. + +If there is no native snapshot plugin available for your storage platform, you can use Velero's [File System Backup][1], which provides a platform-agnostic file-level backup solution for volume data. + +### Air-gapped deployments + +In an air-gapped deployment, there is no access to the public internet, and therefore no access to public container registries. + +In these scenarios, you will need to make sure that you have an internal registry, such as [Harbor][5], installed and the Velero core and plugin images loaded into your internal registry. + +Below you will find instructions to downloading the Velero images to your local machine, tagging them, then uploading them to your custom registry. + +#### Preparing the Velero image + +First, download the Velero image, tag it for the your private registry, then upload it into the registry so that it can be pulled by your cluster. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero:$VELERO_VERSION +docker tag velero/velero:$VELERO_VERSION $PRIVATE_REG/velero:$VELERO_VERSION +docker push $PRIVATE_REG/velero:$VELERO_VERSION +``` + +#### Preparing plugin images + +Next, repeat these steps for any plugins you may need. This example will use the AWS plugin, but the plugin name should be replaced with the plugins you will need. + +```bash +PRIVATE_REG= +PLUGIN_VERSION= + +docker pull velero/velero-plugin-for-aws:$PLUGIN_VERSION +docker tag velero/velero-plugin-for-aws:$PLUGIN_VERSION $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +docker push $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +``` + +#### Preparing the restore helper image (optional) + +If you are using File System Backup, you will also need to upload the restore helper image. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero-restore-helper:$VELERO_VERSION +docker tag velero/velero-restore-helper:$VELERO_VERSION $PRIVATE_REG/velero-restore-helper:$VELERO_VERSION +docker push $PRIVATE_REG/velero-restore-helper:$VELERO_VERSION +``` + +#### Pulling specific architecture images (optional) + +Velero uses Docker manifests for its images, allowing Docker to pull the image needed based on your client machine's architecture. + +If you need to pull a specific image, you should replace the `velero/velero` image with the specific architecture image, such as `velero/velero-arm`. + +To see an up-to-date list of architectures, be sure to enable Docker experimental features and use `docker manifest inspect velero/velero` (or whichever image you're interested in), and join the architecture string to the end of the image name with `-`. + +#### Installing Velero + +By default, `velero install` will use the public `velero/velero` image. When using an air-gapped deployment, use your private registry's image for Velero and your private registry's images for any plugins. + +```bash +velero install \ + --image=$PRIVATE_REG/velero:$VELERO_VERSION \ + --plugins=$PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION \ +<....> +``` + + +[0]: supported-providers.md +[1]: file-system-backup.md +[2]: https://min.io +[3]: contributions/minio.md +[4]: https://portworx.com +[5]: https://goharbor.io/ diff --git a/site/content/docs/v1.18/output-file-format.md b/site/content/docs/v1.18/output-file-format.md new file mode 100644 index 000000000..8d4823505 --- /dev/null +++ b/site/content/docs/v1.18/output-file-format.md @@ -0,0 +1,224 @@ +--- +title: "Output file format" +layout: docs +--- + +A backup is a gzip-compressed tar file whose name matches the Backup API resource's `metadata.name` (what is specified during `velero backup create `). + +In cloud object storage, each backup file is stored in its own subdirectory in the bucket specified in the Velero server configuration. This subdirectory includes an additional file called `velero-backup.json`. The JSON file lists all information about your associated Backup resource, including any default values. This gives you a complete historical record of the backup configuration. The JSON file also specifies `status.version`, which corresponds to the output file format. + +The directory structure in your cloud storage looks something like: + +``` +rootBucket/ + backup1234/ + velero-backup.json + backup1234.tar.gz +``` + +## Example backup JSON file + +```json +{ + "kind": "Backup", + "apiVersion": "velero.io/v1", + "metadata": { + "name": "test-backup", + "namespace": "velero", + "selfLink": "/apis/velero.io/v1/namespaces/velero/backups/test-backup", + "uid": "a12345cb-75f5-11e7-b4c2-abcdef123456", + "resourceVersion": "337075", + "creationTimestamp": "2017-07-31T13:39:15Z" + }, + "spec": { + "includedNamespaces": [ + "*" + ], + "excludedNamespaces": null, + "includedResources": [ + "*" + ], + "excludedResources": null, + "labelSelector": null, + "snapshotVolumes": true, + "ttl": "24h0m0s" + }, + "status": { + "version": 1, + "formatVersion": "1.1.0", + "expiration": "2017-08-01T13:39:15Z", + "phase": "Completed", + "volumeBackups": { + "pvc-e1e2d345-7583-11e7-b4c2-abcdef123456": { + "snapshotID": "snap-04b1a8e11dfb33ab0", + "type": "gp2", + "iops": 100 + } + }, + "validationErrors": null + } +} +``` +Note that this file includes detailed info about your volume snapshots in the `status.volumeBackups` field, which can be helpful if you want to manually check them in your cloud provider GUI. + +## Output File Format Versioning + +The Velero output file format is intended to be relatively stable, but may change over time to support new features. + +To accommodate this, Velero follows [Semantic Versioning](http://semver.org/) for the file format version. + +Minor and patch versions will indicate backwards-compatible changes that previous versions of Velero can restore, including new directories or files. + +A major version would indicate that a version of Velero older than the version that created the backup could not restore it, usually because of moved or renamed directories or files. + +Major versions of the file format will be incremented with major version releases of Velero. +However, a major version release of Velero does not necessarily mean that the backup format version changed - Velero 3.0 could still use backup file format 2.0, as an example. + +## Versions + +### File Format Version: 1.1 (Current) + +Version 1.1 added support of API groups versions as part of the backup. Previously, only the preferred version of each API groups was backed up. Each resource has one or more sub-directories: one sub-directory for each supported version of the API group. The preferred version API Group of each resource has the suffix "-preferredversion" as part of the sub-directory name. For backward compatibility, we kept the classic directory structure without the API group version, which sits on the same level as the API group sub-directory versions. + +By default, only the preferred API group of each resource is backed up. To take a backup of all API group versions, you need to run the Velero server with the `--features=EnableAPIGroupVersions` feature flag. This is an experimental flag and the restore logic to handle multiple API group versions is documented at [EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +When unzipped, a typical backup directory (`backup1234.tar.gz`) taken with this file format version looks like the following (with the feature flag): + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + v1-preferredversion/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs.batch/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + horizontalpodautoscalers.autoscaling/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta1/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta2/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + + ... +``` + +### File Format Version: 1 + +When unzipped, a typical backup directory (`backup1234.tar.gz`) looks like the following: + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + ... +``` diff --git a/site/content/docs/v1.18/overview-plugins.md b/site/content/docs/v1.18/overview-plugins.md new file mode 100644 index 000000000..037601d51 --- /dev/null +++ b/site/content/docs/v1.18/overview-plugins.md @@ -0,0 +1,29 @@ +--- +title: "Velero plugin system" +layout: docs +--- + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. + +For server installation, Velero requires that at least one plugin is added (with the `--plugins` flag). The plugin will be either of the type object store or volume snapshotter, or a plugin that contains both. An exception to this is that when the user is not configuring a backup storage location or a snapshot storage location at the time of install, this flag is optional. + +Any plugin can be added after Velero has been installed by using the command `velero plugin add `. + +Example with a dockerhub image: `velero plugin add velero/velero-plugin-for-aws:v1.0.0`. + +In the same way, any plugin can be removed by using the command `velero plugin remove `. + +## Creating a new plugin + +Anyone can add integrations for any platform to provide additional backup and volume storage without modifying the Velero codebase. To write a plugin for a new backup or volume storage platform, take a look at our [example repo][1] and at our documentation for [Custom plugins][2]. + +## Adding a new plugin + +After you publish your plugin on your own repository, open a PR that adds a link to it under the appropriate list of [supported providers][3] page in our documentation. + +You can also add the [`velero-plugin` GitHub Topic][4] to your repo, and it will be shown under the aggregated list of repositories automatically. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example/ +[2]: custom-plugins.md +[3]: supported-providers.md +[4]: https://github.com/topics/velero-plugin diff --git a/site/content/docs/v1.18/performance-guidance.md b/site/content/docs/v1.18/performance-guidance.md new file mode 100644 index 000000000..8596b4a52 --- /dev/null +++ b/site/content/docs/v1.18/performance-guidance.md @@ -0,0 +1,163 @@ +--- +title: "Velero File System Backup Performance Guide" +layout: docs +--- + +When using Velero to do file system backup & restore, Restic uploader or Kopia uploader are both supported now. But the resources used and time consumption are a big difference between them. + +We've done series rounds of tests against Restic uploader and Kopia uploader through Velero, which may give you some guidance. But the test results will vary from different infrastructures, and our tests are limited and couldn't cover a variety of data scenarios, **the test results and analysis are for reference only**. + +## Infrastructure + +Minio is used as Velero backend storage, Network File System (NFS) is used to create the persistent volumes (PVs) and Persistent Volume Claims (PVC) based on the storage. The minio and NFS server are deployed independently in different virtual machines (VM), which with 300 MB/s write throughput and 175 MB/s read throughput representatively. + +The details of environmental information as below: + +``` +### KUBERNETES VERSION +root@velero-host-01:~# kubectl version +Client Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.4" +Server Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.14" + +### DOCKER VERSION +root@velero-host-01:~# docker version +Client: + Version: 20.10.12 + API version: 1.41 + +Server: + Engine: + Version: 20.10.12 + API version: 1.41 (minimum version 1.12) + Go version: go1.16.2 + containerd: + Version: 1.5.9-0ubuntu1~20.04.4 + runc: + Version: 1.1.0-0ubuntu1~20.04.1 + docker-init: + Version: 0.19.0 + +### NODES +root@velero-host-01:~# kubectl get nodes |wc -l +6 // one master with 6 work nodes + +### DISK INFO +root@velero-host-01:~# smartctl -a /dev/sda +smartctl 7.1 2019-12-30 r5022 [x86_64-linux-5.4.0-126-generic] (local build) +Copyright (C) 2002-19, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Vendor: VMware +Product: Virtual disk +Revision: 1.0 +Logical block size: 512 bytes +Rotation Rate: Solid State Device +Device type: disk +### MEMORY INFO +root@velero-host-01:~# free -h + total used free shared buff/cache available +Mem: 3.8Gi 328Mi 3.1Gi 1.0Mi 469Mi 3.3Gi +Swap: 0B 0B 0B + +### CPU INFO +root@velero-host-01:~# cat /proc/cpuinfo | grep name | cut -f2 -d: | uniq -c + 4 Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz + +### SYSTEM INFO +root@velero-host-01:~# cat /proc/version +root@velero-host-01:~# cat /proc/version +Linux version 5.4.0-126-generic (build@lcy02-amd64-072) (gcc version 9.4.0 (Ubuntu 9.4.0-1ubuntu1~20.04.1)) #142-Ubuntu SMP Fri Aug 26 12:12:57 UTC 2022 + +### VELERO VERSION +root@velero-host-01:~# velero version +Client: + Version: main ###v1.10 pre-release version + Git commit: 9b22ca6100646523876b18a491d881561b4dbcf3-dirty +Server: + Version: main ###v1.10 pre-release version +``` + +## Test + +Below we've done 6 groups of tests, for each single group of test, we used limited resources (1 core CPU 2 GB memory or 4 cores CPU 4 GB memory) to do Velero file system backup under Restic path and Kopia path, and then compare the results. + +Recorded the metrics of time consumption, maximum CPU usage, maximum memory usage, and minio storage usage for node-agent daemonset, and the metrics of Velero deployment are not included since the differences are not obvious by whether using Restic uploader or Kopia uploader. + +Compression is either disabled or not unavailable for both uploader. + +### Case 1: 4194304(4M) files, 2396745(2M) directories, 0B per file total 0B content +#### result: +|Uploader| Resources|Times |Max CPU|Max Memory|Repo Usage| +|--------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c2g |24m54s| 65% |1530 MB |80 MB | +| Restic | 1c2g |52m31s| 55% |1708 MB |3.3 GB | +| Kopia | 4c4g |24m52s| 63% |2216 MB |80 MB | +| Restic | 4c4g |52m28s| 54% |2329 MB |3.3 GB | +#### conclusion: +- The memory usage is larger than Velero's default memory limit (1GB) for both Kopia and Restic under massive empty files. +- For both using Kopia uploader and Restic uploader, there is no significant time reduction by increasing resources from 1c2g to 4c4g. +- Restic uploader is one more time slower than Kopia uploader under the same specification resources. +- Restic has an **irrational** repository size (3.3GB) + +### Case 2: Using the same size (100B) of file and default Velero's resource configuration, the testing quantity of files from 20 thousand to 2 million, these groups of cases mainly test the behavior with the increasing quantity of files. + +### Case 2.1: 235298(23K) files, 137257 (10k)directories, 100B per file total 22.440MB content +#### result: +| Uploader | Resources|Times |Max CPU|Max Memory|Repo Usage| +|-------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c1g |2m34s | 70% |692 MB |108 MB | +| Restic| 1c1g |3m9s | 54% |714 MB |275 MB | + +### Case 2.2 470596(40k) files, 137257 (10k)directories, 100B per file total 44.880MB content +#### result: +| Uploader | Resources|Times |Max CPU|Max Memory|Repo Usage| +|-------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c1g |3m45s | 68% |831 MB |108 MB | +| Restic| 1c1g |4m53s | 57% |788 MB |275 MB | + +### Case 2.3 705894(70k) files, 137257(10k) directories, 100B per file total 67.319MB content +#### result: +|Uploader| Resources|Times |Max CPU|Max Memory|Repo Usage| +|--------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c1g |5m06s | 71% |861 MB |108 MB | +| Restic | 1c1g |6m23s | 56% |810 MB |275 MB | + +### Case 2.4 2097152(2M) files, 2396745(2M) directories, 100B per file total 200.000MB content +#### result: +|Uploader| Resources|Times |Max CPU|Max Memory|Repo Usage| +|--------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c1g |OOM | 74% |N/A |N/A | +| Restic | 1c1g |41m47s| 52% |904 MB |3.2 GB | +#### conclusion: +- With the increasing number of files, there is no memory abnormal surge, the memory usage for both Kopia uploader and Restic uploader is linear increasing, until exceeds 1GB memory usage in Case 2.4 Kopia uploader OOM happened. +- Kopia uploader gets increasingly faster along with the increasing number of files. +- Restic uploader repository size is still much larger than Kopia uploader repository. + +### Case 3: 10625(10k) files, 781 directories, 1.000MB per file total 10.376GB content +#### result: +|Uploader| Resources|Times |Max CPU|Max Memory|Repo Usage| +|--------|----------|:----:|------:|:--------:|:--------:| +| Kopia | 1c2g |1m37s | 75% |251 MB |10 GB | +| Restic | 1c2g |5m25s | 100% |153 MB |10 GB | +| Kopia | 4c4g |1m35s | 75% |248 MB |10 GB | +| Restic | 4c4g |3m17s | 171% |126 MB |10 GB | +#### conclusion: +- This case involves a relatively large backup size, there is no significant time reduction by increasing resources from 1c2g to 4c4g for Kopia uploader, but for Restic uploader when increasing CPU from 1 core to 4, backup time-consuming was shortened by one-third, which means in this scenario should allocate more CPU resources for Restic uploader. +- For the large backup size case, Restic uploader's repository size comes to normal + +### Case 4: 900 files, 1 directory, 1.000GB per file total 900.000GB content +#### result: +|Uploader| Resources|Times |Max CPU|Max Memory|Repo Usage| +|--------|----------|:-----:|------:|:--------:|:--------:| +| Kopia | 1c2g |2h30m | 100% |714 MB |900 GB | +| Restic | 1c2g |Timeout| 100% |416 MB |N/A | +| Kopia | 4c4g |1h42m | 138% |786 MB |900 GB | +| Restic | 4c4g |2h15m | 351% |606 MB |900 GB | +#### conclusion: +- When the target backup data is relatively large, Restic uploader starts to Timeout under 1c2g. So it's better to allocate more memory for Restic uploader when backup large sizes of data. +- For backup large amounts of data, Kopia uploader is both less time-consuming and less resource usage. + +## Summary +- With the same specification resources, Kopia uploader is less time-consuming when backup. +- Performance would be better if choosing Kopia uploader for the scenario in backup large mounts of data or massive small files. +- It's better to set one reasonable resource configuration instead of the default depending on your scenario. For default resource configuration, it's easy to be timeout with Restic uploader in backup large amounts of data, and it's easy to be OOM for both Kopia uploader and Restic uploader in backup of massive small files. \ No newline at end of file diff --git a/site/content/docs/v1.18/plugin-release-instructions.md b/site/content/docs/v1.18/plugin-release-instructions.md new file mode 100644 index 000000000..4323d8c9a --- /dev/null +++ b/site/content/docs/v1.18/plugin-release-instructions.md @@ -0,0 +1,30 @@ +--- +title: Releasing Velero plugins +layout: docs +toc: "true" +--- + +Velero plugins maintained by the core maintainers do not have any shipped binaries, only container images, so there is no need to invoke a GoReleaser script. +Container images are built via a CI job on git push. + +Plugins the Velero core team is responsible include all those listed in [the Velero-supported providers list](supported-providers.md) _except_ the vSphere plugin. + + +## Steps +### Open a PR to prepare the repo +1. Update the README.md file to update the compatibility matrix and `velero install` instructions with the expected version number and open a PR. +1. Determining the version number is based on semantic versioning and whether the plugin uses any newly introduced, changed, or removed methods or variables from Velero. +2. Roll all unreleased changelogs into a new `CHANGELOG-v.md` file and delete the content of the `unreleased` folder. Edit the new changelog file as needed. +### Tag +1. Once the PR is merged, checkout the upstream `main` branch. Your local upstream might be named `upstream` or `origin`, so use this command: `git checkout /main`. +1. Tag the git version - `git tag v`. +1. Push the git tag - `git push --tags ` to trigger the image build. +2. Wait for the container images to build. You may check the progress of the GH action that triggers the image build at `https://github.com/vmware-tanzu//actions` +3. Verify that an image with the new tag is available at `https://hub.docker.com/repository/docker/velero//`. +4. Run the Velero [e2e tests][2] using the new image. Until it is made configurable, you will have to edit the [plugin version][1] in the test. +### Release +1. If all e2e tests pass, go to the GitHub release page of the plugin (`https://github.com/vmware-tanzu//releases`) and manually create a release for the new tag. +1. Copy and paste the content of the new changelog file into the release description field. + +[1]: https://github.com/vmware-tanzu/velero/blob/c8dfd648bbe85db0184ea53296de4220895497e6/test/e2e/velero_utils.go#L27 +[2]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.18/proxy.md b/site/content/docs/v1.18/proxy.md new file mode 100644 index 000000000..2c379c9b8 --- /dev/null +++ b/site/content/docs/v1.18/proxy.md @@ -0,0 +1,103 @@ +--- +title: "Behind Proxy" +layout: docs +toc: "true" +--- + +This document explains how to make Velero work behind proxy. +The procedures described in this document are concluded from the scenario that Velero is deployed behind proxy, and Velero needs to connect to a public MinIO server as storage location. Maybe other scenarios' configurations are not exactly the same, but basically they should share most parts. + +## Set the proxy server address +Specify the proxy server address by environment variables in Velero deployment and node-agent DaemonSet. +Take the following as an example: +``` yaml + ... + spec: + containers: + - args: + - server + - --features=EnableCSI + command: + - /velero + env: + ... + - name: HTTP_PROXY + value: + - name: HTTPS_PROXY + value: + # In case not all destinations that Velero connects to need go through proxy, users can specify the NO_PROXY to bypass proxy. + - name: NO_PROXY + value: +``` + +## Set the proxy required certificates +In some cases, the proxy requires certificate to connect. You can provide certificates in the BSL configuration. +It's possible that the object storage also requires certificate, then include both certificates together. + +### Method 1: Using Kubernetes Secrets (Recommended) + +The recommended approach is to store certificates in a Kubernetes Secret and reference them using `caCertRef`: + +1. Create a file containing all required certificates: + + ``` bash + cat certs + -----BEGIN CERTIFICATE----- + certificates first content + -----END CERTIFICATE----- + + -----BEGIN CERTIFICATE----- + certificates second content + -----END CERTIFICATE----- + ``` + +2. Create a Secret from the certificate file: + + ``` bash + kubectl create secret generic proxy-ca-certs \ + --from-file=ca-bundle.crt=certs \ + -n velero + ``` + +3. Reference the Secret in your BackupStorageLocation: + +``` yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + provider: + default: true + objectStorage: + bucket: velero + caCertRef: + name: proxy-ca-certs + key: ca-bundle.crt + # ... other configuration +``` + +### Method 2: Using inline certificates (Deprecated) + +**Note:** The `caCert` field is deprecated. Use `caCertRef` for better security and management. + +If you must use the inline method, encode the certificate content with base64: + +``` bash +cat certs | base64 +LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlcyBmaXJzdCBjb250ZW50Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpjZXJ0aWZpY2F0ZXMgc2Vjb25kIGNvbnRlbnQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +``` + +``` yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +# ... +spec: + # ... + default: true + objectStorage: + bucket: velero + caCert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlcyBmaXJzdCBjb250ZW50Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpjZXJ0aWZpY2F0ZXMgc2Vjb25kIGNvbnRlbnQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + # ... +``` diff --git a/site/content/docs/v1.18/rbac.md b/site/content/docs/v1.18/rbac.md new file mode 100644 index 000000000..a3e31be87 --- /dev/null +++ b/site/content/docs/v1.18/rbac.md @@ -0,0 +1,95 @@ +--- +title: "Run Velero more securely with restrictive RBAC settings" +layout: docs +--- + +By default Velero runs with an RBAC policy of ClusterRole `cluster-admin`. This is to make sure that Velero can back up or restore anything in your cluster. But `cluster-admin` access is wide open -- it gives Velero components access to everything in your cluster. Depending on your environment and your security needs, you should consider whether to configure additional RBAC policies with more restrictive access. + +**Note:** Roles and RoleBindings are associated with a single namespaces, not with an entire cluster. PersistentVolume backups are associated only with an entire cluster. This means that any backups or restores that use a restrictive Role and RoleBinding pair can manage only the resources that belong to the namespace. You do not need a wide open RBAC policy to manage PersistentVolumes, however. You can configure a ClusterRole and ClusterRoleBinding that allow backups and restores only of PersistentVolumes, not of all objects in the cluster. + +For more information about RBAC and access control generally in Kubernetes, see the Kubernetes documentation about [access control][1], [managing service accounts][2], and [RBAC authorization][3]. + +## Set up with restricted RBAC permissions + +Here's a sample of restricted permission setting. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: YOUR_NAMESPACE_HERE + name: ROLE_NAME_HERE + labels: + component: velero +rules: + - apiGroups: + - velero.io + verbs: + - "*" + resources: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ROLEBINDING_NAME_HERE + namespace: YOUR_NAMESPACE_HERE +subjects: + - kind: ServiceAccount + name: YOUR_SERVICEACCOUNT_HERE +roleRef: + kind: Role + name: ROLE_NAME_HERE + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: velero-clusterrole +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - namespaces + verbs: + - '*' +- apiGroups: + - '*' + resources: + - '*' + verbs: + - list +- apiGroups: + - 'apiextensions.k8s.io' + resources: + - 'customresourcedefinitions' + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: velero-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: velero-clusterrole +subjects: + - kind: ServiceAccount + name: YOUR_SERVICEACCOUNT_HERE + namespace: YOUR_NAMESPACE_HERE +``` + +You can add more permissions into the `Role` setting according to the need. +`velero-clusterrole` ClusterRole is verified to work in most cases. +`Namespaces` resource permission is needed to create namespace during restore. If you don't need that, the `create` permission can be removed, but `list` and `get` permissions of `Namespaces` resource is still needed, because Velero needs to know whether the namespace it's assigned exists in the cluster. +`PersistentVolumes` resource permission is needed for back up and restore volumes. If that is not needed, it can be removed too. +`CustomResourceDefinitions` resource permission is needed to backup CR instances' CRD. It's better to keep them. +It's better to have the `list` permission for all resources, because Velero needs to read some resources during backup, for example, `ClusterRoles` is listed for backing `ServiceAccount` up, and `VolumeSnapshotContent` for CSI `PersistentVolumeClaim`. If you just enable `list` permissions for the resources you want to back up and restore, it's possible that backup or restore end with failure. + +[1]: https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/ +[2]: https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/ +[3]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +[4]: namespace.md + diff --git a/site/content/docs/v1.18/release-instructions.md b/site/content/docs/v1.18/release-instructions.md new file mode 100644 index 000000000..d7f78739e --- /dev/null +++ b/site/content/docs/v1.18/release-instructions.md @@ -0,0 +1,179 @@ +--- +title: "Release Instructions" +layout: docs +toc: "true" +--- +This page covers the steps to perform when releasing a new version of Velero. + +## General notes +- Please read the documented variables in each script to understand what they are for and how to properly format their values. +- You will need to have an upstream remote configured to use to the [vmware-tanzu/velero](https://github.com/vmware-tanzu/velero) repository. + You can check this using `git remote -v`. + The release script ([`tag-release.sh`](https://github.com/vmware-tanzu/velero/blob/v1.18.0/hack/release-tools/tag-release.sh)) will use `upstream` as the default remote name if it is not specified using the environment variable `REMOTE`. +- GA release: major and minor releases only. Example: 1.0 (major), 1.5 (minor). +- Pre-releases: Any release leading up to a GA. Example: 1.4.0-beta.1, 1.5.0-rc.1 +- RC releases: Release Candidate, contains everything that is supposed to ship with the GA release. This is still a pre-release. + +## Velero Release Requirements + +Velero is on a "train leaves the station" model for releases. We will generate a release candidate (RC) +at the scheduled time. Multiple release candidates may be generated, depending on if bugs are found during testing. +When testing has passed a release build will be generated. + +### Release Candidate criteria +The release candidate commit must meet the following criteria: + +* No major bugs outstanding +* Unit tests pass +* E2E tests against latest Kubernetes on AWS, vSphere and kind pass + +Once the release has moved to RC, a code freeze is in effect. Only changes needed to release are allowable. + +### Release criteria +In order for a release candidate to be released, it must meet the following criteria: + +* Unit tests pass +* E2E tests against latest K8S and earliest supported K8S on Azure, vSphere, Kind, AWS, GCP +* Manual tests pass (manual tests will be converted to automated testing) + +When bugs are identified by any of these methods, we will determine whether the bug is a release blocker or not and +a fix generated if it is. When release blocker bugs identifies in an release candidate are fixed, another RC will +be generated and the test cycle will restart. + +## Preparing + +### Create release blog post (GA only) +For each major or minor release, create and publish a blog post to let folks know what's new. Please follow these [instructions](how-to-write-and-release-a-blog-post). + +### Changelog and Docs PR +#### Troubleshooting +- If you encounter the error `You don't have enough free space in /var/cache/apt/archives/` when running `make serve-docs`: run `docker system prune`. + +#### Steps +1. If it doesn't already exist: in a branch, create the file `changelogs/CHANGELOG-..md` by copying the most recent one. +1. Update the file `changelogs/CHANGELOG-..md` + - Run `make changelog` to generate a list of all unreleased changes. + - Copy/paste the output into `CHANGELOG-..md`, under the "All Changes" section for the release. + - You *may* choose to tweak formatting on the list of changes by adding code blocks, etc. + - Update links at the top of the file to point to the new release version +1. Update the main `CHANGELOG.md` file to properly reference the release-specific changelog file + - Under "Current release": + - Should contain only the current GA release. + - Under "Development release": + - Should contain only the latest pre-release + - Move any prior pre-release into "Older releases" +1. GA Only: Remove all changelog files from `changelogs/unreleased`. +1. Generate new docs + - Run `make gen-docs`, passing the appropriate variables. Examples: + a) `VELERO_VERSION=v1.5.0-rc.1 NEW_DOCS_VERSION=v1.5.0-rc.1 make gen-docs`. + b) `VELERO_VERSION=v1.5.0 NEW_DOCS_VERSION=v1.5 make gen-docs`). + - Note: + - `PREVIOUS_DOCS_VERSION=` is optional; when not set, it will default to the latest doc version. + - `VELERO_VERSION` and `NEW_DOCS_VERSION` are slightly different, the `VELERO_VERSION` may have lots of small release versions for one specific $major.minor, such as 'v1.5.0' and 'v1.5.1', but `NEW_DOCS_VERSION` may still be 'v1.5' for not document update. +1. Clean up when there is an existing set of pre-release versioned docs for the version you are releasing + - Example: `site/content/docs/v1.5.0-beta.1` exists, and you're releasing `v1.5.0-rc.1` or `v1.5` + - Remove the directory containing the pre-release docs, i.e. `site/content/docs/`. + - Delete the pre-release docs table of contents file, i.e. `site/data/docs/-toc.yml`. + - Remove the pre-release docs table of contents mapping entry from `site/data/toc-mapping.yml`. + - Remove all references to the pre-release docs from `site/config.yml`. +1. Create the "Upgrade to $major.minor" page if it does not already exist ([example](https://velero.io/docs/v1.5/upgrade-to-1.5/)). + If it already exists, update any usage of the previous version string within this file to use the new version string instead ([example](https://github.com/vmware-tanzu/velero/pull/2941/files#diff-d594f8fd0901fed79c39aab4b348193d)). + This needs to be done in both the versioned and the `main` folders. +1. Review and submit PR + - Follow the additional instructions at `site/README-HUGO.md` to complete the docs generation process. + - Do a review of the diffs, and/or run `make serve-docs` and review the site. + - Submit a PR containing the changelog and the version-tagged docs. + +### Pin the base image +The image of velero is built based on [Distroless docker image](https://github.com/GoogleContainerTools/distroless). +For the reproducibility of the release, before the release candidate is tagged, we need to make sure the in the Dockerfile +on the release branch, the base image is referenced by digest, such as +https://github.com/vmware-tanzu/velero/blob/release-1.7/Dockerfile#L53-L54 + +## Velero release +### Notes +- Pre-requisite: PR with the changelog and docs is merged, so that it's included in the release tag. +- This process is the same for both pre-release and GA. +- Refer to the [General notes](general-notes) above for instructions. + +#### Troubleshooting +- If the dry-run fails with random errors, try running it again. + +#### Steps +1. Manually create the release branch on Github, in the form like `release-$major.$minor` +1. Create a tagged release in dry-run mode + - This won't push anything to GitHub. + - Run `VELERO_VERSION=v1.9.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ON_RELEASE_BRANCH=TRUE ./hack/release-tools/tag-release.sh`. + - Fix any issue. +1. Create a tagged release and push it to GitHub + - Run `VELERO_VERSION=v1.9.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ON_RELEASE_BRANCH=TRUE ./hack/release-tools/tag-release.sh publish`. +1. Publish the release + - Navigate to the draft GitHub release at https://github.com/vmware-tanzu/velero/releases and edit the release. + - If this is a patch release (e.g. `v1.9.1`), note that the full `CHANGELOG-1.9.md` contents will be included in the body of the GitHub release. You need to delete the previous releases' content (e.g. `v1.9.0`'s changelog) so that only the latest patch release's changelog shows. + - Do a quick review for formatting. + - **Note:** the `goreleaser` process should have detected if it's a pre-release version and, if so, checked the box at the bottom of the GitHub release page appropriately, but it's always worth double-checking. + - Verify that GitHub has built and pushed all the images (it takes a while): https://github.com/vmware-tanzu/velero/actions + - Verify that the images are on Docker Hub: https://hub.docker.com/r/velero/velero/tags + - Verify that the assets were published to the GitHub release + - Publish the release. +1. Test the release + - By now, the Docker images should have been published. + - Perform a smoke-test - for example: + - Download the CLI from the GitHub release + - Use it to install Velero into a cluster (or manually update an existing deployment to use the new images) + - Verify that `velero version` shows the expected output + - Run a backup/restore and ensure it works + +## Homebrew release (GA only) +These are the steps to update the Velero Homebrew version. + +### Steps +- If you don't already have one, create a [GitHub access token for Homebrew](https://github.com/settings/tokens/new?scopes=gist,public_repo&description=Homebrew) +- Run `export HOMEBREW_GITHUB_API_TOKEN=your_token_here` on your command line to make sure that `brew` can work on GitHub on your behalf. +- Run `hack/release-tools/brew-update.sh`. This script will download the necessary files, do the checks, and invoke the brew helper to submit the PR, which will open in your browser. +- Update Windows Chocolatey version. From a Windows computer, follow the step-by-step instructions to [create the Windows Chocolatey package for Velero CLI](https://github.com/adamrushuk/velero-choco/blob/main/README.md). Please update the `tools\chocolateyinstall.ps1` file content according to [the existing Velero chocolatey package install script file](https://community.chocolatey.org/packages/velero#files). The current Velero chocolatey package maintainer is [Adam Rush](https://github.com/adamrushuk). It's possible others don't have permission to upload the new version. If so, please contact [Adam Rush](https://github.com/adamrushuk) for help. + +## Plugins + +To release plugins maintained by the Velero team, follow the [plugin release instructions](plugin-release-instructions.md). + +After the plugin images are built, be sure to update any [e2e tests][3] that use these plugins. + +## Helm Chart (GA only) + +### Steps +- Update the CRDs under helm chart folder `crds` according to the current Velero GA version, and add the labels for the helm chart CRDs. For example: https://github.com/vmware-tanzu/helm-charts/pull/248. +- Bump the Chart version `version` on the `Chart.yaml`. +- Bump the Velero version `appVersion` on the `Chart.yaml` file and `tag` on the `values.yaml` file. +- Bump the plugin version on the `values.yaml` if needed. +- Update the _upgrade_ instruction and related tag on the `README.md` file. + +## How to write and release a blog post +What to include in a release blog: +* Thank all contributors for their involvement in the release. + * Where possible shoutout folks by name or consider spotlighting new maintainers. +* Highlight the themes, or areas of focus, for the release. Some examples of themes are security, bug fixes, feature improvements. See past Velero [release blog posts][1] for more examples. +* Include summaries of new features or workflows introduced in a release. + * This can also include new project initiatives, like a code-of-conduct update. + * Consider creating additional blog posts that go through new features in more detail. Plan to publish additional blogs after the release blog (all blogs don’t have to be publish all at once). + +Release blog post PR: +* Prepare a PR containing the release blog post. Read the [website guidelines][2] for more information on creating a blog post. It's usually easiest to make a copy of the most recent existing post, then replace the content as appropriate. +* You also need to update `site/content/_index.md` to have "Latest Release Information" contain a link to the new post. +* Plan to publish the blog post the same day as the release. + +## Announce a release +Once you are finished doing the release, let the rest of the world know it's available by posting messages in the following places. +1. GA Only: Merge the blog post PR. +1. Velero's Twitter account. Maintainers are encouraged to help spread the word by posting or reposting on social media. +1. Community Slack channel. +1. Google group message. + +What to include: +* Thank all contributors +* A brief list of highlights in the release +* Link to the release blog post, release notes, and/or github release page + +[1]: https://velero.io/blog +[2]: website-guidelines.md +[3]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.18/release-schedule.md b/site/content/docs/v1.18/release-schedule.md new file mode 100644 index 000000000..7c7ce83b8 --- /dev/null +++ b/site/content/docs/v1.18/release-schedule.md @@ -0,0 +1,15 @@ +--- +title: "Release Schedule" +layout: docs +toc: "true" +--- + +Definitions borrowed from [the Kubernetes release process document](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-release/release.md#definitions) + +General phases for a Velero release +- Enhancement/Design freeze +- Implementation phase +- Feature freeze & pruning +- Code freeze & prerelease +- Release +- Velero plugin repositories would be bumped up one minor version when new Velero is released diff --git a/site/content/docs/v1.18/repository-maintenance.md b/site/content/docs/v1.18/repository-maintenance.md new file mode 100644 index 000000000..5ca4f3a8a --- /dev/null +++ b/site/content/docs/v1.18/repository-maintenance.md @@ -0,0 +1,200 @@ +--- +title: "Repository Maintenance" +layout: docs +--- + +From v1.14 on, Velero decouples repository maintenance from the Velero server by launching a k8s job to do maintenance when needed, to mitigate the impact on the Velero server during backups. + +Before v1.14.0, Velero performs periodic maintenance on the repository within Velero server pod, this operation may consume significant CPU and memory resources in some cases, leading to Velero server being killed by OOM. Now Velero will launch independent k8s jobs to do the maintenance in Velero installation namespace. + +For repository maintenance jobs, there's no limit on resources by default. You could configure the job resource limitation based on target data to be backed up. + +From v1.15 and on, Velero introduces a new ConfigMap, specified by `velero server --repo-maintenance-job-configmap` parameter, to set repository maintenance Job configuration, including Node Affinity and resources. The old `velero server` parameters ( `--maintenance-job-cpu-request`, `--maintenance-job-mem-request`, `--maintenance-job-cpu-limit`, `--maintenance-job-mem-limit`, and `--keep-latest-maintenance-jobs`) introduced in v1.14 are deprecated, and will be deleted in v1.17. + +The users can specify the ConfigMap name during velero installation by CLI: +`velero install --repo-maintenance-job-configmap=` + +## Settings +### Resource Limitation and Node Affinity +Those are specified by the ConfigMap specified by `velero server --repo-maintenance-job-configmap` parameter. + +This ConfigMap content is a Map. +If there is a key value as `global` in the map, the key's value is applied to all BackupRepositories maintenance jobs that cannot find their own specific configuration in the ConfigMap. +The other keys in the map is the combination of three elements of a BackupRepository, because those three keys can identify a unique BackupRepository: +* The namespace in which BackupRepository backs up volume data. +* The BackupRepository referenced BackupStorageLocation's name. +* The BackupRepository's type. Possible values are `kopia` and `restic`. + +If there is a key match with BackupRepository, the key's value is applied to the BackupRepository's maintenance jobs. +By this way, it's possible to let user configure before the BackupRepository is created. +This is especially convenient for administrator configuring during the Velero installation. +For example, the following BackupRepository's key should be `test-default-kopia`. + +``` yaml +- apiVersion: velero.io/v1 + kind: BackupRepository + metadata: + generateName: test-default-kopia- + labels: + velero.io/repository-type: kopia + velero.io/storage-location: default + velero.io/volume-namespace: test + name: test-default-kopia-kgt6n + namespace: velero + spec: + backupStorageLocation: default + maintenanceFrequency: 1h0m0s + repositoryType: kopia + resticIdentifier: gs:jxun:/restic/test + volumeNamespace: test +``` + +You can still customize the maintenance job resource requests and limit when using the [velero install][1] CLI command. + +The `LoadAffinity` structure is reused from design [node-agent affinity configuration][2]. + +### Affinity Example +It's possible that the users want to choose nodes that match condition A or condition B to run the job. +For example, the user want to let the nodes is in a specified machine type or the nodes locate in the us-central1-x zones to run the job. +This can be done by adding multiple entries in the `LoadAffinity` array. + +The sample of the ```repo-maintenance-job-configmap``` ConfigMap for the above scenario is as below: +``` bash +cat < repo-maintenance-job-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: repo-maintenance-job-config + namespace: velero +data: + global: | + { + "podResources": { + "cpuRequest": "100m", + "cpuLimit": "200m", + "memoryRequest": "100Mi", + "memoryLimit": "200Mi" + }, + "keepLatestMaintenanceJobs": 1, + "loadAffinity": [ + { + "nodeSelector": { + "matchExpressions": [ + { + "key": "topology.kubernetes.io/zone", + "operator": "In", + "values": [ + "us-central1-a", + "us-central1-b", + "us-central1-c" + ] + } + ] + } + } + ] + } + kibishii-default-kopia: | + { + "podResources": { + "cpuRequest": "200m", + "cpuLimit": "400m", + "memoryRequest": "200Mi", + "memoryLimit": "400Mi" + }, + "keepLatestMaintenanceJobs": 2 + } +EOF +``` +Notice: although loadAffinity is an array, Velero only takes the first element of the array. + +This sample showcases how to use affinity configuration: +- matchLabels: maintenance job runs on nodes located in `us-central1-a`, `us-central1-b` and `us-central1-c`. + +To create the configMap, users need to save something like the above sample to a json file and then run below command: +``` +kubectl apply -f repo-maintenance-job-config.yaml +``` + +### Log +Maintenance job inherits the log level and log format settings from the Velero server, so if the Velero server enabled the debug log, the maintenance job will also open the debug level log. + +### Num of Keeping Latest Maintenance Jobs +Velero will keep one specific number of the latest maintenance jobs for each repository. By default, we only keep 3 latest maintenance jobs for each repository, and Velero support configures this setting by the below command when Velero installs: + +```bash +velero install --keep-latest-maintenance-jobs +``` + +### Default Repository Maintenance Frequency +The frequency of running maintenance jobs could be set by the below command when Velero is installed: +```bash +velero install --default-repo-maintain-frequency +``` +For Kopia the default maintenance frequency is 1 hour, and Restic is 7 * 24 hours. + +### Full Maintenance Interval customization +See [backup repository configuration][3] + +### Maintenance History +You can view the maintenance history by describing the corresponding backupRepository CR: + +``` +Status: + Last Maintenance Time: + Recent Maintenance: + Complete Timestamp: + Result: Succeeded + Start Timestamp: + Complete Timestamp: + Result: Succeeded + Start Timestamp: + Message: + Result: Failed + Start Timestamp: +``` + +- `Last Maintenance Time` indicates the time of the latest successful maintenance job +- `Recent Maintenance` keeps the status of the recent 3 maintenance jobs, including its start time, result (succeeded/failed), completion time (if the maintenance job succeeded), or error message (if the maintenance failed) + +### Others +Maintenance jobs will inherit toleration, nodeSelector, service account, image, environment variables, cloud-credentials, priorityClassName etc. from Velero deployment. + +For labels and annotations, maintenance jobs do NOT inherit all labels and annotations from the Velero deployment. Instead, they include: + +**Labels:** + +* `velero.io/repo-name: ` - automatically added to identify which repository they are maintaining +* Only specific [third-party labels][4] from the Velero server deployment that are in the predefined list, currently limited to: + * `azure.workload.identity/use` + +**Annotations:** + +* Only specific [third-party annotations][5] from the Velero server deployment that are in the predefined list, currently limited to: + * `iam.amazonaws.com/role` + +**Important:** Other labels and annotations from the Velero deployment are NOT inherited by maintenance jobs. This is by design to ensure only specific labels and annotations required for cloud provider identity systems are propagated. +Maintenance jobs will not run for backup repositories whose backup storage location is set as readOnly. + +#### Priority Class Configuration +Maintenance jobs can be configured with a specific priority class through the repository maintenance job ConfigMap. The priority class name should be specified in the global configuration section: + +```json +{ + "global": { + "priorityClassName": "low-priority", + "podResources": { + "cpuRequest": "100m", + "memoryRequest": "128Mi" + } + } +} +``` + +Note that priority class configuration is only read from the global configuration section, ensuring all maintenance jobs use the same priority class regardless of which repository they are maintaining. + +[1]: velero-install.md#usage +[2]: node-agent-concurrency.md +[3]: backup-repository-configuration.md#full-maintenance-interval-customization +[4]: https://github.com/vmware-tanzu/velero/blob/d5a2e7e6b9512e8ba52ec269ed5ce9a0fa23548c/pkg/util/third_party.go#L19-L21 +[5]: https://github.com/vmware-tanzu/velero/blob/d5a2e7e6b9512e8ba52ec269ed5ce9a0fa23548c/pkg/util/third_party.go#L23-L25 diff --git a/site/content/docs/v1.18/resource-filtering.md b/site/content/docs/v1.18/resource-filtering.md new file mode 100644 index 000000000..69b2cb5e1 --- /dev/null +++ b/site/content/docs/v1.18/resource-filtering.md @@ -0,0 +1,633 @@ +--- +title: "Resource filtering" +layout: docs +--- + +*Filter objects by namespace, type, labels or resource policies.* + +This page describes how to filter resource for backup and restore. +User could use the include and exclude flags with the `velero backup` and `velero restore` commands. And user could also use resource policies to handle backup. +By default, Velero includes all objects in a backup or restore when no filtering options are used. + +## Includes + +Only specific resources are included, all others are excluded. + +Wildcard takes precedence when both a wildcard and specific resource are included. + +### --include-namespaces + +Namespaces to include. Accepts glob patterns (`*`, `?`, `[abc]`). Default is `*`, all namespaces. + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. + +Note: `*` alone is reserved for empty fields, which means all namespaces. + +* Backup a namespace and it's objects. + + ```bash + velero backup create --include-namespaces + ``` + +* Restore two namespaces and their objects. + + ```bash + velero restore create --include-namespaces , + ``` + +### --include-resources + +Kubernetes resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io (use `*` for all resources). Cannot work with `--include-cluster-scoped-resources`, `--exclude-cluster-scoped-resources`, `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources`. + +* Backup all deployments in the cluster. + + ```bash + velero backup create --include-resources deployments + ``` + +* Restore all deployments and configmaps in the cluster. + + ```bash + velero restore create --include-resources deployments,configmaps + ``` + +* Backup the deployments in a namespace. + + ```bash + velero backup create --include-resources deployments --include-namespaces + ``` + +### --include-cluster-resources + +Includes cluster-scoped resources. Cannot work with `--include-cluster-scoped-resources`, `--exclude-cluster-scoped-resources`, `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources`. This option can have three possible values: + +* `true`: all cluster-scoped resources are included. + +* `false`: no cluster-scoped resources are included. + +* `nil` ("auto" or not supplied): + + - Cluster-scoped resources are included when backing up or restoring all namespaces. Default: `true`. + + - Cluster-scoped resources are not included when namespace filtering is used. Default: `false`. + + * Some related cluster-scoped resources may still be backed/restored up if triggered by a custom action (for example, PVC->PV) unless `--include-cluster-resources=false`. + +* Backup entire cluster including cluster-scoped resources. + + ```bash + velero backup create + ``` + +* Restore only namespaced resources in the cluster. + + ```bash + velero restore create --include-cluster-resources=false + ``` + +* Backup a namespace and include cluster-scoped resources. + + ```bash + velero backup create --include-namespaces --include-cluster-resources=true + ``` + +### --selector + +* Include resources matching the label selector. + + ```bash + velero backup create --selector = + ``` +* Include resources that are not matching the selector + ```bash + velero backup create --selector " notin ()" + ``` + +For more information read the [Kubernetes label selector documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) + +### --or-selector + +To include the resources that match at least one of the label selectors from the list. Separate the selectors with ` or `. The ` or ` is used as a separator to split label selectors, and it is not an operator. + +This option cannot be used together with `--selector`. + +* Include resources matching any one of the label selector, `foo=bar` or `baz=qux` + + ```bash + velero backup create backup1 --or-selector "foo=bar or baz=qux" + ``` + +* Include resources that are labeled `environment=production` or `env=prod` or `env=production` or `environment=prod`. + + ```bash + velero restore create restore-prod --from-backup=prod-backup --or-selector "env in (prod,production) or environment in (prod, production)" + ``` + +### --include-cluster-scoped-resources +Kubernetes cluster-scoped resources to include in the backup, formatted as resource.group, such as `storageclasses.storage.k8s.io`(use '*' for all resources). Cannot work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources`. This parameter only works for backup, not for restore. + +* Backup all StorageClasses and ClusterRoles in the cluster. + + ```bash + velero backup create --include-cluster-scoped-resources="storageclasses,clusterroles" + ``` + +* Backup all cluster-scoped resources in the cluster. + + ```bash + velero backup create --include-cluster-scoped-resources="*" + ``` + + +### --include-namespace-scoped-resources +Kubernetes namespace resources to include in the backup, formatted as resource.group, such as `deployments.apps`(use '*' for all resources). Cannot work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources`. This parameter only works for backup, not for restore. + +* Backup all Deployments and ConfigMaps in the cluster. + + ```bash + velero backup create --include-namespace-scoped-resources="deployments.apps,configmaps" + ``` + +* Backup all namespace resources in the cluster. + + ```bash + velero backup create --include-namespace-scoped-resources="*" + ``` + +## Excludes + +Exclude specific resources from the backup. + +Wildcard excludes are ignored. + +### --exclude-namespaces + +Namespaces to exclude. Accepts glob patterns (`*`, `?`, `[abc]`). + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. + +* Exclude kube-system from the cluster backup. + + ```bash + velero backup create --exclude-namespaces kube-system + ``` + +* Exclude two namespaces during a restore. + + ```bash + velero restore create --exclude-namespaces , + ``` + +### --exclude-resources + +Kubernetes resources to exclude, formatted as resource.group, such as storageclasses.storage.k8s.io. Cannot work with `--include-cluster-scoped-resources`, `--exclude-cluster-scoped-resources`, `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources`. + +* Exclude secrets from the backup. + + ```bash + velero backup create --exclude-resources secrets + ``` + +* Exclude secrets and rolebindings. + + ```bash + velero backup create --exclude-resources secrets,rolebindings + ``` + +### velero.io/exclude-from-backup=true + +* Resources with the label `velero.io/exclude-from-backup=true` are not included in backup, even if it contains a matching selector label. + +### --exclude-cluster-scoped-resources +Kubernetes cluster-scoped resources to exclude from the backup, formatted as resource.group, such as `storageclasses.storage.k8s.io`(use '*' for all resources). Cannot work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources`. This parameter only works for backup, not for restore. + +* Exclude StorageClasses and ClusterRoles from the backup. + + ```bash + velero backup create --exclude-cluster-scoped-resources="storageclasses,clusterroles" + ``` + +* Exclude all cluster-scoped resources from the backup. + + ```bash + velero backup create --exclude-cluster-scoped-resources="*" + ``` + +### --exclude-namespace-scoped-resources +Kubernetes namespace resources to exclude from the backup, formatted as resource.group, such as `deployments.apps`(use '*' for all resources). Cannot work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources`. This parameter only works for backup, not for restore. + +* Exclude all Deployments and ConfigMaps from the backup. + + ```bash + velero backup create --exclude-namespace-scoped-resources="deployments.apps,configmaps" + ``` + +* Exclude all namespace resources from the backup. + + ```bash + velero backup create --exclude-namespace-scoped-resources="*" + ``` + +## Resource policies +Velero provides resource policies to filter resources to do backup, which may contain `includeExcludePolicy` and `volumePolicies`. + +### Creating resource policies + +Below is the two-step of using resource policies in backup: +1. Creating resource policies configmap + + Users need to create one configmap in Velero install namespace from a YAML file that defined resource policies. The creating command would be like the below: + ```bash + kubectl create cm --from-file -n velero + ``` +2. Creating a backup reference to the defined resource policies + + Users create a backup with the flag `--resource-policies-configmap`, which will reference the current backup to the defined resource policies. The creating command would be like the below: + ```bash + velero backup create --resource-policies-configmap + ``` + This flag could also be combined with the other include and exclude filters above + +### YAML template +The policies YAML config file would look like this: +- Yaml template: + ```yaml + # currently only supports v1 version + version: v1 + # The filters in includeExcludePolicy work the same as the scoped resources filters in the Spec of a Backup + # NOTE: similar to scoped filters in Backup Spec, the includeExcludePolicy does not work with --include-resources, --exclude-resources and --include-cluster-resources filters in Backup. + includeExcludePolicy: + includedClusterScopedResources: + - "crd" + - "pv" + excludedClusterScopedResources: [] + includedNamespaceScopedResources: + - "pod" + - "service" + - "deployment" + - "pvc" + excludedNamespaceScopedResources: + - "configmap" + - "secret" + volumePolicies: + # each policy consists of a list of conditions and an action + # we could have lots of policies, but if the resource matched the first policy, the latter will be ignored + # each key in the object is one condition, and one policy will apply to resources that meet ALL conditions + # NOTE: capacity or storageClass is suited for [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes), and pod [Volume](https://kubernetes.io/docs/concepts/storage/volumes) not support it. + - conditions: + # capacity condition matches the volumes whose capacity falls into the range + capacity: "10,100Gi" + # pv matches specific csi driver + csi: + driver: ebs.csi.aws.com + # pv matches one of the storage class list + storageClass: + - gp2 + - standard + # pvc matches specific phase(s) + pvcPhase: + - Pending + action: + type: skip + - conditions: + capacity: "0,100Gi" + # nfs volume source with specific server and path (nfs could be empty or only config server or path) + nfs: + server: 192.168.200.90 + path: /mnt/data + action: + type: skip + - conditions: + nfs: + server: 192.168.200.90 + action: + type: fs-backup + - conditions: + # nfs could be empty which matches any nfs volume source + nfs: {} + action: + type: skip + - conditions: + # csi could be empty which matches any csi volume source + csi: {} + action: + type: snapshot + - conditions: + volumeTypes: + - emptyDir + - downwardAPI + - configmap + - cinder + action: + type: skip + ``` +### IncludeExcludePolicy +The `includeExcludePolicy` is used to filter resources based on the namespace-scoped and cluster-scoped resources. User can use it +to define a group of filters and reuse them across different backups. + +For example, user can configmap `my-policy` of resource policies with following content: +```yaml +version: v1 +includeExcludePolicy: + includedClusterScopedResources: + - "crd" + excludedClusterScopedResources: [] + includedNamespaceScopedResources: [] + excludedNamespaceScopedResources: + - "configmap" + - "event" +``` +If the user creates a backup via command like +```bash +velero backup create --resource-policies-configmap my-policy --include-namespaces my-workload-ns +``` +The backup will include all resources in namespace `my-workload-ns` except for `configmap` and `event`, and all CRDs in the cluster. + +#### Limitations +The `includeExcludePolicy` does not work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources` filters in Backup. +If the user create the backup with command like `velero backup create my-backup --include-cluster-resources --include-namespaces workload-ns --resource-policies-configmap my-policy` +the backup will fail with status `FailedValidation` + +The filters in `includeExcludePolicy` cannot include `*`. Only specific resources can be set in the filters. + +#### "includeExcludePolicy" .vs. filters in Backup Spec +User can use the includeExcludePolicy with other _scoped filters_ when creating a backup. velero will combine the filters +when it's collecting the resources during the backup. During this process the filters in the Backup Spec have higher priority. +For example, if the user use this command to create a backup, reusing the resource policies created in the previous example: +```bash +velero backup create --resource-policies-configmap my-policy --include-namespace-scoped-resources * --include-cluster-scoped-resources -apiservices --include-namespaces my-workload-ns +``` +The backup will include all resources in namespace `my-workload-ns`, including `configmap` and `event`, and all CRDs and +`apiservices` in the cluster. + +### VolumePolicy +VolumePolicy is a data structure to control how velero handle the volumes matching certain conditions. + +#### Supported VolumePolicy actions +There are three actions supported via the VolumePolicy feature: +* skip: don't back up the action matching volume's data. +* snapshot: back up the action matching volume's data by the snapshot way. +* fs-backup: back up the action matching volumes' data by the fs-backup way. + +#### Supported conditions + +Currently, Velero supports the volume attributes listed below: +- capacity: matching volumes have the capacity that falls within this `capacity` range. The capacity value should include the lower value and upper value concatenated by commas, the unit of each value in capacity could be `Ti`, `Gi`, `Mi`, `Ki` etc, which is a standard storage unit in Kubernetes. And it has several combinations below: + - "0,5Gi" or "0Gi,5Gi" which means capacity or size matches from 0 to 5Gi, including value 0 and value 5Gi + - ",5Gi" which is equal to "0,5Gi" + - "5Gi," which means capacity or size matches larger than 5Gi, including value 5Gi + - "5Gi" which is not supported and will be failed in validating the configuration +- storageClass: matching volumes those with specified `storageClass`, such as `gp2`, `ebs-sc` in eks +- volume sources: matching volumes that used specified volume sources. Currently we support nfs or csi backend volume source +- pvcPhase: matching volumes based on the phase of their associated PVCs (Pending, Bound, Lost) + +Velero supported conditions and format listed below: +- capacity + ```yaml + # match volume has the size between 10Gi and 100Gi + capacity: "10Gi,100Gi" + ``` +- storageClass + ```yaml + # match volume has the storage class gp2 or ebs-sc + storageClass: + - gp2 + - ebs-sc + ``` +- volume sources (currently only support below format and attributes) +1. Specify the volume source name, the name could be `nfs`, `rbd`, `iscsi`, `csi` etc, but Velero only support `nfs` and `csi` currently. + ```yaml + # match any volume has nfs volume source + nfs : {} + # match any volume has csi volume source + csi : {} + ``` + +2. Specify details for the related volume source (currently we only support csi driver filter and nfs server or path filter) + ```yaml + # match volume has csi volume source and using `aws.efs.csi.driver` + csi: + driver: aws.efs.csi.driver + # match volume has nfs volume source and using below server and path + nfs: + server: 192.168.200.90 + path: /mnt/nfs + ``` + For volume provisioned by [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes) support all above attributes, but for pod [Volume](https://kubernetes.io/docs/concepts/storage/volumes) only support filtered by volume source. + +- volume types + + Support filter volumes by types + ```yaml + volumeTypes: + # matches volumes listed below + - emptyDir + - downwardAPI + - configmap + - cinder + ``` + Volume types could be found in [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes) and pod [Volume](https://kubernetes.io/docs/concepts/storage/volumes) + +- pvc Labels + + This condition filters volumes based on the labels on their associated PVCs. The condition is specified as a simple key/value mapping. The volume matches this condition if all the key/value pairs defined in the policy are present on the PVC. + ```yaml + pvcLabels: + environment: production + ``` + + Some examples: + - Environment specific labels: Snapshot volumes whose associated PVC has the label `environment: production`. + ```yaml + volumePolicies: + - conditions: + pvcLabels: + environment: production + action: + type: snapshot + ``` + - Subset Matching: Even if the PVC contains extra labels, it will match as long as the required key/value pair is present. For example, if the PVC has: + ```yaml + labels: + environment: production + team: backend + ``` + the following policy will match because it only requires `environment: production`: + ```yaml + volumePolicies: + - conditions: + pvcLabels: + environment: production + action: + type: skip + ``` + - Mismatched PVC Labels: If the policy requires both `environment: production` and `app: frontend`, but the PVC only has `environment: production`, the volume will not match. + ```yaml + volumePolicies: + - conditions: + pvcLabels: + environment: production + app: frontend + action: + type: skip + ``` + +- pvc Phase + + This condition filters volumes based on the phase of their associated PVCs. The condition is specified as a list of phases to match. The volume matches this condition if the PVC's phase matches any of the phases in the list. Supported phases are: `Pending`, `Bound`, and `Lost`. + ```yaml + pvcPhase: + - Pending + ``` + + Some examples: + - Skip Pending PVCs: Skip backup of volumes whose associated PVC is in `Pending` phase (useful for PVCs that haven't been bound to a PV yet). + ```yaml + volumePolicies: + - conditions: + pvcPhase: + - Pending + action: + type: skip + ``` + - Skip multiple phases: Skip backup of volumes whose associated PVC is either in `Pending` or `Lost` phase. + ```yaml + volumePolicies: + - conditions: + pvcPhase: + - Pending + - Lost + action: + type: skip + ``` + - Backup only Bound PVCs: Only backup volumes whose associated PVC is in `Bound` phase. + ```yaml + volumePolicies: + - conditions: + pvcPhase: + - Bound + action: + type: snapshot + ``` + - Combine with other conditions: You can combine PVC phase conditions with other conditions like storage class or labels. + ```yaml + volumePolicies: + - conditions: + pvcPhase: + - Pending + storageClass: + - gp2 + action: + type: skip + ``` + + + +### Resource policies rules +- Velero already has lots of include or exclude filters. the resource policies are the final filters after others include or exclude filters in one backup processing workflow. So if use a defined similar filter like the opt-in approach to backup one pod volume but skip backup of the same pod volume in resource policies, as resource policies are the final filters that are applied, the volume will not be backed up. +- If volume resource policies conflict with themselves the first matched policy will be respected when many policies are defined. + +#### VolumePolicy priority with existing filters +* [Includes filters](#includes) and [Excludes filters](#excludes) have the highest priority. The filtered-out resources by them cannot reach to the VolumePolicy. +* The VolumePolicy has the second priority. It supersedes all the other filters. +* The filesystem volume backup opt-in/opt-out way has the third priority. +* The `backup.Spec.SnapshotVolumes` has the fourth priority. + +#### Support for `fs-backup` and `snapshot` actions via volume policy feature +- Starting from velero 1.14, the volume policy feature has been extended to support more actions like `fs-backup` and `snapshot`. +- This feature only extends the action aspect of volume policy and not criteria aspect, the criteria components as described above remain the same. +- When we are using the volume policy approach for backing up the volumes then the volume policy criteria and action need to be specific and explicit, +there is no default behaviour, if a volume matches fs-backup action then fs-backup method will be used for that volume and similarly if the volume matches +the criteria for snapshot action then the snapshot workflow will be used for the volume backup. +- Another thing to note is that the volume policy workflow uses the legacy opt-in/opt-out approach as a fallback option. For instance, the user specifies +a volume policy but for a particular volume included in the backup there are no actions(fs-backup/snapshot) matching in the volume policy for that volume, +in such a scenario the legacy approach will be used for backing up the particular volume. Considering everything, the recommendation would be to use only one +of the approaches to backup volumes - volume policy approach or the opt-in/opt-out legacy approach, and not mix them for clarity. +- Snapshot action can either be a native snapshot or a csi snapshot or csi snapshot datamover, as is the case with the current flow where velero itself makes the decision based on the backup CR's existing options. +- The `snapshot` action via Volume Policy has higher priority if there is a `snapshot` action matching for a particular volume, this volume would be backed up via snapshot irrespective of the value of `backup.Spec.SnapshotVolumes`. +- If for a particular volume there is no `snapshot` matching action then the volume will be backed up via snapshot given that `backup.Spec.SnapshotVolumes` is not explicitly set to false. +- Let's see some examples on how to use the volume policy feature for `fs-backup` and `snapshot` action purposes: + +We will use a simple application example in which there is an application pod which has 2 volumes: +- Volume 1 has associated Persistent Volume Claim 1 and Persistent Volume 1 which uses storage class `gp2-csi` +- Volume 2 has associated Persistent Volume Claim 2 and Persistent Volume 2 which uses storage class `gp3-csi` + +Now lets go through some example uses-cases and their outcomes: + +***Example 1: User wants to use `fs-backup` action for backing up the volumes having storage class as `gp2-csi`*** +1. User specifies the volume policy as follows: +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - gp2-csi + action: + type: fs-backup +``` + +2. User creates a backup using this volume policy +3. The outcome would be that velero would perform `fs-backup` operation ***only*** on `Volume 1` as ***only*** `Volume 1` satisfies the criteria for `fs-backup` action. + +***Example 2: User wants to use `snapshot` action for backing up the volumes having storage class as `gp2-csi`*** +1. User specifies the volume policy as follows: +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - gp2-csi + action: + type: snapshot +``` +2. User creates a backup using this volume policy +3. The outcome would be that velero would perform `snapshot` operation ***only*** on `Volume 1` as ***only*** `Volume 1` satisfies the criteria for `snapshot` action. + +***Example 3: User wants to use `snapshot` action for backing up the volumes having storage class as `gp2-csi` and wants to use `fs-backup` action for backing up the volumes having storage class as `gp3-csi`*** +1. User specifies the volume policy as follows: +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - gp2-csi + action: + type: snapshot +- conditions: + storageClass: + - gp3-csi + action: + type: fs-backup +``` +2. User creates a backup using this volume policy +3. The outcome would be that velero would perform `snapshot` operation ***only*** on `Volume 1` as ***only*** `Volume 1` satisfies the criteria for `snapshot` action. Also, velero would perform `fs-backup` operation ***only*** on `Volume 2` as ***only*** `Volume 2` satisfies the criteria for `fs-backup` action. + +***Example 4: User wants to use `snapshot` action for backing up the volumes having storage class as `gp3-csi` and at the same time also annotates the pod to use opt-in fs-backup legacy approach for Volume 1*** +1. User specifies the volume policy as follows and also annotates the pod with `backup.velero.io/backup-volumes=Volume 1` +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - gp3-csi + action: + type: snapshot +``` +2. User creates a backup using this volume policy +3. The outcome would be that velero would perform `snapshot` operation for `Volume 2` as it matches the action criteria and velero would also perform the `fs-backup` operation for `Volume-1` via the legacy annotations based fallback approach as there is no matching action for `Volume-1` + +***Example 5: User wants to use `fs-backup` action for backing up the volumes having storage class as `gp2-csi` and at the same time also specifies `defaultVolumesToFSBackup: true` (fallback option for no action matching volumes)*** +1. User specifies the volume policy as follows and specifies `defaultVolumesToFSBackup: true`: +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - gp2-csi + action: + type: fs-backup +``` +2. User creates a backup using this volume policy +3. The outcome would be that velero would perform `fs-backup` operation on both the volumes + - `fs-backup` on `Volume 1` because `Volume 1` satisfies the criteria for `fs-backup` action. + - Also, for Volume 2 as no matching action was found so legacy approach will be used as a fallback option for this volume (`fs-backup` operation will be done as `defaultVolumesToFSBackup: true` is specified by the user). diff --git a/site/content/docs/v1.18/restore-hooks.md b/site/content/docs/v1.18/restore-hooks.md new file mode 100644 index 000000000..b5681733e --- /dev/null +++ b/site/content/docs/v1.18/restore-hooks.md @@ -0,0 +1,300 @@ +--- +title: "Restore Hooks" +layout: docs +--- + +Velero supports Restore Hooks, custom actions that can be executed during or after the restore process. There are two kinds of Restore Hooks: + +1. InitContainer Restore Hooks: These will add init containers into restored pods to perform any necessary setup before the application containers of the restored pod can start. +1. Exec Restore Hooks: These can be used to execute custom commands or scripts in containers of a restored Kubernetes pod. + +## InitContainer Restore Hooks + +Use an `InitContainer` hook to add init containers into a pod before it's restored. You can use these init containers to run any setup needed for the pod to resume running from its backed-up state. +The InitContainer added by the restore hook will be the first init container in the `podSpec` of the restored pod. +In the case where the pod had volumes backed up using File System Backup, then, the restore hook InitContainer will be added after the `restore-wait` InitContainer. + +NOTE: This ordering can be altered by any mutating webhooks that may be installed in the cluster. + +There are two ways to specify `InitContainer` restore hooks: +1. Specifying restore hooks in annotations +1. Specifying restore hooks in the restore spec + +### Specifying Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify restore hooks: +* `init.hook.restore.velero.io/container-image` + * The container image for the init container to be added. Optional. +* `init.hook.restore.velero.io/container-name` + * The name for the init container that is being added. Optional. +* `init.hook.restore.velero.io/command` + * This is the `ENTRYPOINT` for the init container being added. This command is not executed within a shell and the container image's `ENTRYPOINT` is used if this is not provided. If a shell is needed to run your command, include a shell command, like `/bin/sh`, that is supported by the container at the beginning of your command. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]`. See [InitContainer As Pod Annotation Example](#initcontainer-restore-hooks-as-pod-annotation-example). Optional. + +#### InitContainer Restore Hooks As Pod Annotation Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + init.hook.restore.velero.io/container-name=restore-hook \ + init.hook.restore.velero.io/container-image=alpine:latest \ + init.hook.restore.velero.io/command='["/bin/ash", "-c", "date"]' +``` + +With the annotation above, Velero will add the following init container to the pod when it's restored. + +```json +{ + "command": [ + "/bin/ash", + "-c", + "date" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook" + ... +} +``` + +### Specifying Restore Hooks In Restore Spec + +Init container restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. +Init container restore hook command is not executed within a shell by default. If a shell is needed to run your command, include a shell command, like /bin/sh, that is supported by the container at the beginning of your command. + +#### Example + +Below is an example of specifying restore hooks in `RestoreSpec` + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - init: + initContainers: + - name: restore-hook-init1 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc1-vm + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc2-vm + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed +``` + +The `hooks` in the above `RestoreSpec`, when restored, will add two init containers to every pod in the `app` namespace + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"FOOBARBAZ\" >> /restores/pvc1-vm/foobarbaz" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init1", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc1-vm", + "name": "pvc1-vm" + } + ] + ... +} +``` + +and + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"DEADFEED\" >> /restores/pvc2-vm/deadfeed" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init2", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc2-vm", + "name": "pvc2-vm" + } + ] + ... +} +``` + +## Exec Restore Hooks + +Use an Exec Restore hook to execute commands in a restored pod's containers after they start. + +There are two ways to specify `Exec` restore hooks: +1. Specifying exec restore hooks in annotations +1. Specifying exec restore hooks in the restore spec + +If a pod has the annotation `post.hook.restore.velero.io/command` then that is the only hook that will be executed in the pod. +No hooks from the restore spec will be executed in that pod. + +### Specifying Exec Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify exec restore hooks: +* `post.hook.restore.velero.io/container` + * The container name where the hook will be executed. Defaults to the first container. Optional. +* `post.hook.restore.velero.io/command` + * The command that will be executed in the container. This command is not executed within a shell by default. If a shell is needed to run your command, include a shell command, like `/bin/sh`, that is supported by the container at the beginning of your command. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]`. See [Exec Restore Hooks As Pod Annotation Example](#exec-restore-hooks-as-pod-annotation-example). Optional. +* `post.hook.restore.velero.io/on-error` + * How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, no more restore hooks will be executed in any container in any pod and the status of the Restore will be `PartiallyFailed`. Optional. +* `post.hook.restore.velero.io/exec-timeout` + * How long to wait once execution begins. Defaults is 30 seconds. Optional. +* `post.hook.restore.velero.io/wait-timeout` + * How long to wait for a container to become ready. This should be long enough for the container to start plus any preceding hooks in the same container to complete. The wait timeout begins when the container is restored and may require time for the image to pull and volumes to mount. If not set the restore will wait indefinitely. Optional. +* `post.hook.restore.velero.io/wait-for-ready` + * String representation of a boolean that ensure command will be launched when underlying container is fully [Ready](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-readiness-probes). Use with caution because if only one restore hook for a pod consists of `WaitForReady` flag as "true", all the other hook executions for that pod, whatever their origin (`Backup` or `Restore` CRD), will wait for `Ready` state too. Any value except "true" will be considered as "false". Defaults is false. Optional. + +#### Exec Restore Hooks As Pod Annotation Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + post.hook.restore.velero.io/container=postgres \ + post.hook.restore.velero.io/command='["/bin/bash", "-c", "psql < /backup/backup.sql"]' \ + post.hook.restore.velero.io/wait-timeout=5m \ + post.hook.restore.velero.io/exec-timeout=45s \ + post.hook.restore.velero.io/on-error=Continue +``` + +### Specifying Exec Restore Hooks in Restore Spec + +Exec restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. +Exec restore hook command is not executed within a shell by default. If a shell is needed to run your command, include a shell command, like /bin/sh, that is supported by the container at the beginning of your command. + +#### Multiple Exec Restore Hooks Example + +Below is an example of specifying restore hooks in a `RestoreSpec`. +When using the restore spec it is possible to specify multiple hooks for a single pod, as this example demonstrates. + +All hooks applicable to a single container will be executed sequentially in that container once it starts. +The ordering of hooks executed in a single container follows the order of the restore spec. +In this example, the `pg_isready` hook is guaranteed to run before the `psql` hook because they both apply to the same container and the `pg_isready` hook is defined first. + +If a pod has multiple containers with applicable hooks, all hooks for a single container will be executed before executing hooks in another container. +In this example, if the postgres container starts before the sidecar container, both postgres hooks will run before the hook in the sidecar. +This means the sidecar container may be running for several minutes before its hook is executed. + +Velero guarantees that no two hooks for a single pod are executed in parallel, but hooks executing in different pods may run in parallel. + + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - exec: + execTimeout: 1m + waitTimeout: 5m + onError: Fail + container: postgres + command: + - /bin/bash + - '-c' + - 'while ! pg_isready; do sleep 1; done' + - exec: + container: postgres + waitTimeout: 6m + execTimeout: 1m + command: + - /bin/bash + - '-c' + - 'psql < /backup/backup.sql' + - exec: + container: sidecar + command: + - /bin/bash + - '-c' + - 'date > /start' +``` + +## Restore hook commands using scenarios +### Using environment variables + +You are able to use environment variables from your pods in your pre and post hook commands by including a shell command before using the environment variable. For example, `MYSQL_ROOT_PASSWORD` is an environment variable defined in pod called `mysql`. To use `MYSQL_ROOT_PASSWORD` in your pre-hook, you'd include a shell, like `/bin/sh`, before calling your environment variable: + +``` +postHooks: +- exec: + container: mysql + command: + - /bin/sh + - -c + - mysql --password=$MYSQL_ROOT_PASSWORD -e "FLUSH TABLES WITH READ LOCK" + onError: Fail +``` + +Note that the container must support the shell command you use. + +## Restore Hook Execution Results +### Viewing Results + +Velero records the execution results of hooks, allowing users to obtain this information by running the following command: + +```bash +$ velero restore describe +``` + +The displayed results include the number of hooks that were attempted to be executed and the number of hooks that failed execution. Any detailed failure reasons will be present in `Errors` section if applicable. + +```bash +HooksAttempted: 1 +HooksFailed: 0 +``` + + +[1]: api-types/restore.md diff --git a/site/content/docs/v1.18/restore-reference.md b/site/content/docs/v1.18/restore-reference.md new file mode 100644 index 000000000..6c5394cf0 --- /dev/null +++ b/site/content/docs/v1.18/restore-reference.md @@ -0,0 +1,375 @@ +--- +title: "Restore Reference" +layout: docs +--- + +The page outlines how to use the `velero restore` command, configuration options for restores, and describes the main process Velero uses to perform restores. + +## Restore command-line options +To see all commands for restores, run `velero restore --help`. + +To see all options associated with a specific command, provide the `--help` flag to that command. For example, `velero restore create --help` shows all options associated with the `create` command. + +```Usage: + velero restore [command] + +Available Commands: + create Create a restore + delete Delete restores + describe Describe restores + get Get restores + logs Get restore logs +``` + +## Detailed Restore workflow + +The following is an overview of Velero's restore process that starts after you run `velero restore create`. + +1. The Velero client makes a call to the Kubernetes API server to create a [`Restore`](api-types/restore.md) object. + +1. The `RestoreController` notices the new Restore object and performs validation. + +1. The `RestoreController` fetches basic information about the backup being restored, like the [BackupStorageLocation](locations.md) (BSL). It also fetches a tarball of the cluster resources in the backup, any volumes that will be restored using File System Backup, and any volume snapshots to be restored. + +1. The `RestoreController` then extracts the tarball of backup cluster resources to the /tmp folder and performs some pre-processing on the resources, including: + + * Sorting the resources to help Velero decide the [restore order](#resource-restore-order) to use. + + * Attempting to discover the resources by their Kubernetes [Group Version Resource (GVR)](https://kubernetes.io/docs/reference/using-api/api-concepts/). If a resource is not discoverable, Velero will exclude it from the restore. See more about how [Velero backs up API versions](#backed-up-api-versions). + + * Applying any configured [resource filters](resource-filtering.md). + + * Verify the target namespace, if you have configured [`--namespace-mappings`](#restoring-into-a-different-namespace) restore option. + + +1. The `RestoreController` begins restoring the eligible resources one at a time. Velero extracts the current resource into a Kubernetes resource object. Depending on the type of resource and restore options you specified, Velero will make the following modifications to the resource or preparations to the target cluster before attempting to create the resource: + + * The `RestoreController` makes sure the target namespace exists. If the target namespace does not exist, then the `RestoreController` will create a new one on the cluster. + + * If the resource is a Persistent Volume (PV), the `RestoreController` will [rename](#persistent-volume-rename) the PV and [remap](#restoring-into-a-different-namespace) its namespace. + + * If the resource is a Persistent Volume Claim (PVC), the `RestoreController` will modify the [PVC metadata](#pvc-restore). + + * Execute the resource’s `RestoreItemAction` [custom plugins](custom-plugins/), if you have configured one. + + * Update the resource object’s namespace if you've configured [namespace remapping](#restoring-into-a-different-namespace). + + * The `RestoreController` adds a `velero.io/backup-name` label with the backup name and a `velero.io/restore-name` with the restore name to the resource. This can help you easily identify restored resources and which backup they were restored from. + +1. The `RestoreController` creates the resource object on the target cluster. If the resource is a PV then the `RestoreController` will restore the PV data from the [durable snapshot](#durable-snapshot-pv-restore), [File System Backup](#file-system-backup-pv-restore), or [CSI snapshot](#csi-pv-restore) depending on how the PV was backed up. + + If the resource already exists in the target cluster, which is determined by the Kubernetes API during resource creation, the `RestoreController` will skip the resource. The only [exception](#restore-existing-resource-policy) are Service Accounts, which Velero will attempt to merge differences between the backed up ServiceAccount into the ServiceAccount on the target cluster. You can [change the default existing resource restore policy](#restore-existing-resource-policy) to update resources instead of skipping them using the `--existing-resource-policy`. + +1. Once the resource is created on the target cluster, Velero may take some additional steps or wait for additional processes to complete before moving onto the next resource to restore. + + * If the resource is a Pod, the `RestoreController` will execute any [Restore Hooks](restore-hooks.md) and wait for the hook to finish. + * If the resource is a PV restored by File System Backup, the `RestoreController` waits for File System Backup’s restore to complete. The `RestoreController` sets a timeout for any resources restored with File System Backup during a restore. The default timeout is 4 hours, but you can configure this be setting using `--fs-backup-timeout` restore option. + * If the resource is a Custom Resource Definition, the `RestoreController` waits for its availability in the cluster. The timeout is 1 minute. + + If any failures happen finishing these steps, the `RestoreController` will log an error in the restore result and will continue restoring. + +## Restore order + +By default, Velero will restore resources in the following order: + +* Custom Resource Definitions +* Namespaces +* StorageClasses +* VolumeSnapshotClass +* VolumeSnapshotContents +* VolumeSnapshots +* PersistentVolumes +* PersistentVolumeClaims +* Secrets +* ConfigMaps +* ServiceAccounts +* LimitRanges +* Pods +* ReplicaSets +* Clusters +* ClusterResourceSets + +It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. Velero will append resources not listed to the end of your customized list in alphabetical order. + +```shell +velero server \ +--restore-resource-priorities=customresourcedefinitions,namespaces,storageclasses,\ +volumesnapshotclass.snapshot.storage.k8s.io,volumesnapshotcontents.snapshot.storage.k8s.io,\ +volumesnapshots.snapshot.storage.k8s.io,persistentvolumes,persistentvolumeclaims,secrets,\ +configmaps,serviceaccounts,limitranges,pods,replicasets.apps,clusters.cluster.x-k8s.io,\ +clusterresourcesets.addons.cluster.x-k8s.io +``` + + +## Restoring Persistent Volumes and Persistent Volume Claims + +Velero has three approaches when restoring a PV, depending on how the backup was taken. + +1. When restoring a snapshot, Velero statically creates the PV and then binds it to a restored PVC. Velero's PV rename and remap process is used only in this case because this is the only case where Velero creates the PV resource directly. +1. When restoring with File System Backup, Velero uses Kubernetes’ [dynamic provision process](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) to provision the PV after creating the PVC. In this case, the PV object is not actually created by Velero. +1. When restoring with the [CSI plugin](csi.md), the PV is created from a CSI snapshot by the CSI driver. Velero doesn’t create the PV directly. Instead Velero creates a PVC with its DataSource referring to the CSI VolumeSnapshot object. + +### Snapshot PV Restore + +PV data backed up by durable snapshots is restored by VolumeSnapshot plugins. Velero calls the plugins’ interface to create a volume from a snapshot. The plugin returns the volume’s `volumeID`. This ID is created by storage vendors and will be updated in the PV object created by Velero, so that the PV object is connected to the volume restored from a snapshot. + +### File System Backup PV Restore + +For more information on File System Backup restores, see the [File System Backup](file-system-backup.md#restore) page. + +### CSI PV Restore + +A PV backed up by CSI snapshots is restored by the [CSI plugin](csi). This happens when restoring the PVC object that has been snapshotted by CSI. The CSI VolumeSnapshot object name is specified with the PVC during backup as the annotation `velero.io/volume-snapshot-name`. After validating the VolumeSnapshot object, Velero updates the PVC by adding a `DataSource` field and setting its value to the VolumeSnapshot name. + +### Persistent Volume Rename + +When restoring PVs, if the PV being restored does not exist on the target cluster, Velero will create the PV using the name from the backup. Velero will rename a PV before restoring if both of the following conditions are met: + +1. The PV already exists on the target cluster. +1. The PV’s claim namespace has been [remapped](#restoring-into-a-different-namespace). + +If both conditions are met, Velero will create the PV with a new name. The new name is the prefix `velero-clone-` and a random UUID. Velero also preserves the original name of the PV by adding an annotation `velero.io/original-pv-name` to the restored PV object. + +If you attempt to restore the PV's referenced PVC into its original namespace without remapping the namespace, Velero will not rename the PV. If a PV's referenced PVC exists already for that namespace, the restored PV creation attempt will fail, with an `Already Exist` error from the Kubernetes API Server. + +### PVC Restore + +PVC objects are created the same way as other Kubernetes resources during a restore, with some specific changes: +* For a dynamic binding PVCs, Velero removes the fields related to bindings from the PVC object. This enables the default Kubernetes [dynamic binding process](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#binding) to be used for this PVC. The fields include: + * volumeName + * pv.kubernetes.io/bind-completed annotation + * pv.kubernetes.io/bound-by-controller annotation +* For a PVC that is bound by Velero Restore, if the target PV has been renamed by the [PV restore process](#persistent-volume-rename), the RestoreController renames the `volumeName` field of the PVC object. + +### Changing PV/PVC Storage Classes + +Velero can change the storage class of persistent volumes and persistent volume claims during restores. To configure a storage class mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-storage-class-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-storage-class: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key is the old + # storage class name and the value is the new storage + # class name. + : +``` +### Changing Image Repositories +Velero can change the image name of pod/deployment/statefulsets/daemonset/replicaset/replicationcontroller/job/cronjob during restores. To configure a image name mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-image-name-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-image-name: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key can be any + # words that ConfigMap accepts. + # the value should be： + # "" + # for current implementation the can only be "," + # e.x: in case your old image name is 1.1.1.1:5000/abc:test + "case1":"1.1.1.1:5000,2.2.2.2:3000" + "case2":"5000,3000" + "case3":"abc:test,edf:test" + "case5":"test,latest" + "case4":"1.1.1.1:5000/abc:test,2.2.2.2:3000/edf:test" + # Please note that image name may contain more than one part that + # matching the replacing words. + # e.x:in case your old image names are: + # dev/image1:dev and dev/image2:dev + # you want change to: + # test/image1:dev and test/image2:dev + # the suggested replacing rule is: + "case5":"dev/,test/" + # this will avoid unexpected replacement to the second "dev". +``` + +### PVC selected-node + +Velero removes PVC's `volume.kubernetes.io/selected-node` annotation during restore, so that the restored PVC could be provisioned appropriately according to ```WaitForFirstConsumer``` rules, storage topologies and the restored pod's schedule result, etc. + +For more information of how this selected-node annotation matters to PVC restore, see issue https://github.com/vmware-tanzu/velero/issues/9053. + +## Restoring into a different namespace + +Velero can restore resources into a different namespace than the one they were backed up from. To do this, use the `--namespace-mappings` flag: + +```bash +velero restore create \ + --from-backup \ + --namespace-mappings old-ns-1:new-ns-1,old-ns-2:new-ns-2 +``` + +For example, A Persistent Volume object has a reference to the Persistent Volume Claim’s namespace in the field `Spec.ClaimRef.Namespace`. If you specify that Velero should remap the target namespace during the restore, Velero will change the `Spec.ClaimRef.Namespace` field on the PV object from `old-ns-1` to `new-ns-1`. + +## Restore existing resource policy + +By default, Velero is configured to be non-destructive during a restore. This means that it will never overwrite data that already exists in your cluster. When Velero attempts to create a resource during a restore, the resource being restored is compared to the existing resources on the target cluster. If the resource already exists in the target cluster, Velero skips restoring the current resource and moves onto the next resource to restore, without making any changes to the target cluster. + +An exception to the default restore policy is ServiceAccounts. When restoring a ServiceAccount that already exists on the target cluster, Velero will attempt to merge the fields of the ServiceAccount from the backup into the existing ServiceAccount. Secrets and ImagePullSecrets are appended from the backed-up ServiceAccount. Velero adds any non-existing labels and annotations from the backed-up ServiceAccount to the existing resource, leaving the existing labels and annotations in place. + +You can change this policy for a restore by using the `--existing-resource-policy` restore flag. The available options +are `none` (default) and `update`. If you choose to update existing resources during a restore +(`--existing-resource-policy=update`), Velero will attempt to update an existing resource to match the resource from the backup: + +* If the existing resource in the target cluster is the same as the resource Velero is attempting to restore, Velero will add a `velero.io/backup-name` label with the backup name and a `velero.io/restore-name` label with the restore name to the existing resource. If patching the labels fails, Velero adds a restore error and continues restoring the next resource. + +* If the existing resource in the target cluster is different from the backup, Velero will first try to patch the existing resource to match the backup resource. If the patch is successful, Velero will add a `velero.io/backup-name` label with the backup name and a `velero.io/restore-name` label with the restore name to the existing resource. If the patch fails, Velero adds a restore warning and tries to add the `velero.io/backup-name` and `velero.io/restore-name` labels on the resource. If the labels patch also fails, then Velero logs a restore error and continues restoring the next resource. + +You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. + +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. + +## Restore "status" field of objects + +By default, Velero will remove the `status` field of an object before it's restored. This is because the value `status` field is typically set by the controller during reconciliation. However, some custom resources are designed to store environment specific information in the `status` field, and it is important to preserve such information during restore. + +You can use `--status-include-resources` and `--status-exclude-resources` flags to select the resources whose `status` field will be restored by Velero. If there are resources selected via these flags, velero will trigger another API call to update the restored object to restore `status` field after it's created. + +## Object-Level Resource Status Restoration + +Starting from Velero 1.16, Velero now supports object-level control over status restoration during restores. Previously, status restoration could only be configured at the resource type level using the `restoreStatus` field in the Restore CR. With this enhancement, users and controllers can now specify status restoration at the individual resource instance level using the annotation: +```yaml +metadata: + annotations: + velero.io/restore-status: "true" +``` +This feature provides fine-grained control over status restoration, allowing users and controllers to mark specific objects for status restore while keeping others unaffected. + +Velero determines whether to restore the status of an object based on the following precedence: +1. Object-Level Annotation (`velero.io/restore-status`) +- If the annotation is present with a valid value ("true" or "false"), it overrides the global `restoreStatus` setting. +- "true" → Status will be restored for this object, even if the resource type is not included in `restoreStatus.includedResources`. +- "false" → Status will not be restored for this object, even if the resource type is included in `restoreStatus.includedResources`. +2. Restore CR (`restoreStatus` Field) +• If the annotation is missing or invalid, Velero falls back to the `restoreStatus` configuration in the Restore CR. +• Objects of resource types listed in `restoreStatus.includedResources` will have their status restored. +• Objects of resource types excluded from `restoreStatus.excludedResources` will not have their status restored. +3. Default Behavior +• If an object has no annotation and its resource type is not included in `restoreStatus.includedResources`, status restoration is skipped. + +Let's go over some examples: + +1. Restore Status for a Specific Object: If you want Velero to restore the status for a specific resource instance regardless of global restore settings, add the annotation: +```yaml +metadata: + annotations: + velero.io/restore-status: "true" +``` +Even if the resource type is not listed in `restoreStatus.includedResources`, this object’s status will be restored. + +2. Prevent Status Restore for a Specific Object: If you want to exclude a specific object from status restoration, even if its resource type is included in `restoreStatus.includedResources`, use: +```yaml +metadata: + annotations: + velero.io/restore-status: "false" +``` +This ensures that this specific object will not have its status restored. + +3. Restore Status Based on Resource Type: If you want to restore the status for all objects of a resource type, use the `restoreStatus` field in the Restore CR: +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: restore-with-status + namespace: velero +spec: + restoreStatus: + includedResources: + - foo.bar.io +``` +All `foo.bar.io` objects will have their status restored, except those that have `velero.io/restore-status`: "false" annotation. + +4. Default Behavior (No Annotations & No Restore Spec): If no `velero.io/restore-status` annotation is set and the resource type is not listed in `restoreStatus.includedResources`, Velero skips restoring status for that object. This maintains the existing default behavior. + +## Write Sparse files +If using fs-restore or CSI snapshot data movements, it's supported to write sparse files during restore by the below command: +```bash +velero restore create --from-backup --write-sparse-files --wait +``` + +## Parallel Files Download +If using fs-restore or CSI snapshot data movements, it's possible to configure one option for parallel file downloads during the restore by Kopia uploader using the command below: +```bash +velero restore create --from-backup --parallel-files-download --wait +``` + +## Removing a Restore object + +There are two ways to delete a Restore object: + +1. Deleting with `velero restore delete` will delete the Custom Resource representing the restore, along with its individual log and results files. It will not delete any objects that were created by the restore in your cluster. +2. Deleting with `kubectl -n velero delete restore` will delete the Custom Resource representing the restore. It will not delete restore log or results files from object storage, or any objects that were created during the restore in your cluster. + +## What happens to NodePorts and HealthCheckNodePort when restoring Services + +During a restore, Velero deletes **Auto assigned** NodePorts and HealthCheckNodePort by default and Services get new **auto assigned** nodePorts and healthCheckNodePort after restore. + +Velero auto detects **explicitly specified** NodePorts using **`last-applied-config`** annotation and **`managedFields`**. They are **preserved** after restore. NodePorts can be explicitly specified as `.spec.ports[*].nodePort` field on Service definition. + +Velero will do the same to the `HealthCheckNodePort` as `NodePorts`. + +### Always Preserve NodePorts and HealthCheckNodePort + +It is not always possible to set nodePorts and healthCheckNodePort explicitly on some big clusters because of operational complexity. As the Kubernetes [NodePort documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) states, "if you want a specific port number, you can specify a value in the `nodePort` field. The control plane will either allocate you that port or report that the API transaction failed. This means that you need to take care of possible port collisions yourself. You also have to use a valid port number, one that's inside the range configured for NodePort use."" + +The clusters which are not explicitly specifying nodePorts may still need to restore original NodePorts in the event of a disaster. Auto assigned nodePorts are typically defined on Load Balancers located in front of cluster. Changing all these nodePorts on Load Balancers is another operation complexity you are responsible for updating after disaster if nodePorts are changed. + +Use the `velero restore create ` command's `--preserve-nodeports` flag to preserve Service nodePorts and healthCheckNodePort always, regardless of whether nodePorts are explicitly specified or not. This flag is used for preserving the original nodePorts and healthCheckNodePort from a backup and can be used as `--preserve-nodeports` or `--preserve-nodeports=true`. If this flag is present, Velero will not remove the nodePorts and healthCheckNodePort when restoring a Service, but will try to use the nodePorts from the backup. + +Trying to preserve nodePorts and healthCheckNodePort may cause port conflicts when restoring on situations below: + +- If the nodePort from the backup is already allocated on the target cluster then Velero prints error log as shown below and continues the restore operation. + + ``` + time="2020-11-23T12:58:31+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is already allocated" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123125825 + ``` + +- If the nodePort from the backup is not in the nodePort range of target cluster then Velero prints error log as below and continues with the restore operation. Kubernetes default nodePort range is 30000-32767 but on the example cluster nodePort range is 20000-22767 and tried to restore Service with nodePort 31536. + + ``` + time="2020-11-23T13:09:17+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is not in the valid range. The range of valid ports is 20000-22767" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123130915 + ``` diff --git a/site/content/docs/v1.18/restore-resource-modifiers.md b/site/content/docs/v1.18/restore-resource-modifiers.md new file mode 100644 index 000000000..0c1f2f217 --- /dev/null +++ b/site/content/docs/v1.18/restore-resource-modifiers.md @@ -0,0 +1,187 @@ +--- +title: "Restore Resource Modifiers" +layout: docs +--- + +## Resource Modifiers +Velero provides a generic ability to modify the resources during restore by specifying json patches. The json patches are applied to the resources before they are restored. The json patches are specified in a configmap and the configmap is referenced in the restore command. + +**Creating resource Modifiers** + +Below is the two-step of using resource modifiers to modify the resources during restore. +1. Creating resource modifiers configmap + + You need to create one configmap in Velero install namespace from a YAML file that defined resource modifiers. The creating command would be like the below: + ```bash + kubectl create cm --from-file -n velero + ``` +2. Creating a restore reference to the defined resource policies + + You can create a restore with the flag `--resource-modifier-configmap`, which will apply the defined resource modifiers to the current restore. The creating command would be like the below: + ```bash + velero restore create --resource-modifier-configmap + ``` + +**YAML template** + +- Yaml template: +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: persistentvolumeclaims + resourceNameRegex: "^mysql.*$" + namespaces: + - bar + - foo + labelSelector: + matchLabels: + foo: bar + patches: + - operation: replace + path: "/spec/storageClassName" + value: "premium" + - operation: remove + path: "/metadata/labels/test" + ``` + +- The above configmap will apply the JSON Patch to all the PVCs in the namespaces bar and foo with name starting with mysql and match label `foo: bar`. The JSON Patch will replace the storageClassName with "premium" and remove the label "test" from the PVCs. +- Note that the Namespace here is the original namespace of the backed up resource, not the new namespace where the resource is going to be restored. +- You can specify multiple JSON Patches for a particular resource. The patches will be applied in the order specified in the configmap. A subsequent patch is applied in order and if multiple patches are specified for the same path, the last patch will override the previous patches. +- You can specify multiple resourceModifierRules in the configmap. The rules will be applied in the order specified in the configmap. + +### Operations supported by the JSON Patch RFC: +- add +- remove +- replace +- move +- copy +- test (covered below) + +### Advanced scenarios +#### **Conditional patches using test operation** + The `test` operation can be used to check if a particular value is present in the resource. If the value is present, the patch will be applied. If the value is not present, the patch will not be applied. This can be used to apply a patch only if a particular value is present in the resource. For example, if you wish to change the storage class of a PVC only if the PVC is using a particular storage class, you can use the following configmap. +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: persistentvolumeclaims + resourceNameRegex: ".*" + namespaces: + - bar + - foo + patches: + - operation: test + path: "/spec/storageClassName" + value: "premium" + - operation: replace + path: "/spec/storageClassName" + value: "standard" +``` + +#### **Other examples** +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: deployments.apps + resourceNameRegex: "^test-.*$" + namespaces: + - bar + - foo + patches: + # Dealing with complex values by escaping the yaml + - operation: add + path: "/spec/template/spec/containers/0" + value: "{\"name\": \"nginx\", \"image\": \"nginx:1.14.2\", \"ports\": [{\"containerPort\": 80}]}" + # Copy Operator + - operation: copy + from: "/spec/template/spec/containers/0" + path: "/spec/template/spec/containers/1" +``` + +**Note:** +- The design and approach is inspired from [kubectl patch command](https://github.com/kubernetes/kubectl/blob/0a61782351a027411b8b45b1443ec3dceddef421/pkg/cmd/patch/patch.go#L102C2-L104C1) +- Update a container's image using a json patch with positional arrays +kubectl patch pod valid-pod -type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"new image"}]' +- Before creating the resource modifier yaml, you can try it out using kubectl patch command. The same commands should work as it is. + +#### JSON Merge Patch +You can modify a resource using JSON Merge Patch +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: pods + namespaces: + - ns1 + mergePatches: + - patchData: | + { + "metadata": { + "annotations": { + "foo": null + } + } + } +``` +- The above configmap will apply the Merge Patch to all the pods in namespace ns1 and remove the annotation `foo` from the pods. +- Both json and yaml format are supported for the patchData. +- For more details, please refer to [this doc](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/) + +#### Strategic Merge Patch +You can modify a resource using Strategic Merge Patch +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: pods + resourceNameRegex: "^my-pod$" + namespaces: + - ns1 + strategicPatches: + - patchData: | + { + "spec": { + "containers": [ + { + "name": "nginx", + "image": "repo2/nginx" + } + ] + } + } +``` +- The above configmap will apply the Strategic Merge Patch to the pod with name my-pod in namespace ns1 and update the image of container nginx to `repo2/nginx`. +- Both json and yaml format are supported for the patchData. +- For more details, please refer to [this doc](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/) + + +### Conditional Patches in ALL Patch Types +A new field `matches` is added in conditions to support conditional patches. + +Example of matches in conditions +```yaml +version: v1 +resourceModifierRules: +- conditions: + groupResource: persistentvolumeclaims.storage.k8s.io + matches: + - path: "/spec/storageClassName" + value: "premium" + mergePatches: + - patchData: | + { + "metadata": { + "annotations": { + "foo": null + } + } + } +``` +- The above configmap will apply the Merge Patch to all the PVCs in all namespaces with storageClassName premium and remove the annotation `foo` from the PVCs. +- You can specify multiple rules in the `matches` list. The patch will be applied only if all the matches are satisfied. + +### Wildcard Support for GroupResource +The user can specify a wildcard for groupResource in the conditions' struct. This will allow the user to apply the patches for all the resources of a particular group or all resources in all groups. For example, `*.apps` will apply to all the resources in the `apps` group, `*` will apply to all the resources in core group, `*.*` will apply to all the resources in all groups. +- If both `*.groupName` and `namespaces` are specified, the patches will be applied to all the namespaced resources in this group in the specified namespaces and all the cluster resources in this group. \ No newline at end of file diff --git a/site/content/docs/v1.18/run-locally.md b/site/content/docs/v1.18/run-locally.md new file mode 100644 index 000000000..897ee1893 --- /dev/null +++ b/site/content/docs/v1.18/run-locally.md @@ -0,0 +1,53 @@ +--- +title: "Run Velero locally in development" +layout: docs +--- + +Running the Velero server locally can speed up iterative development. This eliminates the need to rebuild the Velero server +image and redeploy it to the cluster with each change. + +## Run Velero locally with a remote cluster + +Velero runs against the Kubernetes API server as the endpoint (as per the `kubeconfig` configuration), so both the Velero server and client use the same `client-go` to communicate with Kubernetes. This means the Velero server can be run locally just as functionally as if it was running in the remote cluster. + +### Prerequisites + +When running Velero, you will need to ensure that you set up all of the following: + +* Appropriate RBAC permissions in the cluster + * Read access for all data from the source cluster and namespaces + * Write access to the target cluster and namespaces +* Cloud provider credentials + * Read/write access to volumes + * Read/write access to object storage for backup data +* A [BackupStorageLocation][20] object definition for the Velero server +* (Optional) A [VolumeSnapshotLocation][21] object definition for the Velero server, to take PV snapshots + +### 1. Install Velero + +See documentation on how to install Velero in some specific providers: [Install overview][22] + +### 2. Scale deployment down to zero + +After you use the `velero install` command to install Velero into your cluster, you scale the Velero deployment down to 0 so it is not simultaneously being run on the remote cluster and potentially causing things to get out of sync: + +`kubectl scale --replicas=0 deployment velero -n velero` + +#### 3. Start the Velero server locally + +* To run the server locally, use the full path according to the binary you need. Example, if you are on a Mac, and using `AWS` as a provider, this is how to run the binary you built from source using the full path: `AWS_SHARED_CREDENTIALS_FILE= ./_output/bin/darwin/amd64/velero`. Alternatively, you may add the `velero` binary to your `PATH`. + +* Start the server: `velero server [CLI flags]`. The following CLI flags may be useful to customize, but see `velero server --help` for full details: + * `--log-level`: set the Velero server's log level (default `info`, use `debug` for the most logging) + * `--kubeconfig`: set the path to the kubeconfig file the Velero server uses to talk to the Kubernetes apiserver (default `$KUBECONFIG`) + * `--namespace`: the set namespace where the Velero server should look for backups, schedules, restores (default `velero`) + * `--plugin-dir`: set the directory where the Velero server looks for plugins (default `/plugins`) + * The `--plugin-dir` flag requires the plugin binary to be present locally, and should be set to the directory containing this built binary. + * `--metrics-address`: set the bind address and port where Prometheus metrics are exposed (default `:8085`) + +[15]: https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#the-shared-credentials-file +[16]: https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable +[18]: https://eksctl.io/ +[20]: api-types/backupstoragelocation.md +[21]: api-types/volumesnapshotlocation.md +[22]: basic-install.md diff --git a/site/content/docs/v1.18/self-signed-certificates.md b/site/content/docs/v1.18/self-signed-certificates.md new file mode 100644 index 000000000..6d9af90c1 --- /dev/null +++ b/site/content/docs/v1.18/self-signed-certificates.md @@ -0,0 +1,157 @@ +--- +title: "Use Velero with a storage provider secured by a self-signed certificate" +layout: docs +--- + +If you are using an S3-Compatible storage provider that is secured with a self-signed certificate, connections to the object store may fail with a `certificate signed by unknown authority` message. +To proceed, provide a certificate bundle when adding the storage provider. + +## Trusting a self-signed certificate during installation + +When using the `velero install` command, you can use the `--cacert` flag to provide a path +to a PEM-encoded certificate bundle to trust. + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --cacert +``` + +Velero will then automatically use the provided CA bundle to verify TLS connections to +that storage provider when backing up and restoring. + +## Trusting a self-signed certificate using Kubernetes Secrets (Recommended) + +The recommended approach for managing CA certificates is to store them in a Kubernetes Secret and reference them in the BackupStorageLocation using `caCertRef`. This provides better security and easier certificate management: + +1. Create a Secret containing your CA certificate: + +```bash +kubectl create secret generic storage-ca-cert \ + --from-file=ca-bundle.crt= \ + -n velero +``` + +2. Create or update your BackupStorageLocation to reference the Secret: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + provider: + objectStorage: + bucket: + caCertRef: + name: storage-ca-cert + key: ca-bundle.crt + # ... other configuration +``` + +### Benefits of using Secrets + +- **Security**: Certificates are stored encrypted in etcd +- **Certificate Rotation**: Update the Secret to rotate certificates without modifying the BackupStorageLocation +- **RBAC**: Control access to certificates using Kubernetes RBAC +- **Separation of Concerns**: Keep sensitive certificate data separate from configuration + +## Trusting a self-signed certificate with the Velero client + +**Note**: As of Velero v1.15, the CLI automatically discovers certificates configured in the BackupStorageLocation. If you have configured certificates using either `caCert` (deprecated) or `caCertRef` (recommended) in your BSL, you no longer need to specify the `--cacert` flag for backup describe, download, or logs commands. + +### Automatic Certificate Discovery + +The Velero CLI automatically discovers and uses CA certificates from the BackupStorageLocation configuration. The resolution order is: + +1. **`--cacert` flag** (if provided) - Takes highest precedence +2. **`caCertRef`** - References a Secret containing the certificate (recommended) +3. **`caCert`** - Inline certificate in the BSL (deprecated) + +Examples: + +```bash +# Automatic discovery (no flag needed if BSL has caCertRef or caCert configured) +velero backup describe my-backup +velero backup download my-backup +velero backup logs my-backup + +# Manual override (takes precedence over BSL configuration) +velero backup describe my-backup --cacert +``` + +### Configuring CA Certificates in BackupStorageLocation + +You can configure CA certificates in the BackupStorageLocation using either method: + +1. **Using `caCertRef` (Recommended)**: + + ```yaml + apiVersion: velero.io/v1 + kind: BackupStorageLocation + metadata: + name: default + namespace: velero + spec: + provider: aws + objectStorage: + bucket: velero-backups + caCertRef: + name: storage-ca-cert + key: ca-bundle.crt + config: + region: us-east-1 + ``` + +2. **Using inline `caCert` (Deprecated)**: + + ```yaml + apiVersion: velero.io/v1 + kind: BackupStorageLocation + metadata: + name: default + namespace: velero + spec: + provider: aws + objectStorage: + bucket: velero-backups + caCert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi4uLiAoYmFzZTY0IGVuY29kZWQgY2VydGlmaWNhdGUgY29udGVudCkgLi4uCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + config: + region: us-east-1 + ``` + +When the CA certificate is configured in the BackupStorageLocation using either method, Velero client commands will automatically discover and use it without requiring the `--cacert` flag. + +## Error with client certificate with custom S3 server + +In case you are using a custom S3-compatible server, you may encounter that the backup fails with an error similar to one below. + +```text +rpc error: code = Unknown desc = RequestError: send request failed caused by: +Get https://minio.com:3000/k8s-backup-bucket?delimiter=%2F&list-type=2&prefix=: remote error: tls: alert(116) +``` + +Error 116 represents certificate required as seen here in [error codes](https://datatracker.ietf.org/doc/html/rfc8446#appendix-B.2). +Velero as a client does not include its certificate while performing SSL handshake with the server. +From [TLS 1.3 spec](https://tools.ietf.org/html/rfc8446), verifying client certificate is optional on the server. +You will need to change this setting on the server to make it work. + +## Skipping TLS verification + +**Note:** The `--insecure-skip-tls-verify` flag is insecure and susceptible to man-in-the-middle attacks and meant to help your testing and developing scenarios in an on-premises environment. Using this flag in production is not recommended. + +Velero provides a way for you to skip TLS verification on the object store when using the [AWS provider plugin](https://github.com/vmware-tanzu/velero-plugin-for-aws) or [File System Backup](file-system-backup.md) by passing the `--insecure-skip-tls-verify` flag with the following Velero commands, + +* velero backup describe +* velero backup download +* velero backup logs +* velero restore describe +* velero restore log + +If true, the object store's TLS certificate will not be checked for validity before Velero or backup repository connects to the object storage. You can permanently skip TLS verification for an object store by setting `Spec.Config.InsecureSkipTLSVerify` to true in the [BackupStorageLocation](api-types/backupstoragelocation.md) CRD. + +Note that Velero's File System Backup uses Restic or Kopia to do data transfer between object store and Kubernetes cluster disks. This means that when you specify `--insecure-skip-tls-verify` in Velero operations that involve File System Backup, Velero will convey this information to Restic or Kopia. For example, for Restic, Velero will add the Restic global command parameter `--insecure-tls` to Restic commands. diff --git a/site/content/docs/v1.18/start-contributing.md b/site/content/docs/v1.18/start-contributing.md new file mode 100644 index 000000000..5c32825ad --- /dev/null +++ b/site/content/docs/v1.18/start-contributing.md @@ -0,0 +1,34 @@ +--- +title: "Start contributing" +layout: docs +--- + +## Before you start + +* Please familiarize yourself with the [Code of Conduct][1] before contributing. +* Also, see [CONTRIBUTING.md][2] for instructions on the developer certificate of origin that we require. + +## Creating a design doc + +Having a high level design document with the proposed change and the impacts helps the maintainers evaluate if a major change should be incorporated. + +To make a design pull request, you can copy the template found in the `design/_template.md` file into a new Markdown file. + +## Finding your way around + +You may join the Velero community and contribute in many different ways, including helping us design or test new features. For any significant feature we consider adding, we start with a design document. You may find a list of in progress new designs here: https://github.com/vmware-tanzu/velero/pulls?q=is%3Aopen+is%3Apr+label%3ADesign. Feel free to review and help us with your input. + +You can also vote on issues using :+1: and :-1:, as explained in our [Feature enhancement request][3] and [Bug issue][4] templates. This will help us quantify importance and prioritize issues. + +For information on how to connect with our maintainers and community, join our online meetings, or find good first issues, start on our [Velero community](https://velero.io/community/) page. + +Please browse our list of resources, including a playlist of past online community meetings, blog posts, and other resources to help you get familiar with our project: [Velero resources](https://velero.io/resources/). + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, please use the navigation on the left under `Contribute`. + +[1]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/CODE_OF_CONDUCT.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/CONTRIBUTING.md +[3]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/.github/ISSUE_TEMPLATE/feature-enhancement-request.md +[4]: https://github.com/vmware-tanzu/velero/blob/v1.18.0/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/site/content/docs/v1.18/style-guide.md b/site/content/docs/v1.18/style-guide.md new file mode 100644 index 000000000..5fb5e55dd --- /dev/null +++ b/site/content/docs/v1.18/style-guide.md @@ -0,0 +1,345 @@ +--- +title: "Documentation Style Guide" +layout: docs +--- + +_This style guide is adapted from the [Kubernetes style guide](https://kubernetes.io/docs/contribute/style/style-guide/)._ + +This page outlines writing style guidelines for the Velero documentation and you should use this page as a reference you write or edit content. Note that these are guidelines, not rules. Use your best judgment as you write documentation, and feel free to propose changes to these guidelines. Changes to the style guide are made by the Velero maintainers as a group. To propose a change or addition create an issue/PR, or add a suggestion to the [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA) and attend the meeting to participate in the discussion. + +The Velero documentation uses the [kramdown](https://kramdown.gettalong.org/) Markdown renderer. + +## Content best practices +### Use present tense + +{{< table caption="Do and Don't - Use present tense" >}} +|Do|Don't| +|--- |--- | +|This `command` starts a proxy.|This command will start a proxy.| +{{< /table >}} + +Exception: Use future or past tense if it is required to convey the correct meaning. + +### Use active voice + +{{< table caption="Do and Don't - Use active voice" >}} +|Do|Don't| +|--- |--- | +|You can explore the API using a browser.|The API can be explored using a browser.| +|The YAML file specifies the replica count.|The replica count is specified in the YAML file.| +{{< /table >}} + +Exception: Use passive voice if active voice leads to an awkward sentence construction. + +### Use simple and direct language + +Use simple and direct language. Avoid using unnecessary phrases, such as saying "please." + +{{< table caption="Do and Don't - Use simple and direct language" >}} +|Do|Don't| +|--- |--- | +|To create a ReplicaSet, ...|In order to create a ReplicaSet, ...| +|See the configuration file.|Please see the configuration file.| +|View the Pods.|With this next command, we'll view the Pods.| +{{< /table >}} + +### Address the reader as "you" + +{{< table caption="Do and Don't - Addressing the reader" >}} +|Do|Don't| +|--- |--- | +|You can create a Deployment by ...|We'll create a Deployment by ...| +|In the preceding output, you can see...|In the preceding output, we can see ...| +{{< /table >}} + +### Avoid Latin phrases + +Prefer English terms over Latin abbreviations. + +{{< table caption="Do and Don't - Avoid Latin phrases" >}} +|Do|Don't| +|--- |--- | +|For example, ...|e.g., ...| +|That is, ...|i.e., ...| +{{< /table >}} + +Exception: Use "etc." for et cetera. + +## Patterns to avoid + + +### Avoid using "we" + +Using "we" in a sentence can be confusing, because the reader might not know +whether they're part of the "we" you're describing. + +{{< table caption="Do and Don't - Avoid using we" >}} +|Do|Don't| +|--- |--- | +|Version 1.4 includes ...|In version 1.4, we have added ...| +|Kubernetes provides a new feature for ...|We provide a new feature ...| +|This page teaches you how to use Pods.|In this page, we are going to learn about Pods.| +{{< /table >}} + +### Avoid jargon and idioms + +Many readers speak English as a second language. Avoid jargon and idioms to help them understand better. + +{{< table caption="Do and Don't - Avoid jargon and idioms" >}} +|Do|Don't| +|--- |--- | +|Internally, ...|Under the hood, ...| +|Create a new cluster.|Turn up a new cluster.| +{{< /table >}} + +### Avoid statements about the future or that will soon be out of date + +Avoid making promises or giving hints about the future. If you need to talk about +a beta feature, put the text under a heading that identifies it as beta +information. + +Also avoid words like “recently”, "currently" and "new." A feature that is new today might not be +considered new in a few months. + +{{< table caption="Do and Don't - Avoid statements that will soon be out of date" >}} +|Do|Don't| +|--- |--- | +|In version 1.4, ...|In the current version, ...| +|The Federation feature provides ...|The new Federation feature provides ...| +{{< /table >}} + +### Language + +This documentation uses U.S. English spelling and grammar. + +## Documentation formatting standards + +### Use camel case for API objects + +When you refer to an API object, use the same uppercase and lowercase letters +that are used in the actual object name. Typically, the names of API +objects use +[camel case](https://en.wikipedia.org/wiki/Camel_case). + +Don't split the API object name into separate words. For example, use +PodTemplateList, not Pod Template List. + +Refer to API objects without saying "object," unless omitting "object" +leads to an awkward sentence construction. + +{{< table caption="Do and Don't - Do and Don't - API objects" >}} +|Do|Don't| +|--- |--- | +|The Pod has two containers.|The pod has two containers.| +|The Deployment is responsible for ...|The Deployment object is responsible for ...| +|A PodList is a list of Pods.|A Pod List is a list of pods.| +|The two ContainerPorts ...|The two ContainerPort objects ...| +|The two ContainerStateTerminated objects ...|The two ContainerStateTerminateds ...| +{{< /table >}} + +### Use angle brackets for placeholders + +Use angle brackets for placeholders. Tell the reader what a placeholder represents. + +1. Display information about a Pod: + + kubectl describe pod -n + + If the pod is in the default namespace, you can omit the '-n' parameter. + +### Use bold for user interface elements + +{{< table caption="Do and Don't - Bold interface elements" >}} +|Do|Don't| +|--- |--- | +|Click **Fork**.|Click "Fork".| +|Select **Other**.|Select "Other".| +{{< /table >}} + +### Use italics to define or introduce new terms + +{{< table caption="Do and Don't - Use italics for new terms" >}} +|Do|Don't| +|--- |--- | +|A _cluster_ is a set of nodes ...|A "cluster" is a set of nodes ...| +|These components form the _control plane_.|These components form the **control plane**.| +{{< /table >}} + +### Use code style for filenames, directories, paths, object field names and namespaces +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|Open the `envars.yaml` file.|Open the envars.yaml file.| +|Go to the `/docs/tutorials` directory.|Go to the /docs/tutorials directory.| +|Open the `/_data/concepts.yaml` file.|Open the /\_data/concepts.yaml file.| +{{< /table >}} + + +### Use punctuation inside quotes +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|events are recorded with an associated "stage."|events are recorded with an associated "stage".| +|The copy is called a "fork."|The copy is called a "fork".| +{{< /table >}} + +Exception: When the quoted word is a user input. + +Example: +* My user ID is “IM47g”. +* Did you try the password “mycatisawesome”? + +## Inline code formatting + + +### Use code style for inline code and commands + +For inline code in an HTML document, use the `` tag. In a Markdown +document, use the backtick (`` ` ``). + +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|The `kubectl run` command creates a Deployment.|The "kubectl run" command creates a Deployment.| +|For declarative management, use `kubectl apply`.|For declarative management, use "kubectl apply".| +|Use single backticks to enclose inline code. For example, `var example = true`.|Use two asterisks (`**`) or an underscore (`_`) to enclose inline code. For example, **var example = true**.| +|Use triple backticks (\`\`\`) before and after a multi-line block of code for fenced code blocks.|Use multi-line blocks of code to create diagrams, flowcharts, or other illustrations.| +|Use meaningful variable names that have a context.|Use variable names such as 'foo','bar', and 'baz' that are not meaningful and lack context.| +|Remove trailing spaces in the code.|Add trailing spaces in the code, where these are important, because a screen reader will read out the spaces as well.| +{{< /table >}} + +### Starting a sentence with a component tool or component name + +{{< table caption="Do and Don't - Starting a sentence with a component tool or component name" >}} +|Do|Don't| +|--- |--- | +|The `kubeadm` tool bootstraps and provisions machines in a cluster.|`kubeadm` tool bootstraps and provisions machines in a cluster.| +|The kube-scheduler is the default scheduler for Kubernetes.|kube-scheduler is the default scheduler for Kubernetes.| +{{< /table >}} + +### Use normal style for string and integer field values + +For field values of type string or integer, use normal style without quotation marks. + +{{< table caption="Do and Don't - Use normal style for string and integer field values" >}} +|Do|Don't| +|--- |--- | +|Set the value of `imagePullPolicy` to `Always`.|Set the value of `imagePullPolicy` to "Always".| +|Set the value of `image` to `nginx:1.16`.|Set the value of `image` to nginx:1.16.| +|Set the value of the `replicas` field to `2`.|Set the value of the `replicas` field to 2.| +{{< /table >}} + +## Code snippet formatting + + +### Don't include the command prompt + +{{< table caption="Do and Don't - Don't include the command prompt" >}} +|Do|Don't| +|--- |--- | +|kubectl get pods|$ kubectl get pods| +{{< /table >}} + +### Separate commands from output + +Verify that the Pod is running on your chosen node: + +``` +kubectl get pods --output=wide +``` + +The output is similar to this: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +nginx 1/1 Running 0 13s 10.200.0.4 worker0 +``` + +## Velero.io word list + + +A list of Velero-specific terms and words to be used consistently across the site. + +{{< table caption="Velero.io word list" >}} +|Term|Usage| +|--- |--- | +|Kubernetes|Kubernetes should always be capitalized.| +|Docker|Docker should always be capitalized.| +|Velero|Velero should always be capitalized.| +|VMware|VMware should always be correctly capitalized.| +|On-premises|On-premises or on-prem rather than on-premise or other variations.| +|Backup|Backup for noun or adjective, rather than back-up, back up or other variations.| +|Back up|Back up for verb, rather than other variations.| +|Plugin|Plugin rather than plug-in or other variations.| +|Allowlist|Use allowlist instead of whitelist.| +|Denylist|Use denylist instead of blacklist.| +{{< /table >}} + +## Markdown elements + +### Headings +People accessing this documentation may use a screen reader or other assistive technology (AT). [Screen readers](https://en.wikipedia.org/wiki/Screen_reader) are linear output devices, they output items on a page one at a time. If there is a lot of content on a page, you can use headings to give the page an internal structure. A good page structure helps all readers to easily navigate the page or filter topics of interest. + +{{< table caption="Do and Don't - Headings" >}} +|Do|Don't| +|--- |--- | +|Include a title on each page or blog post.|Include more than one title headings (#) in a page.| +|Use ordered headings to provide a meaningful high-level outline of your content.|Use headings level 4 through 6, unless it is absolutely necessary. If your content is that detailed, it may need to be broken into separate articles.| +|Use sentence case for headings. For example, **Extend kubectl with plugins**|Use title case for headings. For example, **Extend Kubectl With Plugins**| +{{< /table >}} + +### Paragraphs + +{{< table caption="Do and Don't - Paragraphs" >}} + +|Do|Don't| +|--- |--- | +|Try to keep paragraphs under 6 sentences.|Write long-winded paragraphs.| +|Use three hyphens (`---`) to create a horizontal rule for breaks in paragraph content.|Use horizontal rules for decoration.| +{{< /table >}} + +### Links + +{{< table caption="Do and Don't - Links" >}} +|Do|Don't| +|--- |--- | +|Write hyperlinks that give you context for the content they link to. For example: Certain ports are open on your machines. See [check required ports](#check-required-ports) for more details.|Use ambiguous terms such as “click here”. For example: Certain ports are open on your machines. See [here](#check-required-ports) for more details.| +|Write Markdown-style links: `[link text](URL)`. For example: `[community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA)` and the output is [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA).|Write HTML-style links: `Visit our tutorial!`| +{{< /table >}} + + +### Lists + +Group items in a list that are related to each other and need to appear in a specific order or to indicate a correlation between multiple items. When a screen reader comes across a list—whether it is an ordered or unordered list—it will be announced to the user that there is a group of list items. The user can then use the arrow keys to move up and down between the various items in the list. +Website navigation links can also be marked up as list items; after all they are nothing but a group of related links. + + - End each item in a list with a period if one or more items in the list are complete sentences. For the sake of consistency, normally either all items or none should be complete sentences. + + - Ordered lists that are part of an incomplete introductory sentence can be in lowercase and punctuated as if each item was a part of the introductory sentence. + + - Use the number one (`1.`) for ordered lists. + + - Use (`+`), (`*`), or (`-`) for unordered lists - be consistent within the same document. + + - Leave a blank line after each list. + + - Indent nested lists with four spaces (for example, ⋅⋅⋅⋅). + + - List items may consist of multiple paragraphs. Each subsequent paragraph in a list item must be indented by either four spaces or one tab. + +### Tables + +The semantic purpose of a data table is to present tabular data. Sighted users can quickly scan the table but a screen reader goes through line by line. A table [caption](https://www.w3schools.com/tags/tag_caption.asp) is used to create a descriptive title for a data table. Assistive technologies (AT) use the HTML table caption element to identify the table contents to the user within the page structure. + +If you need to create a table, create the table in markdown and use the table [Hugo shortcode](https://gohugo.io/content-management/shortcodes/) to include a caption. + +``` +{{}} +Parameter | Description | Default +:---------|:------------|:------- +`timeout` | The timeout for requests | `30s` +`logLevel` | The log level for log output | `INFO` +{{< /table */>}} + +``` +**Note:** This shortcode does not support markdown reference-style links. Use inline-style links in tables. See more information about [markdown link styles](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#links). diff --git a/site/content/docs/v1.18/support-process.md b/site/content/docs/v1.18/support-process.md new file mode 100644 index 000000000..e0a292aa3 --- /dev/null +++ b/site/content/docs/v1.18/support-process.md @@ -0,0 +1,43 @@ +--- +title: "Support Process" +layout: docs +--- + + +Velero provides best effort support through the process on this page for the current version of Velero and n-1 Velero version, including all patch releases in the supported minor releases. For example, if the current version is 1.9, the Velero maintainers would offer best effort support for v1.9 and v1.8. If you have a question about a previous Velero version (for example, 1.7), please note that maintainers may ask you to upgrade to a supported version before doing any investigation into your issue. + +For more information about Velero testing and supported Kubernetes versions, see Velero's [compatibility matrix](https://github.com/vmware-tanzu/velero/blob/v1.18.0/README.md#velero-compatibility-matrix). + + +## Weekly Rotation + +The Velero maintainers use a weekly rotation to manage community support. Each week, a different maintainer is the point person for responding to incoming support issues via Slack, GitHub, and the Google group. The point person is *not* expected to be on-call 24x7. Instead, they choose one or more hour(s) per day to be available/responding to incoming issues. They will communicate to the community what that time slot will be each week. + +## Start of Week + +We will update the public Slack channel's topic to indicate that you are the point person for the week, and what hours you'll be available. + +## During the Week + +### Where we will monitor +- `#velero-users` and `#velero-dev` public Slack channels in Kubernetes org +- [all Velero-related repos][0] in GitHub (`velero`, `velero-plugin-for-[aws|gcp|microsoft-azure|csi]`, `helm-charts`) + +### GitHub issue flow + +Generally speaking, new GitHub issues will fall into one of several categories. We use the following process for each: + +1. **Feature request** + - Label the issue with `kind/requirement` +1. **Bug** + - Label the issue with `Bug` +1. **User question/problem** that does not clearly fall into one of the previous categories + - Add comments as you go, so both the user and future support people have as much context as possible + - Use the `Needs investigation` label to indicate additional work needed to truly understand the problem or the root cause. + - Use the `Needs Info` label to indicate an issue is waiting for information from the user. Remove/re-add the label as needed. + - For the issues that require reproduction, add label `Needs reproduction` or `status/not-reproducible` to indicate the status. + - If you resolve the issue with the user, close it out + - If the issue ends up being a feature request or a bug, update the title and follow the appropriate process for it + - If the reporter becomes unresponsive after multiple pings, close out the issue due to inactivity and comment that the user can always reach out again as needed + +[0]: https://github.com/vmware-tanzu?q=velero&type=&language= diff --git a/site/content/docs/v1.18/supported-configmaps/_index.md b/site/content/docs/v1.18/supported-configmaps/_index.md new file mode 100644 index 000000000..b6fc07992 --- /dev/null +++ b/site/content/docs/v1.18/supported-configmaps/_index.md @@ -0,0 +1,10 @@ +--- +layout: docs +title: Supported ConfigMaps +--- + +Here's a list of ConfigMaps that Velero support, but their life cycle control are out of Velero's scope. + +* [node-agent ConfigMap][1] + +[1]: node-agent-configmap.md \ No newline at end of file diff --git a/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md new file mode 100644 index 000000000..f17df3815 --- /dev/null +++ b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md @@ -0,0 +1,620 @@ +--- +title: "Node-agent Configuration" +layout: docs +--- + +## Glossary +**Data Mover Pods**: Data Mover Pods are: +* Pods launched by Velero built-in data mover to run the data transfer during [CSI Snapshot Data Movement](../csi-snapshot-data-movement.md), i.e., DataUpload pod and DataDownload pod. +* Pods launched by Velero to run the data transfer during [File System backup](../file-system-backup.md), i.e., PodVolumeBackup pod and PodVolumeRestore pod. + +## Overview +The Velero node-agent is a DaemonSet that hosts modules for completing backup and restore operations, including file system backup/restore and CSI snapshot data movement. This document provides comprehensive configuration options for the ConfigMap provisioned by node-agent's `--node-agent-configmap` parameter. + +Node-agent puts advanced configurations of Data Mover Pods into a ConfigMap that contains JSON configuration. The ConfigMap should be created in the same namespace where Velero is installed, and its name is specified using the `--node-agent-configmap` parameter. + +## Creating and Managing the ConfigMap + +**Notice**: The ConfigMap's life cycle control is out of the scope of Velero. +Users need to create and maintain the ConfigMap themselves. + +**Important**: The node-agent server checks configurations at startup time. After editing the ConfigMap, restart the node-agent DaemonSet for changes to take effect. +`kubectl rollout restart -n daemonset/node-agent` + +To create the ConfigMap: +1. Save your configuration to a JSON file +2. Create the ConfigMap: +```bash +kubectl create cm -n velero --from-file= +``` + +### Specify during install +The ConfigMap name can be specified during Velero installation: +```bash +velero install --node-agent-configmap= +``` + +### Specify after install +To apply the ConfigMap to the node-agent DaemonSet: +```bash +kubectl edit ds node-agent -n velero +``` + +Add the ConfigMap reference to the container arguments: +```yaml +spec: + template: + spec: + containers: + - args: + - --node-agent-configmap= +``` + +## Configuration Sections +### Load Concurrency (`loadConcurrency`) + +Controls the concurrent number of Data Mover Pods per node to optimize resource usage and performance. + +The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and DataDownload pods. + +#### Configuration Options +- **`globalConfig`**: Set default concurrent number applied to all nodes. +- **`perNodeConfig`**: Set different concurrent numbers for specific nodes using label selectors. +- **`prepareQueueLength`**: Set the max number of intermediate backup/restore pods under pending status. + +#### Global Configuration +Sets a default concurrent number applied to all nodes: +```json +{ + "loadConcurrency": { + "globalConfig": 2 + } +} +``` + +#### Per-node Configuration +Specify different concurrent numbers for specific nodes using label selectors: +```json +{ + "loadConcurrency": { + "globalConfig": 2, + "perNodeConfig": [ + { + "nodeSelector": { + "matchLabels": { + "kubernetes.io/hostname": "node1" + } + }, + "number": 3 + }, + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + } + }, + "number": 5 + } + ] + } +} +``` + +- **Range**: Starts from 1 (no concurrency per node), no upper limit +- **Priority**: Per-node configuration overrides global configuration +- **Conflicts**: If a node matches multiple rules, the smallest number is used +- **Default**: 1 if not specified + +**Use Cases:** +- Increase concurrency on nodes with more resources +- Reduce concurrency on nodes with limited resources or critical workloads +- Prevent OOM kills and resource contention + +#### PrepareQueueLength +The prepare queue length controls the maximum number of `DataUpload`/`DataDownload`/`PodVolumeBackup`/`PodVolumeRestore` CRs under the preparation statuses but are not yet processed by any node, which means the CR corresponding pod is pending state. + +If there are thousands of intermediate backup/restore pods, and without this control, they start at the same time, then causing a big burden on the k8s API server. + +```json +{ + "loadConcurrency": { + "prepareQueueLength": 10 + } +} +``` + +- **Range**: Starts from 1 (for all node-agent pods), no upper limit +- **Scope**: This parameter controls all PVB, PVR, DataUpload, and DataDownload pods pending number. It applies to all node-agent pods. +- **Default**: No limitation if not specified + +**Use Cases:** +- Prevent too much workload pods are created, but cannot start. +- Limit resource consumption from intermediate objects (PVCs, VolumeSnapshots, etc.) +- Prevent resource exhaustion when backup/restore concurrency is limited +- Balance between parallelism and resource usage + +**Affected CR Phases:** +- DataUpload/DataDownload CRs in `Accepted` or `Prepared` phases +- PodVolumeBackup/PodVolumeRestore CRs in preparation phases + +### Node Selection (`loadAffinity`) +Constrains which nodes can run Data Mover Pods for CSI Snapshot Data Movement using affinity and anti-affinity rules. + +The configurations work for DataUpload, and DataDownload pods. + +For detailed information, see [Node Selection for Data Movement](../data-movement-node-selection.md). + +Example: + +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + }, + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "values": ["node-1", "node-2", "node-3"], + "operator": "In" + }, + { + "key": "critical-workload", + "operator": "DoesNotExist" + } + ] + } + } + ] +} +``` + +#### Configuration Options +- **`nodeSelector`**: Specify DataUpload and DataDownload pods can run on which nodes. +- **`storageClass`**: Filter DataUpload and DataDownload pods on the PVC's StorageClass. If not set, its corresponding `nodeSelector` applies to all DataUpload and DataDownload pods. + +**Important Limitations:** +- Only the first element without `storageClass` parameter in the `loadAffinity` array is used for general node selection +- Additional elements are only considered if they have a `storageClass` field +- To combine multiple conditions, use both `matchLabels` and `matchExpressions` in a single element + +**Use Cases:** +- Prevent data movement on nodes with critical workloads +- Run data movement only on nodes with sufficient resources +- Ensure data movement runs only on nodes where storage is accessible +- Comply with topology constraints + + +#### Storage Class Specific Selection +Configure different node selection rules for specific storage classes: +* For StorageClass `fast-ssd`, the first match is chosen, which is nodes with label `"environment": "production"`. +* For StorageClass `hdd`, the nodes with label `"environment": "backup"` are chosen. + +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "environment": "production" + } + }, + "storageClass": "fast-ssd" + }, + { + "nodeSelector": { + "matchLabels": { + "environment": "staging" + } + }, + "storageClass": "fast-ssd" + }, + { + "nodeSelector": { + "matchLabels": { + "environment": "backup" + } + }, + "storageClass": "hdd" + } + ] +} +``` + +### Pod Resources (`podResources`) +Configure CPU and memory resources for Data Mover Pods to optimize performance and prevent resource conflict. + +The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and DataDownload pods. + +```json +{ + "podResources": { + "cpuRequest": "1000m", + "cpuLimit": "2000m", + "memoryRequest": "1Gi", + "memoryLimit": "4Gi" + } +} +``` + +**Use Cases:** +- Limit resource consumption in resource-constrained clusters +- Guarantee resources for time-critical backup/restore operations +- Prevent OOM kills during large data transfers +- Control scheduling priority relative to production workloads + +**Values**: Must be valid Kubernetes Quantity expressions +**Validation**: Request values must not exceed limit values +**Default**: BestEffort QoS if not specified +**Failure Handling**: Invalid values cause the entire `podResources` section to be ignored + +For detailed information, see [Data Movement Pod Resource Configuration](../data-movement-pod-resource-configuration.md). + + +### Priority Class (`priorityClassName`) + +Configure the Data Mover Pods' PriorityClass. + +The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and DataDownload pods. + +#### Configuration Options +- **`priorityClassName`**: The name of the PriorityClass to assign to backup/restore pods + +Configure pod priority to control scheduling behavior: + +**High Priority** (e.g., `system-cluster-critical`): +- ✅ Faster scheduling and less likely to be preempted +- ❌ May impact production workload performance + +**Low Priority** (e.g., `low-priority`): +- ✅ Protects production workloads from resource competition +- ❌ May delay backup operations or cause preemption + +```json +{ + "priorityClassName": "low-priority" +} +``` + +**Use Cases:** +- Control scheduling priority of backup/restore operations +- Protect production workloads from resource competition +- Ensure critical backups are scheduled quickly + +### Backup PVC Configuration (`backupPVC`) + +Configure intermediate PVCs used during data movement backup operations for optimal performance. + +The configurations work for DataUpload pods. + +For detailed information, see [BackupPVC Configuration for Data Movement Backup](../data-movement-backup-pvc-configuration.md). + +#### Configuration Options +- **`storageClass`**: Alternative storage class for backup PVCs (defaults to source PVC's storage class) +- **`readOnly`**: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise `ReadWriteOnce` value will be used. +- **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. + +**Use Cases:** +- Use read-only volumes for faster snapshot-to-volume conversion +- Use dedicated storage classes optimized for backup operations +- Reduce replica count for intermediate backup volumes +- Comply with SELinux requirements in secured environments + +**Important Notes:** +- Ensure specified storage classes exist and support required access modes +- In SELinux environments, always set `spcNoRelabeling: true` when using `readOnly: true` +- Failures result in DataUpload CR staying in `Accepted` phase until timeout (30m default) + +#### Storage Class Mapping +`storageClass` specifies alternative storage class for backup PVCs (defaults to source PVC's storage class). + +Configure different backup PVC settings per source storage class: +```json +{ + "backupPVC": { + "fast-storage": { + "storageClass": "backup-storage-1" + }, + "slow-storage": { + "storageClass": "backup-storage-2" + } + } +} +``` + +#### ReadOnly and SPC Configuration + +Create BackupPVC in ReadOnly mode, which can avoid full data clone during backup process in some storage providers, such as Ceph RBD. + +```json +{ + "backupPVC": { + "source-storage-class": { + "storageClass": "backup-optimized-class", + "readOnly": true, + "spcNoRelabeling": true + } + } +} +``` + +### Restore PVC Configuration (`restorePVC`) + +Configure intermediate PVCs used by Data Mover Pods during CSI Snapshot Data Movement restore. + +The configurations work for DataDownload pods. + +```json +{ + "restorePVC": { + "ignoreDelayBinding": true + } +} +``` + +For detailed information, see [RestorePVC Configuration for Data Movement Restore](../data-movement-restore-pvc-configuration.md). + +#### Configuration Options +- **`ignoreDelayBinding`**: Ignore `WaitForFirstConsumer` binding mode constraints + +**Use Cases:** +- Improve restore parallelism by not waiting for pod scheduling +- Enable volume restore without requiring a pod to be mounted +- Work around topology constraints when you know the environment setup + +**Important Notes:** +- Use only when you understand your cluster's topology constraints +- May result in volumes provisioned on nodes where workload pods cannot be scheduled +- Works best with node selection to ensure proper node targeting + +### Privileged FS Backup and Restore (`privilegedFsBackup`) + +Add `privileged` permission in PodVolumeBackup and PodVolumeRestore created pod's `SecurityContext`, because in some k8s environments, mounting HostPath volume needs privileged permission to work. + +The configurations work for PodVolumeBackup, and PodVolumeRestore pods. + +#### Configuration Options +- **`privilegedFsBackup`**: Boolean value to enable privileged security context for file system backup/restore pods + +```json +{ + "privilegedFsBackup": true +} +``` + +**Use Cases:** +- Enable file system backup in environments requiring privileged container access +- Support HostPath volume mounting in restricted Kubernetes environments +- Comply with security policies that restrict container capabilities + +**Important Notes:** +- In v1.17+, PodVolumeBackup and PodVolumeRestore run as independent pods using HostPath volumes +- Required when cluster security policies restrict HostPath volume mounting + +For detailed information, see [Enable file system backup document](../customize-installation.md#enable-file-system-backup) + +### Cache PVC Configuration (`cachePVC`) + +Configure intermediate PVCs used for CSI Snapshot Data Movement restore operations to cache the downloaded data. + +The configurations work for DataDownload pods. + +For detailed information, see [Cache PVC Configuration for Data Movement Restore](../data-movement-cache-volume.md). + +#### Configuration Options +- **`thresholdInGB`**: Minimum backup data size (in GB) to trigger cache PVC creation during restore +- **`storageClass`**: Storage class used to create cache PVCs. + +**Use Cases:** +- Improve restore performance by caching downloaded data locally +- Reduce repeated data downloads from object storage +- Optimize restore operations for large volumes + +**Important Notes:** +- Cache PVC is only created when restored data size exceeds the threshold +- Ensure specified storage class exists and has sufficient capacity +- Cache PVCs are temporary and cleaned up after restore completion + +```json +{ + "cachePVC": { + "thresholdInGB": 1, + "storageClass": "cache-optimized-storage" + } +} +``` + +### Pod Labels Configuration (`podLabels`) + +Add customized labels for data mover pods to support third-party integrations and environment-specific requirements. + +If `podLabels` is configured, it supersedes Velero's [in-tree third-party labels](https://github.com/vmware-tanzu/velero/blob/94f64639cee09c5caaa65b65ab5f42175f41c101/pkg/util/third_party.go#L19-L21). +If `podLabels` is not configured, Velero uses the in-tree third-party labels for compatibility with common cloud providers and networking solutions. + +The configurations work for DataUpload, DataDownload, PodVolumeBackup, and PodVolumeRestore pods. + +#### Configuration Example +```json +{ + "podLabels": { + "spectrocloud.com/connection": "proxy", + "gnp/k8s-api-access": "", + "gnp/monitoring-client": "", + "np/s3-backup-backend": "", + "cp/inject-truststore": "extended" + } +} +``` + +#### Use Cases +- **Proxy Configuration**: Kubernetes environment requires proxy settings for external connections configured via labels +- **Firewall Rules**: Network policies configured based on pod labels for traffic control +- **Cloud Provider Integration**: Labels required by managed Kubernetes services (AKS, EKS, GKE) +- **Security Policy Injection**: Labels that trigger security agent or certificate injection + +#### Important Notes +- **Third-party Label Replacement**: When `podLabels` is configured, Velero's built-in in-tree labels are NOT automatically added +- **Explicit Configuration Required**: If you need both custom labels and in-tree third-party labels, explicitly include the in-tree labels in the `podLabels` configuration +- **In-tree Labels**: The default in-tree labels include support for Azure workload identity + +### Pod Annotations Configuration (`podAnnotations`) + +Add customized annotations for data mover pods to support third-party integrations and pod-level configuration. + +If `podAnnotations` is configured, it supersedes Velero's [in-tree third-party annotations](https://github.com/vmware-tanzu/velero/blob/94f64639cee09c5caaa65b65ab5f42175f41c101/pkg/util/third_party.go#L23-L25). +If `podAnnotations` is not configured, Velero uses the in-tree third-party annotations for compatibility with common cloud providers and networking solutions. + +The configurations work for DataUpload, DataDownload, PodVolumeBackup, and PodVolumeRestore pods. + +#### Configuration Example +```json +{ + "podAnnotations": { + "iam.amazonaws.com/role": "velero-backup-role", + "vault.hashicorp.com/agent-inject": "true", + "prometheus.io/scrape": "true", + "custom.company.com/environment": "production" + } +} +``` + +#### Use Cases +- **Secret Management Integration**: HashiCorp Vault or other secret managers using annotations for automatic secret injection +- **Monitoring and Observability**: Prometheus scrape configurations and other monitoring tool annotations +- **Custom Application Integration**: Company-specific annotations for operational tooling + +#### Important Notes +- **Third-party Annotation Replacement**: When `podAnnotations` is configured, Velero's built-in in-tree annotations are NOT automatically added +- **Explicit Configuration Required**: If you need both custom annotations and in-tree third-party annotations, explicitly include the in-tree annotations in the `podAnnotations` configuration +- **In-tree Annotations**: The default in-tree annotations include support for AWS IAM roles + +## Complete Configuration Example +Here's a comprehensive example showing how all configuration sections work together: + +```json +{ + "loadConcurrency": { + "globalConfig": 2, + "prepareQueueLength": 15, + "perNodeConfig": [ + { + "nodeSelector": { + "matchLabels": { + "kubernetes.io/hostname": "node1" + } + }, + "number": 3 + } + ] + }, + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "node-type": "backup" + }, + "matchExpressions": [ + { + "key": "critical-workload", + "operator": "DoesNotExist" + } + ] + } + }, + { + "nodeSelector": { + "matchLabels": { + "environment": "staging" + } + }, + "storageClass": "fast-ssd" + } + ], + "podResources": { + "cpuRequest": "500m", + "cpuLimit": "1000m", + "memoryRequest": "1Gi", + "memoryLimit": "2Gi" + }, + "priorityClassName": "backup-priority", + "backupPVC": { + "fast-storage": { + "storageClass": "backup-optimized-class", + "readOnly": true, + "spcNoRelabeling": true + }, + "slow-storage": { + "storageClass": "backup-storage-2" + } + }, + "restorePVC": { + "ignoreDelayBinding": true + }, + "privilegedFsBackup": true, + "cachePVC": { + "thresholdInGB": 1, + "storageClass": "cache-optimized-storage" + }, + "podLabels": { + "spectrocloud.com/connection": "proxy", + "gnp/k8s-api-access": "", + "gnp/monitoring-client": "", + "np/s3-backup-backend": "", + "cp/inject-truststore": "extended" + }, + "podAnnotations": { + "iam.amazonaws.com/role": "velero-backup-role", + "vault.hashicorp.com/agent-inject": "true", + "prometheus.io/scrape": "true", + "custom.company.com/environment": "production" + } +} +``` + +This configuration: +- Allows 2 concurrent operations globally, 3 on worker `node1` +- Allows up to 15 operations in preparation phases +- Runs Data Mover Pods only on backup nodes without critical workloads +- Uses fast storage nodes for fast-ssd storage class operations +- Limits pod resources to prevent cluster overload +- Uses backup-priority PriorityClass for backup operations +- Optimizes backup PVCs with read-only access and dedicated storage classes +- Ignores delay binding for faster restores +- Enable privileged permission for PodVolume pods +- Enable cache PVC for file system restore +- The cache threshold is 1GB and use dedicated StorageClass +- Use customized labels and annotations data mover pods + +## Troubleshooting + +### Common Issues + +1. **ConfigMap not taking effect**: Restart node-agent DaemonSet after changes +2. **Invalid resource values**: Check logs for validation errors; entire section ignored on failure +3. **Storage class not found**: Ensure specified storage classes exist in the cluster +4. **SELinux issues**: Set `spcNoRelabeling: true` when using `readOnly: true` +5. **Node selection not working**: Verify node labels and check only first loadAffinity element is used + +### Validation + +To verify your configuration is loaded correctly: +```bash +kubectl logs -n velero -l app=node-agent | grep -i config +``` + +To check current node-agent configuration: +```bash +kubectl get cm -n velero -o yaml +``` + +## Related Documentation +For detailed information on specific configuration sections: +- [Node-agent Concurrency](../node-agent-concurrency.md) +- [Node Selection for Data Movement](../data-movement-node-selection.md) +- [Data Movement Pod Resource Configuration](../data-movement-pod-resource-configuration.md) +- [BackupPVC Configuration for Data Movement Backup](../data-movement-backup-pvc-configuration.md) +- [RestorePVC Configuration for Data Movement Restore](../data-movement-restore-pvc-configuration.md) +- [Node-agent Prepare Queue Length](../node-agent-prepare-queue-length.md) +- [Cache PVC Configuration for Data Movement Restore](../data-movement-cache-volume.md) diff --git a/site/content/docs/v1.18/supported-providers.md b/site/content/docs/v1.18/supported-providers.md new file mode 100644 index 000000000..11ea60a15 --- /dev/null +++ b/site/content/docs/v1.18/supported-providers.md @@ -0,0 +1,70 @@ +--- +title: "Providers" +layout: docs +--- + +Velero supports a variety of storage providers for different backup and snapshot operations. Velero has a plugin system which allows anyone to add compatibility for additional backup and volume storage platforms without modifying the Velero codebase. + +## Provider plugins maintained by the Velero maintainers + +{{< table caption="Velero supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Provider Repo | Setup Instructions | Parameters | +|-----------------------------------|--------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Amazon Web Services (AWS)](https://aws.amazon.com) | AWS S3 | AWS EBS | [Velero plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws) | [AWS Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-aws#setup) | [BackupStorageLocation](https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/backupstoragelocation.md)
[VolumeSnapshotLocation](https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/volumesnapshotlocation.md) | +| [Google Cloud Platform (GCP)](https://cloud.google.com) | [Google Cloud Storage](https://github.com/vmware-tanzu/velero-plugin-for-gcp/blob/main/backupstoragelocation.md) | [Google Compute Engine Disks](https://github.com/vmware-tanzu/velero-plugin-for-gcp/blob/main/volumesnapshotlocation.md) | [Velero plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp) | [GCP Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) | [BackupStorageLocation](https://github.com/vmware-tanzu/velero-plugin-for-gcp/blob/main/backupstoragelocation.md)
[VolumeSnapshotLocation](https://github.com/vmware-tanzu/velero-plugin-for-gcp/blob/main/volumesnapshotlocation.md) | +| [Microsoft Azure](https://azure.com) | Azure Blob Storage | Azure Managed Disks | [Velero plugin for Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure) | [Azure Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) | [BackupStorageLocation](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md)
[VolumeSnapshotLocation](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/volumesnapshotlocation.md) | +| [VMware vSphere](https://www.vmware.com/ca/products/vsphere.html) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | 🚫 | +{{< /table >}} + +Contact: [#Velero Slack](https://kubernetes.slack.com/messages/velero), [GitHub Issues](https://github.com/vmware-tanzu/velero/issues) + +## Provider plugins maintained by the Velero community +{{< table caption="Community supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Documentation | Contact | +|---------------------------|------------------------------|------------------------------------|------------------------|---------------------------------| +| [AlibabaCloud](https://www.alibabacloud.com/) | Alibaba Cloud OSS | Alibaba Cloud | [AlibabaCloud](https://github.com/AliyunContainerService/velero-plugin) | [GitHub Issue](https://github.com/AliyunContainerService/velero-plugin/issues) | +| [DigitalOcean](https://www.digitalocean.com/) | DigitalOcean Object Storage | DigitalOcean Volumes Block Storage | [StackPointCloud](https://github.com/StackPointCloud/ark-plugin-digitalocean) | | +| [Hewlett Packard](https://www.hpe.com/us/en/storage.html) | 🚫 | HPE Storage | [Hewlett Packard](https://github.com/hpe-storage/velero-plugin) | [Slack](https://slack.hpedev.io/), [GitHub Issue](https://github.com/hpe-storage/velero-plugin/issues) | +| [HuaweiCloud](https://www.huaweicloud.com) | HuaweiCloud OBS | 🚫 | [HuaweiCloud](https://github.com/setoru/velero-plugin-for-huaweicloud) | [GitHub Issue](https://github.com/setoru/velero-plugin-for-huaweicloud/issues) | +| [OpenEBS](https://openebs.io/) | 🚫 | OpenEBS CStor Volume | [OpenEBS](https://github.com/openebs/velero-plugin) | [Slack](https://openebs-community.slack.com/), [GitHub Issue](https://github.com/openebs/velero-plugin/issues) | +| [OpenStack](https://www.openstack.org/) | Swift | Cinder | [OpenStack](https://github.com/Lirt/velero-plugin-for-openstack) | [GitHub Issue](https://github.com/Lirt/velero-plugin-for-openstack/issues) | +| [Portworx](https://portworx.com/) | 🚫 | Portworx Volume | [Portworx](https://docs.portworx.com/scheduler/kubernetes/ark.html) | [Slack](https://portworx.slack.com/messages/px-k8s), [GitHub Issue](https://github.com/portworx/ark-plugin/issues) | +| [Storj](https://storj.io) | Storj Object Storage | 🚫 | [Storj](https://github.com/storj-thirdparty/velero-plugin) | [GitHub Issue](https://github.com/storj-thirdparty/velero-plugin/issues) | +{{< /table >}} + +## S3-Compatible object store providers + +Velero's AWS Object Store plugin uses [Amazon's Go SDK][0] to connect to the AWS S3 API. Some third-party storage providers also support the S3 API, and users have reported the following providers work with Velero: + +_Note that these storage providers are not regularly tested by the Velero team._ + + * [IBM Cloud][1] + * [Oracle Cloud][2] + * [Minio][3] + * [DigitalOcean][4] + * [NooBaa][5] + * [Tencent Cloud][7] + * Ceph RADOS v12.2.7 + * Quobyte + * [Cloudian HyperStore][38] + +_Some storage providers, like Quobyte, may need a different [signature algorithm version][6]._ + +## Non-supported volume snapshots + +In the case you want to take volume snapshots but didn't find a plugin for your provider, Velero has support for snapshotting using File System Backup. Please see the [File System Backup][30] documentation. + +[0]: https://github.com/aws/aws-sdk-go-v2 +[1]: contributions/ibm-config.md +[2]: contributions/oracle-config.md +[3]: contributions/minio.md +[4]: https://github.com/StackPointCloud/ark-plugin-digitalocean +[5]: http://www.noobaa.com/ +[6]: https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/backupstoragelocation.md +[7]: contributions/tencent-config.md +[25]: https://github.com/hpe-storage/velero-plugin +[30]: file-system-backup.md +[36]: https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup +[38]: https://www.cloudian.com/ diff --git a/site/content/docs/v1.18/tilt.md b/site/content/docs/v1.18/tilt.md new file mode 100644 index 000000000..6d802220c --- /dev/null +++ b/site/content/docs/v1.18/tilt.md @@ -0,0 +1,209 @@ +--- +title: "Rapid iterative Velero development with Tilt " +layout: docs +--- + +## Overview +This document describes how to use [Tilt](https://tilt.dev) with any cluster for a simplified +workflow that offers easy deployments and rapid iterative builds. + +This setup allows for continuing deployment of the Velero server and, if specified, any provider plugin or the node-agent daemonset. +It does this work by: + +1. Deploying the necessary Kubernetes resources, such as the Velero CRDs and Velero deployment +1. Building a local binary for Velero and (if specified) provider plugins as a `local_resource` +1. Invoking `docker_build` to live update any binary into the container/init container and trigger a re-start + +Tilt will look for configuration files under `velero/tilt-resources`. Most of the +files in this directory are gitignored so you may configure your setup according to your needs. + +## Prerequisites +1. [Docker](https://docs.docker.com/install/) v19.03 or newer +1. A Kubernetes cluster v1.16 or greater (does not have to be Kind) +1. [Tilt](https://docs.tilt.dev/install.html) v0.12.0 or newer +1. Clone the [Velero project](https://github.com/vmware-tanzu/velero) repository + locally +1. Access to an S3 object storage +1. Clone any [provider plugin(s)](https://velero.io/plugins/) you want to make changes to and deploy (optional, must be configured to be deployed by the Velero Tilt's setup, [more info below](#provider-plugins)) + +Note: To properly configure any plugin you use, please follow the plugin's documentation. + +## Getting started + +### tl;dr +- Copy all sample files under `velero/tilt-resources/examples` into `velero/tilt-resources`. +- Configure the `velero_v1_backupstoragelocation.yaml` file, and the `cloud` file for the storage credentials/secret. + +- Run `tilt up`. + +### Create a Tilt settings file +Create a configuration file named `tilt-settings.json` and place it in your local copy of `velero/tilt-resources`. Alternatively, +you may copy and paste the sample file found in `velero/tilt-resources/examples`. + +Here is an example: + +```json +{ + "default_registry": "", + "enable_providers": [ + "aws", + "gcp", + "azure", + "csi" + ], + "providers": { + "aws": "../velero-plugin-for-aws", + "gcp": "../velero-plugin-for-gcp", + "azure": "../velero-plugin-for-microsoft-azure", + }, + "allowed_contexts": [ + "development" + ], + "use_node_agent": false, + "create_backup_locations": true, + "setup-minio": true, + "enable_debug": false, + "debug_continue_on_start": true +} +``` + +#### tilt-settings.json fields +**default_registry** (String, default=""): The image registry to use if you need to push images. See the [Tilt +*documentation](https://docs.tilt.dev/api.html#api.default_registry) for more details. + +**provider_repos** (Array[]String, default=[]): A list of paths to all the provider plugins you want to make changes to. Each provider must have a +`tilt-provider.json` file describing how to build the provider. + +**enable_providers** (Array[]String, default=[]): A list of the provider plugins to enable. See [provider plugins](provider-plugins) +for more details. Note: when not making changes to a plugin, it is not necessary to load them into +Tilt: an existing image and version might be specified in the Velero deployment instead, and Tilt will load that. + +**allowed_contexts** (Array, default=[]): A list of kubeconfig contexts Tilt is allowed to use. See the Tilt documentation on +*[allow_k8s_contexts](https://docs.tilt.dev/api.html#api.allow_k8s_contexts) for more details. Note: Kind is automatically allowed. + +**use_node_agent** (Bool, default=false): Indicate whether to deploy the node-agent Daemonset. If set to `true`, Tilt will look for a `velero/tilt-resources/node-agent.yaml` file +containing the configuration of the Velero node-agent DaemonSet. + +**create_backup_locations** (Bool, default=false): Indicate whether to create one or more backup storage locations. If set to `true`, Tilt will look for a `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` file +containing at least one configuration for a Velero backup storage location. + +**setup-minio** (Bool, default=false): Configure this to `true` if you want to configure backup storage locations in a Minio instance running inside your cluster. + +**enable_debug** (Bool, default=false): Configure this to `true` if you want to debug the velero process using [Delve](https://github.com/go-delve/delve). + +**debug_continue_on_start** (Bool, default=true): Configure this to `true` if you want the velero process to continue on start when in debug mode. See [Delve CLI documentation](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv.md). + +### Create Kubernetes resource files to deploy +All needed Kubernetes resource files are provided as ready to use samples in the `velero/tilt-resources/examples` directory. You only have to move them to the `velero/tilt-resources` level. + +Because the Velero Kubernetes deployment as well as the node-agent DaemonSet contain the configuration +for any plugin to be used, files for these resources are expected to be provided by the user so you may choose +which provider plugin to load as a init container. Currently, the sample files provided are configured with all the +plugins supported by Velero, feel free to remove any of them as needed. + +For Velero to operate fully, it also needs at least one backup +storage location. A sample file is provided that needs to be modified with the specific +configuration for your object storage. See the next sub-section for more details on this. + +### Configure a backup storage location +You will have to configure the `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` with the proper values according to your storage provider. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your particular provider's backup storage location configuration. + +Below are some ways to configure a backup storage location for Velero. +#### As a storage with a service provider +Follow the provider documentation to provision the storage. We have a [list of all known object storage providers](supported-providers/) with corresponding plugins for Velero. + +#### Using MinIO as an object storage +Note: to use MinIO as an object storage, you will need to use the [`AWS` plugin](https://github.com/vmware-tanzu/velero-plugin-for-aws), and configure the storage location with the `spec.provider` set to `aws` and the `spec.config.region` set to `minio`. Example: +``` +spec: + config: + region: minio + s3ForcePathStyle: "true" + s3Url: http://minio.velero.svc:9000 + objectStorage: + bucket: velero + provider: aws +``` + +Here are two ways to use MinIO as the storage: + +1) As a MinIO instance running inside your cluster (don't do this for production!) + + In the `tilt-settings.json` file, set `"setup-minio": true`. This will configure a Kubernetes deployment containing a running +instance of MinIO inside your cluster. There are [extra steps](contributions/minio/#expose-minio-outside-your-cluster-with-a-service) +necessary to expose MinIO outside the cluster. + + To access this storage, you will need to expose MinIO outside the cluster by forwarding the MinIO port to the local machine using kubectl port-forward -n svc/minio 9000. Update the BSL configuration to use that as its "public URL" by adding `publicUrl: http://localhost:9000` to the BSL config. This is necessary to do things like download a backup file. + + Note: with this setup, when your cluster is terminated so is the storage and any backup/restore in it. + +1) As a standalone MinIO instance running locally in a Docker container + + See [these instructions](https://github.com/vmware-tanzu/velero/discussions/3381) to run MinIO locally on your computer, as a standalone as opposed to running it on a Pod. + +Please see our [locations documentation](locations/) to learn more how backup locations work. + +### Configure the provider credentials (secret) +Whatever object storage provider you use, configure the credentials for in the `velero/tilt-resources/cloud` file. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your provider's credentials. The Tilt file will invoke Kustomize to create the secret under the hard-coded key `secret.cloud-credentials.data.cloud` in the Velero namespace. + +There is a sample credentials file properly formatted for a MinIO storage credentials in `velero/tilt-resources/examples/cloud`. + +### Configure debugging with Delve +If you would like to debug the Velero process, you can enable debug mode by setting the field `enable_debug` to `true` in your `tilt-resources/tile-settings.json` file. +This will enable you to debug the process using [Delve](https://github.com/go-delve/delve). +By enabling debug mode, the Velero executable will be built in debug mode (using the flags `-gcflags="-N -l"` which disables optimizations and inlining), and the process will be started in the Velero deployment using [`dlv exec`](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md). + +The debug server will accept connections on port 2345 and Tilt is configured to forward this port to the local machine. +Once Tilt is [running](#run-tilt) and the Velero resource is ready, you can connect to the debug server to begin debugging. +To connect to the session, you can use the Delve CLI locally by running `dlv connect 127.0.0.1:2345`. See the [Delve CLI documentation](https://github.com/go-delve/delve/tree/master/Documentation/cli) for more guidance on how to use Delve. +Delve can also be used within a number of [editors and IDEs](https://github.com/go-delve/delve/blob/master/Documentation/EditorIntegration.md). + +By default, the Velero process will continue on start when in debug mode. +This means that the process will run until a breakpoint is set. +You can disable this by setting the field `debug_continue_on_start` to `false` in your `tilt-resources/tile-settings.json` file. +When this setting is disabled, the Velero process will not continue to run until a `continue` instruction is issued through your Delve session. + +When exiting your debug session, the CLI and editor integrations will typically ask if the remote process should be stopped. +It is important to leave the remote process running and just disconnect from the debugging session. +By stopping the remote process, that will cause the Velero container to stop and the pod to restart. +If backups are in progress, these will be left in a stale state as they are not resumed when the Velero pod restarts. + +### Run Tilt! +To launch your development environment, run: + +``` bash +tilt up +``` + +This will output the address to a web browser interface where you can monitor Tilt's status and the logs for each Tilt resource. After a brief amount of time, you should have a running development environment, and you should now be able to +create backups/restores and fully operate Velero. + +Note: Running `tilt down` after exiting out of Tilt [will delete all resources](https://docs.tilt.dev/cli/tilt_down.html) specified in the Tiltfile. + +Tip: Create an alias to `velero/_tuiltbuild/local/velero` and you won't have to run `make local` to get a refreshed version of the Velero CLI, just use the alias. + +Please see the documentation for [how Velero works](how-velero-works/). + +## Provider plugins +A provider must supply a `tilt-provider.json` file describing how to build it. Here is an example: + +```json +{ + "plugin_name": "velero-plugin-for-aws", + "context": ".", + "image": "velero/velero-plugin-for-aws", + "live_reload_deps": [ + "velero-plugin-for-aws" + ], + "go_main": "./velero-plugin-for-aws" +} +``` + +## Live updates +Each provider plugin configured to be deployed by Velero's Tilt setup has a `live_reload_deps` list. This defines the files and/or directories that Tilt +should monitor for changes. When a dependency is modified, Tilt rebuilds the provider's binary **on your local +machine**, copies the binary to the init container, and triggers a restart of the Velero container. This is significantly faster +than rebuilding the container image for each change. It also helps keep the size of each development image as small as +possible (the container images do not need the entire go toolchain, source code, module dependencies, etc.). diff --git a/site/content/docs/v1.18/troubleshooting.md b/site/content/docs/v1.18/troubleshooting.md new file mode 100644 index 000000000..e61eb0d7e --- /dev/null +++ b/site/content/docs/v1.18/troubleshooting.md @@ -0,0 +1,252 @@ +--- +title: "Troubleshooting" +layout: docs +--- + +These tips can help you troubleshoot known issues. If they don't help, you can [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. + +## Debug installation/ setup issues + +- [Debug installation/setup issues][2] + +## Debug restores + +- [Debug restores][1] + +## General troubleshooting information + +You can use the `velero bug` command to open a [Github issue][4] by launching a browser window with some prepopulated values. Values included are OS, CPU architecture, `kubectl` client and server versions (if available) and the `velero` client version. This information isn't submitted to Github until you click the `Submit new issue` button in the Github UI, so feel free to add, remove or update whatever information you like. + +You can use the `velero debug` command to generate a debug bundle, which is a tarball +that contains: +* Version information +* Logs of velero server and plugins +* Resources managed by velero server such as backup, restore, podvolumebackup, podvolumerestore, etc. +* Logs of the backup and restore, if specified in the parameters + +Please use command `velero debug --help` to see more usage details. + +### Getting velero debug logs + +You can increase the verbosity of the Velero server by editing your Velero deployment to look like this: + + +``` +kubectl edit deployment/velero -n velero +... + containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --log-level # Add this line + - debug # Add this line +... +``` + +**Note:** Velero plugins are started as separate processes and once the Velero operation is done (either successfully or not), they exit. So, if you see **received EOF, stopping recv loop** messages in debug logs, that does not mean an error occurred, just that a plugin finished executing. + +## Known issue with restoring LoadBalancer Service + +Because of how Kubernetes handles Service objects of `type=LoadBalancer`, when you restore these objects you might encounter an issue with changed values for Service UIDs. Kubernetes automatically generates the name of the cloud resource based on the Service UID, which is different when restored, resulting in a different name for the cloud load balancer. If the DNS CNAME for your application points to the DNS name of your cloud load balancer, you'll need to update the CNAME pointer when you perform a Velero restore. + +Alternatively, you might be able to use the Service's `spec.loadBalancerIP` field to keep connections valid, if your cloud provider supports this value. See [the Kubernetes documentation about Services of Type LoadBalancer](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Known issue with restoring resources when Admission webhooks are enabled + +The [Admission webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) may forbid a resource to be created based on the input, it may optionally mutate the input as well. +Because velero calls the API server to restore resources, it is possible that the admission webhooks are invoked and cause unexpected failures, depending on the implementation and the configuration of the webhooks. +To work around such issue, you may disable the webhooks or create a restore item action plugin to modify the resources before they are restored. + +## Miscellaneous issues + +### Velero reports `custom resource not found` errors when starting up. + +Velero's server will not start if the required Custom Resource Definitions are not found in Kubernetes. Run `velero install` again to install any missing custom resource definitions. + +### `velero backup logs` returns a `SignatureDoesNotMatch` error + +Downloading artifacts from object storage utilizes temporary, signed URLs. In the case of S3-compatible +providers, such as Ceph, there may be differences between their implementation and the official S3 +API that cause errors. + +Here are some things to verify if you receive `SignatureDoesNotMatch` errors: + + * Make sure your S3-compatible layer is using [signature version 4][5] (such as Ceph RADOS v12.2.7) + * For Ceph, try using a native Ceph account for credentials instead of external providers such as OpenStack Keystone + +## Velero (or a pod it was backing up) restarted during a backup and the backup is stuck InProgress + +Velero cannot resume backups that were interrupted. Backups stuck in the `InProgress` phase can be deleted with `kubectl delete backup -n `. +Backups in the `InProgress` phase have not uploaded any files to object storage. + +## Velero is not publishing prometheus metrics + +Steps to troubleshoot: + +- Confirm that your velero deployment has metrics publishing enabled. The [latest Velero helm charts][6] have been setup with [metrics enabled by default][7]. +- Confirm that the Velero server pod exposes the port on which the metrics server listens on. By default, this value is 8085. + +```yaml + ports: + - containerPort: 8085 + name: metrics + protocol: TCP +``` + +- Confirm that the metric server is listening for and responding to connections on this port. This can be done using [port-forwarding][9] as shown below + +```bash +$ kubectl -n port-forward 8085:8085 +Forwarding from 127.0.0.1:8085 -> 8085 +Forwarding from [::1]:8085 -> 8085 +. +. +. +``` + +Now, visiting http://localhost:8085/metrics on a browser should show the metrics that are being scraped from Velero. + +- Confirm that the Velero server pod has the necessary [annotations][8] for prometheus to scrape metrics. +- Confirm, from the Prometheus UI, that the Velero pod is one of the targets being scraped from Prometheus. + + +## Is Velero using the correct cloud credentials? + +Cloud provider credentials are given to Velero to store and retrieve backups from the object store and to perform volume snapshotting operations. + +These credentials are either passed to Velero at install time using: +1. `--secret-file` flag to the `velero install` command. OR +1. `--set-file credentials.secretContents.cloud` flag to the `helm install` command. + +Or, they are specified when creating a `BackupStorageLocation` using the `--credential` flag. + +### Troubleshooting credentials provided during install + +If using the credentials provided at install time, they are stored in the cluster as a Kubernetes secret named `cloud-credentials` in the same namespace in which Velero is installed. + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials: +1. Confirm that the `cloud-credentials` secret exists and has the correct content. + ```bash + $ kubectl -n velero get secrets cloud-credentials + NAME TYPE DATA AGE + cloud-credentials Opaque 1 11h + $ kubectl -n velero get secrets cloud-credentials -ojsonpath={.data.cloud} | base64 --decode + + ``` + +1. Confirm that velero deployment is mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get deploy velero -ojson | jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/plugins", + "name": "plugins" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + + If [File System Backup][3] is enabled, then, confirm that the node-agent daemonset is also mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get ds node-agent -ojson |jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/host_pods", + "mountPropagation": "HostToContainer", + "name": "host-pods" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + +1. Confirm if the correct credentials are mounted into the Velero pod. + ```bash + $ kubectl -n velero exec -ti deploy/velero -- bash + nobody@velero-69f9c874c-l8mqp:/$ cat /credentials/cloud + + ``` + +### Troubleshooting `BackupStorageLocation` and `VolumeSnapshotLocation` credentials + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation` or `VolumeSnapshotLocation`][10]: +1. Confirm that the object storage provider plugin being used supports multiple credentials. + + If the logs from the Velero deployment contain the error message `"config has invalid keys credentialsFile"`, the version of your object storage plugin does not yet support multiple credentials. + + The object storage plugins [maintained by the Velero team][11] support this feature, so please update your plugin to the latest version if you see the above error message. + + If you are using a plugin from a different provider, please contact them for further advice. + +1. Confirm that the secret and key referenced by the `BackupStorageLocation` or `VolumeSnapshotLocation` exists in the Velero namespace and has the correct content: + ```bash + # Determine which secret and key the BackupStorageLocation is using + BSL_SECRET=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.name}) + BSL_SECRET_KEY=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.key}) + + # Confirm that the secret exists + kubectl -n velero get secret $BSL_SECRET + + # Print the content of the secret and ensure it is correct + kubectl -n velero get secret $BSL_SECRET -ojsonpath={.data.$BSL_SECRET_KEY} | base64 --decode + + # Determine which secret and key the VolumeSnapshotLocation is using + VSL_SECRET=$(kubectl get volumesnapshotlocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.name}) + VSL_SECRET_KEY=$(kubectl get volumesnapshotlocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.key}) + + # Confirm that the secret exists + kubectl -n velero get secret $VSL_SECRET + + # Print the content of the secret and ensure it is correct + kubectl -n velero get secret $VSL_SECRET -ojsonpath={.data.$VSL_SECRET_KEY} | base64 --decode + ``` + If the secret can't be found, the secret does not exist within the Velero namespace and must be created. + + If no output is produced when printing the contents of the secret, the key within the secret may not exist or may have no content. + Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET` or `kubectl -n velero describe secret $VSL_SECRET`. + If it does not exist, follow the instructions for [editing a Kubernetes secret][12] to add the base64 encoded credentials data. + +## Kopia repository files' ownership mismatch + +Velero sets the files' ownership created in the Kopia repository to `default@default`. + +If users need to use Kopia CLI to connect to the Velero created Kopia repositories, please use the following CLI as an example to avoid overwriting the ownership by accident. +``` bash +kopia repository connect <--read-only> --bucket= --override-username=default --override-hostname=default +``` + +If the ownership conflict error(`maintenance must be run by designated user`) already happens, +Velero doesn't handle the conflict by design. +To resolve it, please use Kopia maintenance CLI to set the ownership correctly, e.g. `kopia maintenance set --owner=default@default`. + +Please refer to [Issue 9007](https://github.com/vmware-tanzu/velero/issues/9007) for more information. + +[1]: debugging-restores.md +[2]: debugging-install.md +[3]: file-system-backup.md +[4]: https://github.com/vmware-tanzu/velero/issues +[5]: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +[6]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero +[7]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L44 +[8]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L49-L52 +[9]: https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#port-forward +[10]: locations.md +[11]: /plugins +[12]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret +[25]: https://kubernetes.slack.com/messages/velero diff --git a/site/content/docs/v1.18/uninstalling.md b/site/content/docs/v1.18/uninstalling.md new file mode 100644 index 000000000..a20d95e7a --- /dev/null +++ b/site/content/docs/v1.18/uninstalling.md @@ -0,0 +1,10 @@ +--- +title: "Uninstalling Velero" +layout: docs +--- + +If you would like to completely uninstall Velero from your cluster, the following commands will remove all resources created by `velero install`: + +```bash +velero uninstall +``` diff --git a/site/content/docs/v1.18/upgrade-to-1.18.md b/site/content/docs/v1.18/upgrade-to-1.18.md new file mode 100644 index 000000000..0bf839fe3 --- /dev/null +++ b/site/content/docs/v1.18/upgrade-to-1.18.md @@ -0,0 +1,91 @@ +--- +title: "Upgrading to Velero 1.18" +layout: docs +--- + +## Prerequisites + +- Velero [v1.17.x][9] installed. + +If you're not yet running at least Velero v1.17, see the following: + +- [Upgrading to v1.8][1] +- [Upgrading to v1.9][2] +- [Upgrading to v1.10][3] +- [Upgrading to v1.11][4] +- [Upgrading to v1.12][5] +- [Upgrading to v1.13][6] +- [Upgrading to v1.14][7] +- [Upgrading to v1.15][8] +- [Upgrading to v1.16][9] +- [Upgrading to v1.17][10] + +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + +## Instructions + +### Upgrade from v1.17 +1. Install the Velero v1.18 command-line interface (CLI) by following the [instructions here][0]. + + Verify that you've properly installed it by running: + + ```bash + velero version --client-only + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.18.0 + Git commit: + ``` + +2. Update the Velero custom resource definitions (CRDs) to include schema changes across all CRDs that are at the core of the new features in this release: + + ```bash + velero install --crds-only --dry-run -o yaml | kubectl apply -f - + ``` + +3. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: + ```bash + # set the container and image of the init container for plugin accordingly, + # if you are using other plugin + kubectl set image deployment/velero \ + velero=velero/velero:v1.18.0 \ + velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.14.0 \ + --namespace velero + + # optional, if using the node agent daemonset + kubectl set image daemonset/node-agent \ + node-agent=velero/velero:v1.18.0 \ + --namespace velero + ``` +4. Confirm that the deployment is up and running with the correct version by running: + + ```bash + velero version + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.18.0 + Git commit: + + Server: + Version: v1.18.0 + ``` + +[0]: basic-install.md#install-the-cli +[1]: https://velero.io/docs/v1.8/upgrade-to-1.8 +[2]: https://velero.io/docs/v1.9/upgrade-to-1.9 +[3]: https://velero.io/docs/v1.10/upgrade-to-1.10 +[4]: https://velero.io/docs/v1.11/upgrade-to-1.11 +[5]: https://velero.io/docs/v1.12/upgrade-to-1.12 +[6]: https://velero.io/docs/v1.13/upgrade-to-1.13 +[7]: https://velero.io/docs/v1.14/upgrade-to-1.14 +[8]: https://velero.io/docs/v1.15/upgrade-to-1.15 +[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 +[10]: https://velero.io/docs/v1.17/upgrade-to-1.17 \ No newline at end of file diff --git a/site/content/docs/v1.18/velero-install.md b/site/content/docs/v1.18/velero-install.md new file mode 100644 index 000000000..a4925278c --- /dev/null +++ b/site/content/docs/v1.18/velero-install.md @@ -0,0 +1,58 @@ +--- +title: "Velero Install CLI" +layout: docs +--- + +This document serves as a guide to using the `velero install` CLI command to install `velero` server components into your Kubernetes cluster. + +_NOTE_: `velero install` will, by default, use the CLI's version information to determine the version of the server components to deploy. This behavior may be overridden by using the `--image` flag. Refer to [Building Server Component Container Images][1]. + +## Usage + +This section explains some of the basic flags supported by the `velero install` CLI command. For a complete explanation of the flags, please run `velero install --help` + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + --kubelet-root-dir \ + [--use-node-agent] \ + [--default-volumes-to-fs-backup] \ + [--node-agent-pod-cpu-request ] \ + [--node-agent-pod-mem-request ] \ + [--node-agent-pod-cpu-limit ] \ + [--node-agent-pod-mem-limit ] \ + [--maintenance-job-cpu-request ] \ + [--maintenance-job-mem-request ] \ + [--maintenance-job-cpu-limit ] \ + [--maintenance-job-mem-limit ] \ + [--server-priority-class-name ] \ + [--node-agent-priority-class-name ] +``` + +The values for the resource requests and limits flags follow the same format as [Kubernetes resource requirements][3] +For plugin container images, please refer to our [supported providers][2] page. + +The `--server-priority-class-name` and `--node-agent-priority-class-name` flags allow you to set priority classes for the Velero server deployment and node agent daemonset respectively. This can help ensure proper scheduling and eviction behavior in resource-constrained environments. Note that you must create the priority class before installing Velero. + +## Examples + +This section provides examples that serve as a starting point for more customized installations. + +```bash +velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json + +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent + +velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] +``` + +[1]: build-from-source.md#making-images-and-updating-velero +[2]: supported-providers.md +[3]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ diff --git a/site/content/docs/v1.18/volume-group-snapshots.md b/site/content/docs/v1.18/volume-group-snapshots.md new file mode 100644 index 000000000..95cf33ff9 --- /dev/null +++ b/site/content/docs/v1.18/volume-group-snapshots.md @@ -0,0 +1,499 @@ +--- +title: "Volume Group Snapshots" +description: "A guide to using Volume Group Snapshots with Velero for consistent backups of multi-volume applications." +layout: docs +--- + +Velero provides robust support for **Volume Group Snapshots (VGS)**, a powerful Kubernetes feature for creating atomic, crash-consistent snapshots of multiple volumes simultaneously. This capability is essential for stateful applications that distribute data across several PersistentVolumeClaims (PVCs) and require that all data be captured at the exact same moment to ensure data integrity. + +> **Who is this for?** This guide is for application owners and backup administrators who need to ensure data consistency for multi-volume stateful applications, such as distributed databases (e.g., Cassandra, Zookeeper) or complex stateful services. + +## When to Use Volume Group Snapshots + +You should consider using Volume Group Snapshots when: + +- Your application uses multiple PVCs that are logically related. +- You need to ensure write-order consistency across all volumes. +- Your storage provider's CSI driver supports the Volume Group Snapshot feature. + +## Key Concepts + +Before diving in, let's clarify the Kubernetes resources involved in this process: + +- **VolumeGroupSnapshot (VGS):** A request to your storage provider to create a snapshot of a group of volumes. +- **VolumeGroupSnapshotContent (VGSC):** Represents the actual snapshot of the volume group, provisioned by the CSI driver. +- **VolumeGroupSnapshotClass (VGSClass):** A cluster-level resource that defines the configuration for a VGS, including the CSI driver and other parameters. +- **VolumeSnapshot (VS) & VolumeSnapshotContent (VSC):** The individual volume snapshots that are created as part of the VGS process. + +## Velero's VGS Backup and Restore Workflow + +Velero's integration with VGS is designed to be as seamless as possible, automating the complexities of group snapshots. Velero supports three distinct workflows depending on your configuration: + +### Three VGS Workflow Branches + +Velero automatically selects the appropriate workflow based on your backup configuration: + +#### 1. VGS + Data Mover +**When**: PVCs have VGS labels AND `--snapshot-move-data=true` flag is used + +**Use case**: Need atomic consistency + long-term storage/cross-cloud portability + +- Creates `VolumeGroupSnapshot` for write-order consistency across all labeled PVCs +- Extracts individual `VolumeSnapshot` objects from the VGS +- Creates `DataUpload` CRs that move each volume's data to object storage +- Cleans up temporary VGS and VGS Content resources +- **Result**: Volume data stored in object storage, no local snapshots retained + +#### 2. VGS without Data Mover +**When**: PVCs have VGS labels BUT `--snapshot-move-data=false` (or flag omitted) + +**Use case**: Need atomic consistency with local snapshot storage + +- Creates `VolumeGroupSnapshot` for write-order consistency across all labeled PVCs +- Extracts individual `VolumeSnapshot` objects from the VGS +- Backup depends only on `VolumeSnapshots`, no `DataUpload` or data mover involved +- Cleans up temporary VGS and VGS Content resources +- **Result**: Individual `VolumeSnapshots` stored on your storage system + +#### 3. Individual Volume Snapshots +**When**: PVCs have NO VGS labels (standard CSI snapshot behavior) + +**Use case**: Independent volume backups, no consistency requirements + +- Creates individual `VolumeSnapshot` per PVC independently +- No atomic consistency guarantees across volumes +- Optionally uses data movement if `--snapshot-move-data=true` flag is set +- **Result**: Independent volume snapshots (local or in object storage) + +### Choosing the Right Workflow + +Select your workflow based on your application's requirements: + +| Scenario | VGS Labels | Data Movement Flag | Workflow | Best For | +|----------|------------|-------------------|----------|----------| +| Multi-volume app + cross-cloud backup | ✅ | `--snapshot-move-data=true` | **VGS + Data Movement** | Distributed databases with portability needs | +| Multi-volume app + local snapshots | ✅ | `--snapshot-move-data=false` (or omitted) | **VGS Only** | Applications requiring consistency with fast local snapshots | +| Single volumes or independent backups | ❌ | `--snapshot-move-data=true` (optional) | **Individual Snapshots** | Simple applications, testing, or independent services | + +**Example Commands:** +```bash +# VGS + Data Movement (cross-cloud, long-term storage) +velero backup create db-backup --include-namespaces my-database --snapshot-move-data=true + +# VGS Only (atomic consistency, local storage) +velero backup create db-backup --include-namespaces my-database --snapshot-move-data=false + +# Individual Snapshots (standard CSI behavior) +velero backup create app-backup --include-namespaces my-app +``` + +### The Backup Process + +The VGS backup workflow is triggered by a simple label on your PVCs. + +1. **Grouping PVCs:** When a backup is initiated, Velero's `PVCAction` plugin scans for PVCs with the VGS label (the default is `velero.io/volume-group`). All PVCs within the same namespace that share the same label value are collected into a single `ItemBlock`. This ensures they are processed as a single, atomic unit. + +2. **Orchestrating the Snapshot:** The CSI plugin takes over to manage the snapshot creation: + * **Driver Verification:** It first confirms that all PVCs in the group are managed by the same CSI driver. + * **Class Selection:** It then determines the correct `VolumeGroupSnapshotClass` to use based on your configuration. + * **VGS Creation:** A `VolumeGroupSnapshot` resource is created, signaling the CSI driver to begin the snapshot process for the entire group. + +3. **Snapshot Finalization:** Velero monitors the process, and once the `VolumeGroupSnapshot` is ready, it performs these final steps: + * Waits for the CSI driver to create the individual `VolumeSnapshot` objects. + * Applies the backup's labels to each `VolumeSnapshot` for tracking. + +4. **Resource Cleanup:** To keep your cluster tidy, Velero deletes the temporary `VolumeGroupSnapshot` and `VolumeGroupSnapshotContent` resources after the individual `VolumeSnapshots` have been created and secured. + +Here is a visual representation of the backup workflow: + +![VGS Backup Workflow](/img/vgs-flow.svg) + +### The Restore Process + +Restoring from a VGS backup is simple and flexible. During backup, Velero creates individual `VolumeSnapshots` from the `VolumeGroupSnapshot`, so the restore process works with standard volume snapshot restoration. + +> **Good to know:** No special VGS-related logic is needed during the restore. This means you can restore your data to a cluster that doesn't have VGS support enabled, providing excellent portability. + +## Prerequisites + +Before using Volume Group Snapshots with Velero, ensure your environment meets these requirements: + +### 1. Kubernetes Version +- Kubernetes 1.20+ (when VolumeGroupSnapshot API was introduced) +- Check your version: `kubectl version --short` + +### 2. VolumeGroupSnapshot CRDs +Check the Volume Group Snapshot CRDs on your cluster: + +```bash +# Check if VGS CRDs are installed +kubectl get crd | grep volumegroup +``` + +### 3. CSI Driver Support +Verify your CSI driver supports Volume Group Snapshots: + +```bash +# Check if your CSI driver has VolumeGroupSnapshotClass resources +kubectl get volumegroupsnapshotclass + +# Verify CSI driver capabilities (example for AWS EBS) +kubectl describe csidriver ebs.csi.aws.com +``` + +### 4. VolumeGroupSnapshotClass Configuration +Ensure a VolumeGroupSnapshotClass exists for your storage and is properly labeled for Velero discovery: + +```bash +# List available VolumeGroupSnapshotClasses +kubectl get volumegroupsnapshotclass -o wide +``` + +**Important:** The VolumeGroupSnapshotClass must have the label `velero.io/csi-volumegroupsnapshot-class: "true"` for Velero to automatically discover and use it: + +```yaml +apiVersion: groupsnapshot.storage.k8s.io/v1alpha1 +kind: VolumeGroupSnapshotClass +metadata: + name: csi-vgs-class + labels: + velero.io/csi-volumegroupsnapshot-class: "true" +spec: + driver: ebs.csi.aws.com + deletionPolicy: Delete +``` + +Verify your VolumeGroupSnapshotClass has the correct label: +```bash +# Check if VolumeGroupSnapshotClass has the required label +kubectl get volumegroupsnapshotclass --show-labels +``` + +## Step-by-Step: Using VGS with Velero + +Here's how to get started with VGS backups: + +1. **Verify Prerequisites:** Ensure all prerequisites above are met. + +2. **Label Your PVCs:** The key to grouping volumes is to apply a consistent label to all PVCs that should be snapshotted together. **Important:** All PVCs in a group must use the same CSI driver and exist in the same namespace. + +### Complete Workflow Example + +Here's a complete end-to-end example of using VGS with a database application that has multiple volumes: + +#### 1. Set Up the Application + +Deploy a database application with multiple PVCs: + +**PVC for Primary Data:** +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: db-data-pvc + namespace: my-database + labels: + velero.io/volume-group: db-cluster-1 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: my-csi-storage-class +``` + +**PVC for Transaction Logs:** +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: db-logs-pvc + namespace: my-database + labels: + velero.io/volume-group: db-cluster-1 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: my-csi-storage-class +``` + +When you next back up the `my-database` namespace, Velero will see the `velero.io/volume-group: db-cluster-1` label on both PVCs and will trigger a `VolumeGroupSnapshot` for the `db-cluster-1` group. + +#### 2. Create the Backup + +```bash +# Create backup that will use VGS for labeled PVCs +velero backup create my-app-backup --include-namespaces my-database + +# Monitor backup progress +velero backup describe my-app-backup +velero backup logs my-app-backup +``` + +#### 3. Verify VGS Processing + +```bash +# Verify VolumeSnapshots were created from the VGS +kubectl get volumesnapshot -n my-database -o wide + +# Check that snapshots have the correct labels +kubectl get volumesnapshot -n my-database --show-labels + +# Confirm backup completed successfully +velero backup describe my-app-backup | grep Phase +``` + +#### 4. Test Restore + +```bash +# Create a test namespace for restore +kubectl create namespace my-database-restore + +# Restore to the new namespace +velero restore create test-restore \ + --from-backup my-app-backup \ + --namespace-mappings my-database:my-database-restore + +# Monitor restore progress +velero restore describe test-restore +velero restore logs test-restore + +# Verify PVCs were restored correctly +kubectl get pvc -n my-database-restore --show-labels +``` + +#### 5. Cleanup Test Resources + +```bash +# Remove test namespace after verification +kubectl delete namespace my-database-restore + +# List backups and restores +velero backup get +velero restore get +``` + +## Advanced Configuration + +You can customize the label key that Velero uses to identify VGS groups. This is useful if you have pre-existing labels or want to use a different convention. The configuration is applied with the following order of precedence: + +1. **Backup Resource Spec (Highest Priority):** For the most granular control, you can specify the label key directly in your `Backup` resource definition. + ```yaml + apiVersion: velero.io/v1 + kind: Backup + metadata: + name: my-app-backup + namespace: velero + spec: + volumeGroupSnapshotLabelKey: "my-organization.io/snapshot-group" + includedNamespaces: [ "my-database" ] + # ... other backup spec details + ``` + +2. **Velero Server Argument:** You can set a cluster-wide default by providing the `--volume-group-snapshot-label-key` command-line argument when you install or start the Velero server. + +3. **Default Value (Lowest Priority):** If you don't provide any custom configuration, Velero defaults to using `velero.io/volume-group`. + +## Volume Policies and VolumeGroupSnapshots + +Volume policies control which volumes should be backed up and how (snapshot vs filesystem backup). When using VolumeGroupSnapshots, volume policies are applied **before** grouping PVCs. + +### How Volume Policies Affect VGS + +When Velero processes PVCs for a VolumeGroupSnapshot: + +1. **Label Matching:** All PVCs with the matching VGS label are identified +2. **Policy Filtering:** Volume policies are evaluated for each PVC +3. **Group Creation:** Only PVCs that should be snapshotted (not excluded by policy) are included in the VGS +4. **Warning Logging:** If any PVCs are excluded from the group by volume policy, a warning is logged + +This behavior ensures that volume policies take precedence over VGS labels. The VGS label indicates "group these volumes **if they're being backed up**", while the volume policy determines "which volumes to back up". + +### Example Scenario + +Consider an application with mixed storage types where some volumes should be excluded: + +```yaml +# Database PVC using CSI driver (should be backed up) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: db-data + namespace: my-app + labels: + app.kubernetes.io/instance: myapp # VGS label +spec: + storageClassName: csi-storage + # ... + +--- +# Config PVC using NFS (should be excluded) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: config-data + namespace: my-app + labels: + app.kubernetes.io/instance: myapp # Same VGS label +spec: + storageClassName: nfs-storage + # ... +``` + +**Volume Policy Configuration:** +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: velero-volume-policies + namespace: velero +data: + volume-policy: | + version: v1 + volumePolicies: + - conditions: + nfs: {} + action: + type: skip +``` + +**Backup Configuration:** +```yaml +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: myapp-backup +spec: + includedNamespaces: + - my-app + volumeGroupSnapshotLabelKey: app.kubernetes.io/instance + resourcePolicy: + kind: ConfigMap + name: velero-volume-policies +``` + +**Result:** +- The NFS PVC (`config-data`) is filtered out by the volume policy +- Only the CSI PVC (`db-data`) is included in the VolumeGroupSnapshot +- A warning is logged: `PVC my-app/config-data has VolumeGroupSnapshot label app.kubernetes.io/instance=myapp but is excluded by volume policy` +- The backup succeeds with a single-volume VGS instead of failing with "multiple CSI drivers" error + +### Best Practices + +1. **Use Specific Labels:** When possible, use VGS labels that only target volumes you want to group, rather than relying on volume policies for filtering +2. **Monitor Warnings:** Review backup logs for volume policy exclusion warnings to ensure intended PVCs are being backed up +3. **Test Configurations:** Verify that your volume policy and VGS label combinations produce the expected grouping in a test environment + +## Troubleshooting + +### Common Issues and Solutions + +#### VGS Not Created During Backup + +**Symptoms:** Backup completes but individual VolumeSnapshots are created instead of VGS +```bash +# Check if PVCs have the correct label +kubectl get pvc -n my-database --show-labels + +# Verify all PVCs use the same CSI driver +kubectl get pv $(kubectl get pvc -n my-database -o jsonpath='{.items[*].spec.volumeName}') -o jsonpath='{range .items[*]}{.metadata.name}: {.spec.csi.driver}{"\n"}{end}' +``` + +**Solutions:** +- Ensure all PVCs have the same volume group label value +- Verify all PVCs use the same CSI driver +- Check that VolumeGroupSnapshotClass exists for your CSI driver + +#### VGS Creation Fails + +**Symptoms:** Backup fails with VGS-related errors +```bash +# Check Velero logs for VGS errors +velero backup logs my-app-backup | grep -i "VolumeGroup" + +# Check CSI driver logs +kubectl logs -n kube-system -l app=ebs-csi-controller --tail=100 +``` + +**Solutions:** +- Verify CSI driver supports VolumeGroupSnapshots +- Check VolumeGroupSnapshotClass configuration +- Ensure storage backend supports group snapshots + +#### Multiple CSI Drivers Error + +**Symptoms:** Backup fails with error about multiple CSI drivers found +``` +Error backing up item: failed to determine CSI driver for PVCs in VolumeGroupSnapshot group: +found multiple CSI drivers: linstor.csi.linbit.com and nfs.csi.k8s.io +``` + +**Cause:** PVCs with the same VGS label use different CSI drivers or include non-CSI volumes + +**Solutions:** +1. Use more specific labels that only match PVCs using the same CSI driver +2. Use volume policies to exclude PVCs that shouldn't be snapshotted: + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: velero-volume-policies + namespace: velero + data: + volume-policy: | + version: v1 + volumePolicies: + - conditions: + nfs: {} + action: + type: skip + ``` +3. Check backup logs for volume policy warnings to verify filtering is working + +#### VolumeGroupSnapshot Setup: Default VolumeSnapshotClass Required + +**Issue** + +When creating VolumeGroupSnapshot backups, you may encounter this error: + +``` +VolumeSnapshot has a temporary error Failed to set default snapshot class with error cannot find default snapshot class. Snapshot controller will retry later. +``` + +**Cause** + +The Kubernetes snapshot controller requires a default VolumeSnapshotClass to be configured in the cluster, but none is currently set. + +**Solution** + +Set a default VolumeSnapshotClass that uses the same CSI driver as your VolumeGroupSnapshotClass: + +```bash +# List available VolumeSnapshotClasses +kubectl get volumesnapshotclasses + +# Set the appropriate class as default for your CSI driver +kubectl patch volumesnapshotclass \ + -p '{"metadata":{"annotations":{"snapshot.storage.kubernetes.io/is-default-class":"true"}}}' + +# Example for Ceph RBD: +kubectl patch volumesnapshotclass ocs-storagecluster-rbdplugin-snapclass \ + -p '{"metadata":{"annotations":{"snapshot.storage.kubernetes.io/is-default-class":"true"}}}' +``` + +**Important:** Ensure the default VolumeSnapshotClass uses the same CSI driver as your VolumeGroupSnapshotClass. For example, if your VolumeGroupSnapshotClass uses `ebs.csi.aws.com`, the default VolumeSnapshotClass should also use `ebs.csi.aws.com`. + +**Note:** Only one VolumeSnapshotClass should be marked as default per CSI driver to avoid conflicts. The default VolumeSnapshotClass driver must match the CSI driver used by your VolumeGroupSnapshotClass. + +### Best Practices + +1. **Test VGS Support:** Always test VGS functionality in a non-production environment first +2. **Monitor Resource Usage:** VGS operations may consume more resources than individual snapshots +3. **Label Consistency:** Use consistent labeling across your organization +4. **Backup Validation:** Always verify backup success before relying on it for disaster recovery +5. **Storage Quotas:** Ensure sufficient storage quota for group snapshots + diff --git a/site/content/docs/v1.18/website-guidelines.md b/site/content/docs/v1.18/website-guidelines.md new file mode 100644 index 000000000..c5373f4d8 --- /dev/null +++ b/site/content/docs/v1.18/website-guidelines.md @@ -0,0 +1,45 @@ +--- +title: "Website Guidelines" +layout: docs +--- + +## Running the website locally + +When making changes to the website, please run the site locally before submitting a PR and manually verify your changes. + +At the root of the project, run: + +```bash +make serve-docs +``` + +This runs all the Hugo dependencies in a container. + +Alternatively, for quickly loading the website, under the `velero/site/` directory run: + +```bash +hugo serve +``` + +For more information on how to run the website locally, please see our [Hugo documentation](https://gohugo.io/getting-started/). + +## Adding a blog post + +To add a blog post, create a new markdown (.MD) file in the `/site/content/posts/` folder. A blog post requires the following front matter. + +```yaml +title: "Title of the blog" +excerpt: Brief summary of thee blog post that appears as a preview on velero.io/blogs +author_name: Jane Smith +slug: URL-For-Blog +# Use different categories that apply to your blog. This is used to connect related blogs on the site +categories: ['velero','release'] +# Image to use for blog. The path is relative to the site/static/ folder +image: /img/posts/example-image.jpg +# Tag should match author to drive author pages. Tags can have multiple values. +tags: ['Velero Team', 'Nolan Brubaker'] +``` + +Include the `author_name` value in tags field so the page that lists the author's posts will work properly, for example https://velero.io/tags/carlisia-thompson/. + +Ideally each blog will have a unique image to use on the blog home page, but if you do not include an image, the default Velero logo will be used instead. Use an image that is less than 70KB and add it to the `/site/static/img/posts` folder. diff --git a/site/data/docs/main-toc.yml b/site/data/docs/main-toc.yml index c732a9174..271705a1b 100644 --- a/site/data/docs/main-toc.yml +++ b/site/data/docs/main-toc.yml @@ -13,8 +13,8 @@ toc: url: /basic-install - page: Customize Installation url: /customize-installation - - page: Upgrade to 1.17 - url: /upgrade-to-1.17 + - page: Upgrade to 1.18 + url: /upgrade-to-1.18 - page: Supported providers url: /supported-providers - page: Evaluation install @@ -33,6 +33,8 @@ toc: url: /enable-api-group-versions-feature - page: Resource filtering url: /resource-filtering + - page: Namespace glob patterns + url: /namespace-glob-patterns - page: Backup reference url: /backup-reference - page: Backup hooks diff --git a/site/data/docs/toc-mapping.yml b/site/data/docs/toc-mapping.yml index f08001a91..8f96546c1 100644 --- a/site/data/docs/toc-mapping.yml +++ b/site/data/docs/toc-mapping.yml @@ -3,6 +3,7 @@ # that the navigation for older versions still work. main: main-toc +v1.18: v1-18-toc v1.17: v1-17-toc v1.16: v1-16-toc v1.15: v1-15-toc diff --git a/site/data/docs/v1-18-toc.yml b/site/data/docs/v1-18-toc.yml new file mode 100644 index 000000000..271705a1b --- /dev/null +++ b/site/data/docs/v1-18-toc.yml @@ -0,0 +1,129 @@ +toc: + - title: Introduction + subfolderitems: + - page: About Velero + url: /index.html + - page: How Velero works + url: /how-velero-works + - page: About locations + url: /locations + - title: Install + subfolderitems: + - page: Basic Install + url: /basic-install + - page: Customize Installation + url: /customize-installation + - page: Upgrade to 1.18 + url: /upgrade-to-1.18 + - page: Supported providers + url: /supported-providers + - page: Evaluation install + url: /contributions/minio + - page: Examples + url: /examples + - page: Uninstalling + url: /uninstalling + - title: Use + subfolderitems: + - page: Disaster recovery + url: /disaster-case + - page: Cluster migration + url: /migration-case + - page: Enable API group versions + url: /enable-api-group-versions-feature + - page: Resource filtering + url: /resource-filtering + - page: Namespace glob patterns + url: /namespace-glob-patterns + - page: Backup reference + url: /backup-reference + - page: Backup hooks + url: /backup-hooks + - page: Restore reference + url: /restore-reference + - page: Restore hooks + url: /restore-hooks + - page: Restore Resource Modifiers + url: /restore-resource-modifiers + - page: Run in any namespace + url: /namespace + - page: CSI Support + url: /csi + - page: Volume Group Snapshots + url: /volume-group-snapshots + - page: Backup Repository Configuration + url: /backup-repository-configuration + - page: Verifying Self-signed Certificates + url: /self-signed-certificates + - page: Changing RBAC permissions + url: /rbac + - page: Behind proxy + url: /proxy + - page: Repository Maintenance + url: /repository-maintenance + - page: Backup Restore Windows Workloads + url: /backup-restore-windows + - title: Data Mover + subfolderitems: + - page: CSI Snapshot Data Mover + url: /csi-snapshot-data-movement + - page: File system backup + url: /file-system-backup + - page: Data Movement Backup PVC Configuration + url: /data-movement-backup-pvc-configuration + - page: Data Movement Restore PVC Configuration + url: /data-movement-restore-pvc-configuration + - page: Data Movement Pod Resource Configuration + url: /data-movement-pod-resource-configuration + - page: Data Movement Node Selection Configuration + url: /data-movement-node-selection + - page: Data Movement Cache PVC Configuration + url: /data-movement-cache-volume + - page: Node-agent Concurrency + url: /node-agent-concurrency + - title: Plugins + subfolderitems: + - page: Overview + url: /overview-plugins + - page: Custom plugins + url: /custom-plugins + - title: Troubleshoot + subfolderitems: + - page: Troubleshooting + url: /troubleshooting + - page: Troubleshoot an install or setup + url: /debugging-install + - page: Troubleshoot a restore + url: /debugging-restores + - page: Troubleshoot file system backup + url: /file-system-backup#troubleshooting + - title: Contribute + subfolderitems: + - page: Start Contributing + url: /start-contributing + - page: Development + url: /development + - page: Rapid development with Tilt + url: /tilt + - page: Build from source + url: /build-from-source + - page: Run locally + url: /run-locally + - page: Code standards + url: /code-standards + - page: Website guidelines + url: /website-guidelines + - page: Documentation style guide + url: /style-guide + - title: More information + subfolderitems: + - page: Backup file format + url: /output-file-format + - page: API types + url: /api-types + - page: Supported ConfigMap + url: /supported-configmaps + - page: Support process + url: /support-process + - page: For maintainers + url: /maintainers diff --git a/site/static/img/adopters/broadcom.svg b/site/static/img/adopters/broadcom.svg new file mode 100644 index 000000000..2832b5120 --- /dev/null +++ b/site/static/img/adopters/broadcom.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + diff --git a/test/Makefile b/test/Makefile index 7c99e4d70..0e04239d1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -76,7 +76,7 @@ HAS_VSPHERE_PLUGIN ?= false RESTORE_HELPER_IMAGE ?= #Released version only -UPGRADE_FROM_VELERO_VERSION ?= v1.15.2,v1.16.2 +UPGRADE_FROM_VELERO_VERSION ?= v1.16.2,v1.17.2 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION @@ -85,7 +85,7 @@ UPGRADE_FROM_VELERO_VERSION ?= v1.15.2,v1.16.2 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION UPGRADE_FROM_VELERO_CLI ?= -MIGRATE_FROM_VELERO_VERSION ?= v1.16.2,$(VERSION) +MIGRATE_FROM_VELERO_VERSION ?= v1.17.2,$(VERSION) MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 1b46eed65..01cc6e38c 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -240,7 +240,7 @@ func (n *NodeAgentConfigTestCase) Backup() error { Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In backup, only the second element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:]) + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1], nil) Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) @@ -317,7 +317,7 @@ func (n *NodeAgentConfigTestCase) Restore() error { Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In restore, only the first element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1]) + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[0], nil) Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index da57ef19e..e2c57392e 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -365,7 +365,7 @@ func VersionNoOlderThan(version string, targetVersion string) (bool, error) { matches := tagRe.FindStringSubmatch(targetVersion) targetMajor := matches[1] targetMinor := matches[2] - if major > targetMajor && minor >= targetMinor { + if major >= targetMajor && minor >= targetMinor { return true, nil } else { return false, nil diff --git a/test/util/velero/install_test.go b/test/util/velero/install_test.go new file mode 100644 index 000000000..8ea51bd40 --- /dev/null +++ b/test/util/velero/install_test.go @@ -0,0 +1,65 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_VersionNoOlderThan(t *testing.T) { + type versionTest struct { + caseName string + version string + targetVersion string + result bool + err error + } + tests := []versionTest{ + { + caseName: "branch version compare", + version: "release-1.18", + targetVersion: "v1.16", + result: true, + err: nil, + }, + { + caseName: "tag version compare", + version: "v1.18.0", + targetVersion: "v1.16", + result: true, + err: nil, + }, + { + caseName: "main version compare", + version: "main", + targetVersion: "v1.15", + result: true, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.caseName, func(t *testing.T) { + res, err := VersionNoOlderThan(test.version, test.targetVersion) + + require.Equal(t, test.err, err) + require.Equal(t, test.result, res) + }) + } +} diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 96925b0d8..5c9b39a84 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -99,6 +99,15 @@ var ImagesMatrix = map[string]map[string][]string{ "velero": {"velero/velero:v1.16.2"}, "velero-restore-helper": {"velero/velero:v1.16.2"}, }, + "v1.17": { + "aws": {"velero/velero-plugin-for-aws:v1.13.2"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.13.2"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.13.2"}, + "datamover": {"velero/velero-plugin-for-aws:v1.13.2"}, + "velero": {"velero/velero:v1.17.2"}, + "velero-restore-helper": {"velero/velero:v1.17.2"}, + }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, @@ -128,16 +137,13 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig ret.Plugins = "" - versionWithoutPatch := "main" - if version != "main" { - versionWithoutPatch = semver.MajorMinor(version) - } + versionWithoutPatch := getVersionWithoutPatch(version) // Read migration case needs images from the PluginsMatrix map. images, ok := ImagesMatrix[versionWithoutPatch] if !ok { - return config, fmt.Errorf("fail to read the images for version %s from the ImagesMatrix", - versionWithoutPatch) + fmt.Printf("Cannot read the images for version %s from the ImagesMatrix. Use the original values.\n", versionWithoutPatch) + return config, nil } ret.VeleroImage = images[Velero][0] @@ -164,6 +170,27 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig return ret, nil } +func getVersionWithoutPatch(version string) string { + versionWithoutPatch := "" + + mainRe := regexp.MustCompile(`^main$`) + releaseRe := regexp.MustCompile(`^release-(\d+)\.(\d+)(-dev)?$`) + + switch { + case mainRe.MatchString(version): + versionWithoutPatch = "main" + case releaseRe.MatchString(version): + matches := releaseRe.FindStringSubmatch(version) + versionWithoutPatch = fmt.Sprintf("v%s.%s", matches[1], matches[2]) + default: + versionWithoutPatch = semver.MajorMinor(version) + } + + fmt.Println("The version is ", versionWithoutPatch) + + return versionWithoutPatch +} + func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") diff --git a/test/util/velero/velero_utils_test.go b/test/util/velero/velero_utils_test.go new file mode 100644 index 000000000..73df31b6e --- /dev/null +++ b/test/util/velero/velero_utils_test.go @@ -0,0 +1,54 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_getVersionWithoutPatch(t *testing.T) { + versionTests := []struct { + caseName string + version string + result string + }{ + { + caseName: "main version", + version: "main", + result: "main", + }, + { + caseName: "release version", + version: "release-1.18-dev", + result: "v1.18", + }, + { + caseName: "tag version", + version: "v1.17.2", + result: "v1.17", + }, + } + + for _, test := range versionTests { + t.Run(test.caseName, func(t *testing.T) { + res := getVersionWithoutPatch(test.version) + require.Equal(t, test.result, res) + }) + } +}