mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-07-19 22:42:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into vgdp-for-fs-backup-design

Browse Source
This commit is contained in:
lyndon-li
2025-06-16 14:09:05 +08:00
committed by GitHub
parent 5d1a4ee09e 7d8a36a6e0
commit 1d43322ced
87 changed files with 9377 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -58,3 +58,8 @@ debug.test*
# make lint cache
.cache/
# Go telemetry directory created when container sets HOME to working directory
# This happens because Makefile uses 'docker run -w /github.com/vmware-tanzu/velero'
# and Go's os.UserConfigDir() falls back to $HOME/.config when XDG_CONFIG_HOME is unset
.config/
changelogs/unreleased/8778-shubham-pampattiwar
+1
View File
@@ -0,0 +1 @@
Add design for VolumeGroupSnapshot support
changelogs/unreleased/8944-shubham-pampattiwar
+1
View File
@@ -0,0 +1 @@
Extend PVCAction itemblock plugin to support grouping PVCs under VGS label key
changelogs/unreleased/8976-blackpiglet
+1
View File
@@ -0,0 +1 @@
Add BSL status check for backup/restore operations.
changelogs/unreleased/8985-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Fix issue #8960, implement PodVolume exposer for PVB/PVR
changelogs/unreleased/8990-blackpiglet
+1
View File
@@ -0,0 +1 @@
Skip VS and VSC not created by backup.
changelogs/unreleased/8998-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Fix issue #8988, add data path for VGDP ms pvb
changelogs/unreleased/9005-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Fix issue #8988, add data path for VGDP ms PVR
changelogs/unreleased/9014-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Fix issue #8959, add VGDP MS PVR controller
changelogs/unreleased/9015-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Fix issue #8958, add VGDP MS PVB controller
config/crd/v1/bases/velero.io_podvolumebackups.yaml
+39 -20
View File
@@ -15,38 +15,41 @@ spec:
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: Pod Volume Backup status such as New/InProgress
- description: PodVolumeBackup status such as New/InProgress
jsonPath: .status.phase
name: Status
type: string
- description: Time when this backup was started
- description: Time duration since this PodVolumeBackup was started
jsonPath: .status.startTimestamp
name: Created
name: Started
type: date
- description: Namespace of the pod containing the volume to be backed up
jsonPath: .spec.pod.namespace
name: Namespace
type: string
- description: Name of the pod containing the volume to be backed up
jsonPath: .spec.pod.name
name: Pod
type: string
- description: Name of the volume to be backed up
jsonPath: .spec.volume
name: Volume
type: string
- description: The type of the uploader to handle data transfer
jsonPath: .spec.uploaderType
name: Uploader Type
type: string
- description: Completed bytes
format: int64
jsonPath: .status.progress.bytesDone
name: Bytes Done
type: integer
- description: Total bytes
format: int64
jsonPath: .status.progress.totalBytes
name: Total Bytes
type: integer
- description: Name of the Backup Storage Location where this backup should be
stored
jsonPath: .spec.backupStorageLocation
name: Storage Location
type: string
- jsonPath: .metadata.creationTimestamp
- description: Time duration since this PodVolumeBackup was created
jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: Name of the node where the PodVolumeBackup is processed
jsonPath: .status.node
name: Node
type: string
- description: The type of the uploader to handle data transfer
jsonPath: .spec.uploaderType
name: Uploader
type: string
name: v1
schema:
openAPIV3Schema:
@@ -76,6 +79,11 @@ spec:
BackupStorageLocation is the name of the backup storage location
where the backup repository is stored.
type: string
cancel:
description: |-
Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set
when the PodVolumeBackup is in InProgress phase
type: boolean
node:
description: Node is the name of the node that the Pod is running
on.
@@ -165,6 +173,13 @@ spec:
status:
description: PodVolumeBackupStatus is the current status of a PodVolumeBackup.
properties:
acceptedTimestamp:
description: |-
AcceptedTimestamp records the time the pod volume backup is to be prepared.
The server's time is used for AcceptedTimestamp
format: date-time
nullable: true
type: string
completionTimestamp:
description: |-
CompletionTimestamp records the time a backup was completed.
@@ -185,7 +200,11 @@ spec:
description: Phase is the current state of the PodVolumeBackup.
enum:
- New
- Accepted
- Prepared
- InProgress
- Canceling
- Canceled
- Completed
- Failed
type: string
config/crd/v1/bases/velero.io_podvolumerestores.yaml
+48 -27
View File
@@ -15,39 +15,40 @@ spec:
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: Namespace of the pod containing the volume to be restored
jsonPath: .spec.pod.namespace
name: Namespace
- description: PodVolumeRestore status such as New/InProgress
jsonPath: .status.phase
name: Status
type: string
- description: Name of the pod containing the volume to be restored
jsonPath: .spec.pod.name
name: Pod
- description: Time duration since this PodVolumeRestore was started
jsonPath: .status.startTimestamp
name: Started
type: date
- description: Completed bytes
format: int64
jsonPath: .status.progress.bytesDone
name: Bytes Done
type: integer
- description: Total bytes
format: int64
jsonPath: .status.progress.totalBytes
name: Total Bytes
type: integer
- description: Name of the Backup Storage Location where the backup data is stored
jsonPath: .spec.backupStorageLocation
name: Storage Location
type: string
- description: Time duration since this PodVolumeRestore was created
jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: Name of the node where the PodVolumeRestore is processed
jsonPath: .status.node
name: Node
type: string
- description: The type of the uploader to handle data transfer
jsonPath: .spec.uploaderType
name: Uploader Type
type: string
- description: Name of the volume to be restored
jsonPath: .spec.volume
name: Volume
type: string
- description: Pod Volume Restore status such as New/InProgress
jsonPath: .status.phase
name: Status
type: string
- description: Pod Volume Restore status such as New/InProgress
format: int64
jsonPath: .status.progress.totalBytes
name: TotalBytes
type: integer
- description: Pod Volume Restore status such as New/InProgress
format: int64
jsonPath: .status.progress.bytesDone
name: BytesDone
type: integer
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1
schema:
openAPIV3Schema:
@@ -77,6 +78,11 @@ spec:
BackupStorageLocation is the name of the backup storage location
where the backup repository is stored.
type: string
cancel:
description: |-
Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
when the PodVolumeRestore is in InProgress phase
type: boolean
pod:
description: Pod is a reference to the pod containing the volume to
be restored.
@@ -162,6 +168,13 @@ spec:
status:
description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore.
properties:
acceptedTimestamp:
description: |-
AcceptedTimestamp records the time the pod volume restore is to be prepared.
The server's time is used for AcceptedTimestamp
format: date-time
nullable: true
type: string
completionTimestamp:
description: |-
CompletionTimestamp records the time a restore was completed.
@@ -173,11 +186,19 @@ spec:
message:
description: Message is a message about the pod volume restore's status.
type: string
node:
description: Node is name of the node where the pod volume restore
is processed.
type: string
phase:
description: Phase is the current state of the PodVolumeRestore.
enum:
- New
- Accepted
- Prepared
- InProgress
- Canceling
- Canceled
- Completed
- Failed
type: string
config/crd/v1/crds/crds.go
+2 -2
View File
File diff suppressed because one or more lines are too long
design/Implemented/node-agent-affinity.md
+1 -1
View File
@@ -128,5 +128,5 @@ Once this problem happens, the backupPod stays in `Pending` phase, and the corre
On the other hand, the backupPod is deleted after the prepare timeout, so there is no way to tell the cause is one of the above problems or others.
To help the troubleshooting, we can add some diagnostic mechanism to discover the status of the backupPod and node-agent in the same node before deleting it as a result of the prepare timeout.
[1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
[1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
[2]: volume-snapshot-data-movement/volume-snapshot-data-movement.md
design/Implemented/repo_maintenance_job_config.md
+3 -3
View File
@@ -4,7 +4,7 @@
Add this design to make the repository maintenance job can read configuration from a dedicate ConfigMap and make the Job's necessary parts configurable, e.g. `PodSpec.Affinity` and `PodSpec.Resources`.
## Background
Repository maintenance is split from the Velero server to a k8s Job in v1.14 by design [repository maintenance job](Implemented/repository-maintenance.md).
Repository maintenance is split from the Velero server to a k8s Job in v1.14 by design [repository maintenance job](repository-maintenance.md).
The repository maintenance Job configuration was read from the Velero server CLI parameter, and it inherits the most of Velero server's Deployment's PodSpec to fill un-configured fields.
This design introduces a new way to let the user to customize the repository maintenance behavior instead of inheriting from the Velero server Deployment or reading from `velero server` CLI parameters.
@@ -13,7 +13,7 @@ It's possible new configurations are introduced in future releases based on this
For the node selection, the repository maintenance Job also inherits from the Velero server deployment before, but the Job may last for a while and cost noneligible resources, especially memory.
The users have the need to choose which k8s node to run the maintenance Job.
This design reuses the data structure introduced by design [node-agent affinity configuration](Implemented/node-agent-affinity.md) to make the repository maintenance job can choose which node running on.
This design reuses the data structure introduced by design [Velero Generic Data Path affinity configuration](node-agent-affinity.md) to make the repository maintenance job can choose which node running on.
## Goals
- Unify the repository maintenance Job configuration at one place.
@@ -118,7 +118,7 @@ For example, the following BackupRepository's key should be `test-default-kopia`
volumeNamespace: test
```
The `LoadAffinity` structure is reused from design [node-agent affinity configuration](Implemented/node-agent-affinity.md).
The `LoadAffinity` structure is reused from design [Velero Generic Data Path affinity configuration](node-agent-affinity.md).
It's possible that the users want to choose nodes that match condition A or condition B to run the job.
For example, the user want to let the nodes is in a specified machine type or the nodes locate in the us-central1-x zones to run the job.
This can be done by adding multiple entries in the `LoadAffinity` array.
design/vgdp-affinity-enhancement.md
+299
View File
@@ -0,0 +1,299 @@
# Velero Generic Data Path Load Affinity Enhancement Design
## Glossary & Abbreviation
**Velero Generic Data Path (VGDP)**: VGDP is the collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository.
**Exposer**: Exposer is a module that is introduced in [Volume Snapshot Data Movement Design][1]. Velero uses this module to expose the volume snapshots to Velero node-agent pods or node-agent associated pods so as to complete the data movement from the snapshots.
## Background
The implemented [VGDP LoadAffinity design][3] already defined the a structure `LoadAffinity` in `--node-agent-configmap` parameter. The parameter is used to set the affinity of the backupPod of VGDP.
There are still some limitations of this design:
* The affinity setting is global. Say there are two StorageClasses and the underlying storage can only provision volumes to part of the cluster nodes. The supported nodes don't have intersection. Then the affinity will definitely not work in some cases.
* The old design only take the first element of the []*LoadAffinity array. By this way, it cannot support the or logic between Affinity selectors.
* The old design focuses on the backupPod affinity, but the restorePod also needs the affinity setting.
As a result, create this design to address the limitations.
## Goals
- Enhance the node affinity of VGDP instances for volume snapshot data movement: add per StorageClass node affinity.
- Enhance the node affinity of VGDP instances for volume snapshot data movement: support the or logic between affinity selectors.
- Define the behaviors of node affinity of VGDP instances in node-agent for volume snapshot data movement restore, when the PVC restore doesn't require delay binding.
## Non-Goals
- It is also beneficial to support VGDP instances affinity for PodVolume backup/restore, this will be implemented after the PodVolume micro service completes.
## Solution
This design still uses the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` to host the node affinity configurations.
Upon the implemented [VGDP LoadAffinity design][3] introduced `[]*LoadAffinity` structure, this design add a new field `StorageClass`. This field is optional.
* If the `LoadAffinity` element's `StorageClass` doesn't have value, it means this element is applied to global, just as the old design.
* If the `LoadAffinity` element's `StorageClass` has value, it means this element is applied to the VGDP instances' PVCs use the specified StorageClass.
* To support the or logic between LoadAffinity elements, this design allows multiple instances of `LoadAffinity` whose `StorageClass` field have the same value.
* The `LoadAffinity` element whose `StorageClass` has value has higher priority than the `LoadAffinity` element whose `StorageClass` doesn't have value.
```go
type Configs struct {
// LoadConcurrency is the config for load concurrency per node.
LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"`
// LoadAffinity is the config for data path load affinity.
LoadAffinity []*LoadAffinity `json:"loadAffinity,omitempty"`
}
type LoadAffinity struct {
// NodeSelector specifies the label selector to match nodes
NodeSelector metav1.LabelSelector `json:"nodeSelector"`
}
```
``` go
type LoadAffinity struct {
// NodeSelector specifies the label selector to match nodes
NodeSelector metav1.LabelSelector `json:"nodeSelector"`
// StorageClass specifies the VGDPs the LoadAffinity applied to. If the StorageClass doesn't have value, it applies to all. If not, it applies to only the VGDPs that use this StorageClass.
StorageClass string `json:"storageClass"`
}
```
### Decision Tree
```mermaid
flowchart TD
A[VGDP Pod Needs Scheduling] --> B{Is this a restore operation?}
B -->|Yes| C{StorageClass has volumeBindingMode: WaitForFirstConsumer?}
B -->|No| D[Backup Operation]
C -->|Yes| E{restorePVC.ignoreDelayBinding = true?}
C -->|No| F[StorageClass binding mode: Immediate]
E -->|No| G[Wait for target Pod scheduling<br/>Use Pod's selected node<br/>⚠️ Affinity rules ignored]
E -->|Yes| H[Apply affinity rules<br/>despite WaitForFirstConsumer]
F --> I{Check StorageClass in loadAffinity by StorageClass field}
H --> I
D --> J{Using backupPVC with different StorageClass?}
J -->|Yes| K[Use final StorageClass<br/>for affinity lookup]
J -->|No| L[Use original PVC StorageClass<br/>for affinity lookup]
K --> I
L --> I
I -->|StorageClass found| N[Filter the LoadAffinity by <br/>the StorageClass<br/>🎯 and apply the LoadAffinity HIGHEST PRIORITY]
I -->|StorageClass not found| O{Check loadAffinity element without StorageClass field}
O -->|No loadAffinity configured| R[No affinity constraints<br/>Schedule on any available node<br/>🌐 DEFAULT]
N --> S{Multiple rules in array?}
S -->|Yes| T[Apply all rules as OR conditions<br/>Pod scheduled on nodes matching ANY rule]
S -->|No| U[Apply single rule<br/>Pod scheduled on nodes matching this rule]
O --> S
T --> V[Validate node-agent availability<br/>⚠️ Ensure node-agent pods exist on target nodes]
U --> V
V --> W{Node-agent available on selected nodes?}
W -->|Yes| X[✅ VGDP Pod scheduled successfully]
W -->|No| Y[❌ Pod stays in Pending state<br/>Timeout after 30min<br/>Check node-agent DaemonSet coverage]
R --> Z[Schedule on any node<br/>✅ Basic scheduling]
%% Styling
classDef successNode fill:#d4edda,stroke:#155724,color:#155724
classDef warningNode fill:#fff3cd,stroke:#856404,color:#856404
classDef errorNode fill:#f8d7da,stroke:#721c24,color:#721c24
classDef priorityHigh fill:#e7f3ff,stroke:#0066cc,color:#0066cc
classDef priorityMedium fill:#f0f8ff,stroke:#4d94ff,color:#4d94ff
classDef priorityDefault fill:#f8f9fa,stroke:#6c757d,color:#6c757d
class X,Z successNode
class G,V,Y warningNode
class Y errorNode
class N,T,U priorityHigh
class P,Q priorityMedium
class R priorityDefault
```
### Examples
#### Multiple LoadAffinities
``` json
{
"loadAffinity": [
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B4ms"
}
}
},
{
"nodeSelector": {
"matchExpressions": [
{
"key": "topology.kubernetes.io/zone",
"operator": "In",
"values": [
"us-central1-a"
]
}
]
}
}
]
}
```
This sample demonstrates how to use multiple affinities in `loadAffinity`. That can support more complicated scenarios, e.g. need to filter nodes satisfied either of two conditions, instead of satisfied both of two conditions.
In this example, the VGDP pods will be assigned to nodes, which instance type is `Standard_B4ms` or which zone is `us-central1-a`.
#### LoadAffinity interacts with LoadAffinityPerStorageClass
``` json
{
"loadAffinity": [
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B4ms"
}
}
},
{
"nodeSelector": {
"matchExpressions": [
{
"key": "kubernetes.io/os",
"values": [
"linux"
],
"operator": "In"
}
]
},
"storageClass": "kibishii-storage-class"
},
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B8ms"
}
},
"storageClass": "kibishii-storage-class"
}
]
}
```
This sample demonstrates how the `loadAffinity` elements with `StorageClass` field and without `StorageClass` field setting work together.
If the VGDP mounting volume is created from StorageClass `kibishii-storage-class`, its pod will run Linux nodes or instance type as `Standard_B8ms`.
The other VGDP instances will run on nodes, which instance type is `Standard_B4ms`.
#### LoadAffinity interacts with BackupPVC
``` json
{
"loadAffinity": [
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B4ms"
}
},
"storageClass": "kibishii-storage-class"
},
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B2ms"
}
},
"storageClass": "worker-storagepolicy"
}
],
"backupPVC": {
"kibishii-storage-class": {
"storageClass": "worker-storagepolicy"
}
}
}
```
Velero data mover supports to use different StorageClass to create backupPVC by [design](https://github.com/vmware-tanzu/velero/pull/7982).
In this example, if the backup target PVC's StorageClass is `kibishii-storage-class`, its backupPVC should use StorageClass `worker-storagepolicy`. Because the final StorageClass is `worker-storagepolicy`, the backupPod uses the loadAffinity specified by `loadAffinity`'s elements with `StorageClass` field set to `worker-storagepolicy`. backupPod will be assigned to nodes, which instance type is `Standard_B2ms`.
#### LoadAffinity interacts with RestorePVC
``` json
{
"loadAffinity": [
{
"nodeSelector": {
"matchLabels": {
"beta.kubernetes.io/instance-type": "Standard_B4ms"
}
},
"storageClass": "kibishii-storage-class"
}
],
"restorePVC": {
"ignoreDelayBinding": false
}
}
```
##### StorageClass's bind mode is WaitForFirstConsumer
``` yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: kibishii-storage-class
parameters:
svStorageClass: worker-storagepolicy
provisioner: csi.vsphere.vmware.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
```
If restorePVC should be created from StorageClass `kibishii-storage-class`, and it's volumeBindingMode is `WaitForFirstConsumer`.
Although `loadAffinityPerStorageClass` has a section matches the StorageClass, the `ignoreDelayBinding` is set `false`, the Velero exposer will wait until the target Pod scheduled to a node, and returns the node as SelectedNode for the restorePVC.
As a result, the `loadAffinityPerStorageClass` will not take affect.
##### StorageClass's bind mode is Immediate
``` yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: kibishii-storage-class
parameters:
svStorageClass: worker-storagepolicy
provisioner: csi.vsphere.vmware.com
reclaimPolicy: Delete
volumeBindingMode: Immediate
```
Because the StorageClass volumeBindingMode is `Immediate`, although `ignoreDelayBinding` is set to `false`, restorePVC will not be created according to the target Pod.
The restorePod will be assigned to nodes, which instance type is `Standard_B4ms`.
[1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
[2]: Implemented/volume-snapshot-data-movement/volume-snapshot-data-movement.md
[3]: Implemented/node-agent-affinity.md
design/volume-group-snapshot.md
+611
View File
@@ -0,0 +1,611 @@
# Add Support for VolumeGroupSnapshots
This proposal outlines the design and implementation plan for incorporating VolumeGroupSnapshot support into Velero. The enhancement will allow Velero to perform consistent, atomic snapshots of groups of Volumes using the new Kubernetes [VolumeGroupSnapshot API](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/). This capability is especially critical for stateful applications that rely on multiple volumes to ensure data consistency, such as databases and analytics workloads.
## Glossary & Abbreviation
Terminology used in this document:
- VGS: VolumeGroupSnapshot
- VS: VolumeSnapshot
- VGSC: VolumeGroupSnapshotContent
- VSC: VolumeSnapshotContent
- VGSClass: VolumeGroupSnapshotClass
- VSClass: VolumeSnapshotClass
## Background
Velero currently enables snapshot-based backups on an individual Volume basis through CSI drivers. However, modern stateful applications often require multiple volumes for data, logs, and backups. This distributed data architecture increases the risk of inconsistencies when volumes are captured individually. Kubernetes has introduced the VolumeGroupSnapshot(VGS) API [(KEP-3476)](https://github.com/kubernetes/enhancements/pull/1551), which allows for the atomic snapshotting of multiple volumes in a coordinated manner. By integrating this feature, Velero can offer enhanced disaster recovery for multi-volume applications, ensuring consistency across all related data.
## Goals
- Ensure that multiple related volumes are snapshotted simultaneously, preserving consistency for stateful applications via VolumeGroupSnapshots(VGS) API.
- Integrate VolumeGroupSnapshot functionality into Velero’s existing backup and restore workflows.
- Allow users to opt in to volume group snapshots via specifying the group label.
## Non-Goals
- The proposal does not require a complete overhaul of Velero’s CSI integration, it will extend the current mechanism to support group snapshots.
- No any changes pertaining to execution of Restore Hooks
## High-Level Design
### Backup workflow:
#### Accept the label to be used for VGS from the user:
- Accept the label from the user, we will do this in 3 ways:
- Firstly, we will have a hard-coded default label key like `velero.io/volume-group-snapshot` that the users can directly use on their PVCs.
- Secondly, we will let the users override this default VGS label via a velero server arg, `--volume-group-nsaphot-label-key`, if needed.
- And Finally we will have the option to override the default label via Backup API spec, `backup.spec.volumeGroupSnapshotLabelKey`
- In all the instances, the VGS label key will be present on the backup spec, this makes the label key accessible to plugins during the execution of backup operation.
- This label will enable velero to filter the PVC to be included in the VGS spec.
- Users will have to label the PVCs before invoking the backup operation.
- This label would act as a group identifier for the PVCs to be grouped under a specific VGS.
- It will be used to collect the PVCs to be used for a particular instance of VGS object.
**Note:**
- Modifying or adding VGS label on PVCs during an active backup operation may lead to unexpected or undesirable backup results. To avoid inconsistencies, ensure PVC labels remain unchanged throughout the backup execution.
- Label Key Precedence: When determining which label key to use for grouping PVCs into a VolumeGroupSnapshot, Velero applies overrides in the following order (highest to lowest):
- Backup API spec (`backup.spec.volumeGroupSnapshotLabelKey`)
- Server flag (`--volume-group-snapshot-label-key`)
- Built-in default (`velero.io/volume-group-snapshot`)
Whichever key wins this precedence is then injected into the Backup spec so that all Velero plugins can uniformly discover and use it during the backup execution.
#### Changes to the Existing PVC ItemBlockAction plugin:
- Currently the PVC IBA plugin is applied to PVCs and adds the RelatedItems for the particular PVC into the ItemBlock.
- At first it checks whether the PVC is bound and VolumeName is non-empty.
- Then it adds the related PV under the list of relatedItems.
- Following on, the plugin adds the pods mounting the PVC as relatedItems.
- Now we need to extend this PVC IBA plugin to add the PVCs to be grouped for a particular VGS object, so that they are processed together under an ItemBlock by Velero.
- First we will check if the PVC that is being processed by the plugin has the user specified VGS label.
- If it is present then we will execute a List call in the namespace with the label as a matching criteria and see if this results in any PVCs (other than the current one).
- If there are PVCs matching the criteria then we add the PVCs to the relatedItems list.
- This helps in building the ItemBlock we need for VGS processing, i.e. we have the relevant pods and PVCs in the ItemBlock.
**Note:** The ItemBlock to VGS relationship will not always be 1:1. There might be scenarios when the ItemBlock might have multiple VGS instances associated with it.
Lets go over some ItemBlock/VGS scenarios that we might encounter and visualize them for clarity:
1. Pod Mounts: Pod1 mounts both PVC1 and PVC2.
Grouping: PVC1 and PVC2 share the same group label (group: A)
ItemBlock: The item block includes Pod1, PVC1, and PVC2.
VolumeGroupSnapshot (VGS): Because PVC1 and PVC2 are grouped together by their label, they trigger the creation of a single VGS (labeled with group: A).
```mermaid
flowchart TD
subgraph ItemBlock
P1[Pod1]
PVC1[PVC1 group: A]
PVC2[PVC2 group: A]
end
P1 -->|mounts| PVC1
P1 -->|mounts| PVC2
PVC1 --- PVC2
PVC1 -- "group: A" --> VGS[VGS group: A]
PVC2 -- "group: A" --> VGS
```
2. Pod Mounts: Pod1 mounts each of the four PVCs.
Grouping:
Group A: PVC1 and PVC2 share the same grouping label (group: A).
Group B: PVC3 and PVC4 share the grouping label (group: B)
ItemBlock: All objects (Pod1, PVC1, PVC2, PVC3, and PVC4) are collected into a single item block.
VolumeGroupSnapshots:
PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)).
PVC3 and PVC4 (group B) point to a different VGS (VGS (group: B)).
```mermaid
flowchart TD
subgraph ItemBlock
P1[Pod1]
PVC1[PVC1 group: A]
PVC2[PVC2 group: A]
PVC3[PVC3 group: B]
PVC4[PVC4 group: B]
end
%% Pod mounts all PVCs
P1 -->|mounts| PVC1
P1 -->|mounts| PVC2
P1 -->|mounts| PVC3
P1 -->|mounts| PVC4
%% Group A relationships: PVC1 and PVC2
PVC1 --- PVC2
PVC1 -- "group: A" --> VGS_A[VGS-A group: A]
PVC2 -- "group: A" --> VGS_A
%% Group B relationships: PVC3 and PVC4
PVC3 --- PVC4
PVC3 -- "group: B" --> VGS_B[VGS-B group: B]
PVC4 -- "group: B" --> VGS_B
```
3. Pod Mounts: Pod1 mounts both PVC1 and PVC2, Pod2 mounts PVC1 and PVC3.
Grouping:
Group A: PVC1 and PVC2
Group B: PVC3
ItemBlock: All objects-Pod1, Pod2, PVC1, PVC2, and PVC3, are collected into a single item block.
VolumeGroupSnapshots:
PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)).
PVC3 (group B) point to a different VGS (VGS (group: B)).
```mermaid
flowchart TD
subgraph ItemBlock
P1[Pod1]
P2[Pod2]
PVC1[PVC1 group: A]
PVC2[PVC2 group: A]
PVC3[PVC3 group: B]
end
%% Pod mount relationships
P1 -->|mounts| PVC1
P1 -->|mounts| PVC2
P2 -->|mounts| PVC1
P2 -->|mounts| PVC3
%% Grouping for Group A: PVC1 and PVC2 are grouped into VGS_A
PVC1 --- PVC2
PVC1 -- "Group A" --> VGS_A[VGS Group A]
PVC2 -- "Group A" --> VGS_A
%% Grouping for Group B: PVC3 grouped into VGS_B
PVC3 -- "Group B" --> VGS_B[VGS Group B]
```
#### Updates to CSI PVC plugin:
The CSI PVC plugin now supports obtaining a VolumeSnapshot (VS) reference for a PVC in three ways, and then applies common branching for datamover and non‑datamover workflows:
- Scenario 1: PVC has a VGS label and no VS (created via the VGS workflow) exists for its volume group:
- Determine VGSClass: The plugin will pick `VolumeGroupSnapshotClass` by following the same tier based precedence as it does for individual `VolumeSnapshotClasses`:
- Default by Label: Use the one VGSClass labeled
```yaml
metadata:
labels:
velero.io/csi-volumegroupsnapshot-class: "true"
```
whose `spec.driver` matches the CSI driver used by the PVCs.
- Backup‑level Override: If the Backup CR has an annotation
```yaml
metadata:
annotations:
velero.io/csi-volumegroupsnapshot-class_<driver>: <className>
```
(with <driver> equal to the PVCs’ CSI driver), use that class.
- PVC‑level Override: Finally, if the PVC itself carries an annotation
```yaml
metadata:
annotations:
velero.io/csi-volume-group-snapshot-class: <className>
```
and that class exists, use it.
At each step, if the plugin finds zero or multiple matching classes, VGS creation is skipped and backup fails.
- Create VGS: The plugin creates a new VolumeGroupSnapshot (VGS) for the PVC’s volume group. This action automatically triggers creation of the corresponding VGSC, VS, and VSC objects.
- Wait for VS Status: The plugin waits until each VS (one per PVC in the group) has its `volumeGroupSnapshotName` populated. This confirms that the snapshot controller has completed its work. `CSISnapshotTimeout` will be used here.
- Update VS Objects: Once the VS objects are provisioned, the plugin updates them by removing VGS owner references and VGS-related finalizers, and by adding backup metadata labels (including BackupName, BackupUUID, and PVC name). These labels are later used to detect an existing VS when processing another PVC of the same group.
- Patch and Cleanup: The plugin patches the deletionPolicy of the VGSC to "Retain" (ensuring that deletion of the VGSC does not remove the underlying VSC objects or storage snapshots) and then deletes the temporary VGS and VGSC objects.
- Scenario 2: PVC has a VGS label and a VS created via an earlier VGS workflow already exists:
- The plugin lists VS objects in the PVC’s namespace using backup metadata labels (BackupUID, BackupName, and PVCName).
- It verifies that at least one VS has a non‑empty `volumeGroupSnapshotName` in its status.
- If such a VS exists, the plugin skips creating a new VGS (or VS) and proceeds with the legacy workflow using the existing VS.
- If a VS is found but its status does not indicate it was created by the VGS workflow (i.e. its `volumeGroupSnapshotName` is empty), the backup for that PVC is failed, resulting in a partially failed backup.
- Scenario 3: PVC does not have a VGS label:
- The legacy workflow is followed, and an individual VolumeSnapshot (VS) is created for the PVC.
- Common Branching for Datamover and Non‑datamover Workflows:
- Once a VS reference (`vsRef`) is determined—whether through the VGS workflow (Scenario 1 or 2) or the legacy workflow (Scenario 3)—the plugin then applies the common branching:
- Non‑datamover Case: The VS reference is directly added as an additional backup item.
- Datamover Case: The plugin waits until the VS’s associated VSC snapshot handle is ready (using the configured CSISnapshotTimeout), then creates a DataUpload for the VS–PVC pair. The resulting DataUpload is then added as an additional backup item.
```mermaid
flowchart TD
%% Section 1: Accept VGS Label from User
subgraph Accept_Label
A1[User sets VGS label key using default velero.io/volume-group-snapshot or via server arg or Backup API spec]
A2[User labels PVCs before backup]
A1 --> A2
end
%% Section 2: PVC ItemBlockAction Plugin Extension
subgraph PVC_ItemBlockAction
B1[Check PVC is bound and has VolumeName]
B2[Add related PV to relatedItems]
B3[Add pods mounting PVC to relatedItems]
B4[Check if PVC has user-specified VGS label]
B5[List PVCs in namespace matching label criteria]
B6[Add matching PVCs to relatedItems]
B1 --> B2 --> B3 --> B4
B4 -- Yes --> B5
B5 --> B6
end
%% Section 3: CSI PVC Plugin Updates
subgraph CSI_PVC_Plugin
C1[For each PVC, check for VGS label]
C1 -- Has VGS label --> C2[Determine scenario]
C1 -- No VGS label --> C16[Scenario 3: Legacy workflow - create individual VS]
%% Scenario 1: No existing VS via VGS exists
subgraph Scenario1[Scenario 1: No existing VS via VGS]
S1[List grouped PVCs using VGS label]
S2[Determine CSI driver for grouped PVCs]
S3[If single CSI driver then select matching VGSClass; else fail backup]
S4[Create new VGS triggering VGSC, VS, and VSC creation]
S5[Wait for VS objects to have nonempty volumeGroupSnapshotName]
S6[Update VS objects; remove VGS owner refs and finalizers; add backup metadata labels]
S7[Patch VGSC deletionPolicy to Retain]
S8[Delete transient VGS and VGSC]
S1 --> S2 --> S3 --> S4 --> S5 --> S6 --> S7 --> S8
end
%% Scenario 2: Existing VS via VGS exists
subgraph Scenario2[Scenario 2: Existing VS via VGS exists]
S9[List VS objects using backup metadata - BackupUID, BackupName, PVCName]
S10[Check if any VS has nonempty volumeGroupSnapshotName]
S9 --> S10
S10 -- Yes --> S11[Use existing VS]
S10 -- No --> S12[Fail backup for PVC]
end
C2 -- Scenario1 applies --> S1
C2 -- Scenario2 applies --> S9
%% Common Branch: After obtaining a VS reference
subgraph Common_Branch[Common Branch]
CB1[Obtain VS reference as vsRef]
CB2[If non-datamover, add vsRef as additional backup item]
CB3[If datamover, wait for VSC handle and create DataUpload; add DataUpload as additional backup item]
CB1 --> CB2
CB1 --> CB3
end
%% Connect Scenario outcomes and legacy branch to the common branch
S8 --> CB1
S11 --> CB1
C16 --> CB1
end
%% Overall Flow Connections
A2 --> B1
B6 --> C1
```
Restore workflow:
- No changes required for the restore workflow.
## Detailed Design
Backup workflow:
- Accept the label to be used for VGS from the user as a server argument:
- Set a default VGS label key to be used:
```go
// default VolumeGroupSnapshot Label
defaultVGSLabelKey = "velero.io/volume-group-snapshot"
```
- Add this as a server flag and pass it to backup reconciler, so that we can use it during the backup request execution.
```go
flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot")
```
- Update the Backup CRD to accept the VGS Label Key as a spec value:
```go
// VolumeGroupSnapshotLabelKey specifies the label key to be used for grouping the PVCs under
// an instance of VolumeGroupSnapshot, if left unspecified velero.io/volume-group-snapshot is used
// +optional
VolumeGroupSnapshotLabelKey string `json:"volumeGroupSnapshotLabelKey,omitempty"`
```
- Modify the [`prepareBackupRequest` function](https://github.com/openshift/velero/blob/8c8a6cccd78b78bd797e40189b0b9bee46a97f9e/pkg/controller/backup_controller.go#L327) to set the default label key as a backup spec if the user does not specify any value:
```go
if len(request.Spec.VolumeGroupSnapshotLabelKey) == 0 {
// set the default key value
request.Spec.VolumeGroupSnapshotLabelKey = b.defaultVGSLabelKey
}
```
- Changes to the Existing [PVC ItemBlockAction plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/itemblock/actions/pvc_action.go#L64) (Update the GetRelatedItems function):
```go
// Retrieve the VGS label key from the Backup spec.
vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey
if vgsLabelKey != "" {
// Check if the PVC has the specified VGS label.
if groupID, ok := pvc.Labels[vgsLabelKey]; ok {
// List all PVCs in the namespace with the same label key and value (i.e. same group).
pvcList := new(corev1api.PersistentVolumeClaimList)
if err := a.crClient.List(context.Background(), pvcList, crclient.InNamespace(pvc.Namespace), crclient.MatchingLabels{vgsLabelKey: groupID}); err != nil {
return nil, errors.Wrap(err, "failed to list PVCs for VGS grouping")
}
// Add each matching PVC (except the current one) to the relatedItems.
for _, groupPVC := range pvcList.Items {
if groupPVC.Name == pvc.Name {
continue
}
a.log.Infof("Adding grouped PVC %s to relatedItems for PVC %s", groupPVC.Name, pvc.Name)
relatedItems = append(relatedItems, velero.ResourceIdentifier{
GroupResource: kuberesource.PersistentVolumeClaims,
Namespace: groupPVC.Namespace,
Name: groupPVC.Name,
})
}
}
} else {
a.log.Info("No VolumeGroupSnapshotLabelKey provided in backup spec; skipping PVC grouping")
}
```
- Updates to [CSI PVC plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/backup/actions/csi/pvc_action.go#L200) (Update the Execute method):
```go
func (p *pvcBackupItemAction) Execute(
item runtime.Unstructured,
backup *velerov1api.Backup,
) (
runtime.Unstructured,
[]velero.ResourceIdentifier,
string,
[]velero.ResourceIdentifier,
error,
) {
p.log.Info("Starting PVCBackupItemAction")
// Validate backup policy and PVC/PV
if valid := p.validateBackup(*backup); !valid {
return item, nil, "", nil, nil
}
var pvc corev1api.PersistentVolumeClaim
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil {
return nil, nil, "", nil, errors.WithStack(err)
}
if valid, item, err := p.validatePVCandPV(pvc, item); !valid {
if err != nil {
return nil, nil, "", nil, err
}
return item, nil, "", nil, nil
}
shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithBackup(
item,
kuberesource.PersistentVolumeClaims,
*backup,
p.crClient,
p.log,
)
if err != nil {
return nil, nil, "", nil, err
}
if !shouldSnapshot {
p.log.Debugf("CSI plugin skip snapshot for PVC %s according to VolumeHelper setting", pvc.Namespace+"/"+pvc.Name)
return nil, nil, "", nil, nil
}
var additionalItems []velero.ResourceIdentifier
var operationID string
var itemToUpdate []velero.ResourceIdentifier
// vsRef will be our common reference to the VolumeSnapshot (VS)
var vsRef *corev1api.ObjectReference
// Retrieve the VGS label key from the backup spec.
vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey
// Check if the PVC has the user-specified VGS label.
if group, ok := pvc.Labels[vgsLabelKey]; ok && group != "" {
p.log.Infof("PVC %s has VGS label with group %s", pvc.Name, group)
// --- VGS branch ---
// 1. Check if a VS created via a VGS workflow exists for this PVC.
existingVS, err := p.findExistingVSForBackup(backup.UID, backup.Name, pvc.Name, pvc.Namespace)
if err != nil {
return nil, nil, "", nil, err
}
if existingVS != nil && existingVS.Status.VolumeGroupSnapshotName != "" {
p.log.Infof("Existing VS %s found for PVC %s in group %s; skipping VGS creation", existingVS.Name, pvc.Name, group)
vsRef = &corev1api.ObjectReference{
Namespace: existingVS.Namespace,
Name: existingVS.Name,
}
} else {
// 2. No existing VS via VGS; execute VGS creation workflow.
groupedPVCs, err := p.listGroupedPVCs(backup, pvc.Namespace, vgsLabelKey, group)
if err != nil {
return nil, nil, "", nil, err
}
pvcNames := extractPVCNames(groupedPVCs)
// Determine the CSI driver used by the grouped PVCs.
driver, err := p.determineCSIDriver(groupedPVCs)
if err != nil {
return nil, nil, "", nil, errors.Wrap(err, "failed to determine CSI driver for grouped PVCs")
}
if driver == "" {
return nil, nil, "", nil, errors.New("multiple CSI drivers found for grouped PVCs; failing backup")
}
// Retrieve the appropriate VGSClass for the CSI driver.
vgsClass := p.getVGSClassForDriver(driver)
p.log.Infof("Determined CSI driver %s with VGSClass %s for PVC group %s", driver, vgsClass, group)
newVGS, err := p.createVolumeGroupSnapshot(backup, pvc, pvcNames, vgsLabelKey, group, vgsClass)
if err != nil {
return nil, nil, "", nil, err
}
p.log.Infof("Created new VGS %s for PVC group %s", newVGS.Name, group)
// Wait for the VS objects created via VGS to have volumeGroupSnapshotName in status.
if err := p.waitForVGSAssociatedVS(newVGS, pvc.Namespace, backup.Spec.CSISnapshotTimeout.Duration); err != nil {
return nil, nil, "", nil, err
}
// Update the VS objects: remove VGS owner references and finalizers; add backup metadata labels.
if err := p.updateVGSCreatedVS(newVGS, backup); err != nil {
return nil, nil, "", nil, err
}
// Patch the VGSC deletionPolicy to Retain.
if err := p.patchVGSCDeletionPolicy(newVGS, pvc.Namespace); err != nil {
return nil, nil, "", nil, err
}
// Delete the VGS and VGSC
if err := p.deleteVGSAndVGSC(newVGS, pvc.Namespace); err != nil {
return nil, nil, "", nil, err
}
// Fetch the VS that was created for this PVC via VGS.
vs, err := p.getVSForPVC(backup, pvc, vgsLabelKey, group)
if err != nil {
return nil, nil, "", nil, err
}
vsRef = &corev1api.ObjectReference{
Namespace: vs.Namespace,
Name: vs.Name,
}
}
} else {
// Legacy workflow: PVC does not have a VGS label; create an individual VS.
vs, err := p.createVolumeSnapshot(pvc, backup)
if err != nil {
return nil, nil, "", nil, err
}
vsRef = &corev1api.ObjectReference{
Namespace: vs.Namespace,
Name: vs.Name,
}
}
// --- Common Branch ---
// Now we have vsRef populated from one of the above cases.
// Branch further based on backup.Spec.SnapshotMoveData.
if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
// Datamover case:
operationID = label.GetValidName(
string(velerov1api.AsyncOperationIDPrefixDataUpload) + string(backup.UID) + "." + string(pvc.UID),
)
dataUploadLog := p.log.WithFields(logrus.Fields{
"Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name),
"VolumeSnapshot": fmt.Sprintf("%s/%s", vsRef.Namespace, vsRef.Name),
"Operation ID": operationID,
"Backup": backup.Name,
})
// Retrieve the current VS using vsRef
vs := &snapshotv1api.VolumeSnapshot{}
if err := p.crClient.Get(context.TODO(), crclient.ObjectKey{Namespace: vsRef.Namespace, Name: vsRef.Name}, vs); err != nil {
return nil, nil, "", nil, errors.Wrapf(err, "failed to get VolumeSnapshot %s", vsRef.Name)
}
// Wait until the VS-associated VSC snapshot handle is ready.
_, err := csi.WaitUntilVSCHandleIsReady(
vs,
p.crClient,
p.log,
true,
backup.Spec.CSISnapshotTimeout.Duration,
)
if err != nil {
dataUploadLog.Errorf("Failed to wait for VolumeSnapshot to become ReadyToUse: %s", err.Error())
csi.CleanupVolumeSnapshot(vs, p.crClient, p.log)
return nil, nil, "", nil, errors.WithStack(err)
}
dataUploadLog.Info("Starting data upload of backup")
dataUpload, err := createDataUpload(
context.Background(),
backup,
p.crClient,
vs,
&pvc,
operationID,
)
if err != nil {
dataUploadLog.WithError(err).Error("Failed to submit DataUpload")
if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil && !apierrors.IsNotFound(deleteErr) {
dataUploadLog.WithError(deleteErr).Error("Failed to delete VolumeSnapshot")
}
return item, nil, "", nil, nil
}
dataUploadLog.Info("DataUpload submitted successfully")
itemToUpdate = []velero.ResourceIdentifier{
{
GroupResource: schema.GroupResource{
Group: "velero.io",
Resource: "datauploads",
},
Namespace: dataUpload.Namespace,
Name: dataUpload.Name,
},
}
annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name
// For the datamover case, add the dataUpload as an additional item directly.
vsRef = &corev1api.ObjectReference{
Namespace: dataUpload.Namespace,
Name: dataUpload.Name,
}
additionalItems = append(additionalItems, velero.ResourceIdentifier{
GroupResource: schema.GroupResource{
Group: "velero.io",
Resource: "datauploads",
},
Namespace: dataUpload.Namespace,
Name: dataUpload.Name,
})
} else {
// Non-datamover case:
// Use vsRef for snapshot purposes.
additionalItems = append(additionalItems, convertVSToResourceIdentifiersFromRef(vsRef)...)
p.log.Infof("VolumeSnapshot additional item added for VS %s", vsRef.Name)
}
// Update PVC metadata with common labels and annotations.
labels := map[string]string{
velerov1api.VolumeSnapshotLabel: vsRef.Name,
velerov1api.BackupNameLabel: backup.Name,
}
annotations := map[string]string{
velerov1api.VolumeSnapshotLabel: vsRef.Name,
velerov1api.MustIncludeAdditionalItemAnnotation: "true",
}
kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations)
kubeutil.AddLabels(&pvc.ObjectMeta, labels)
p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", len(additionalItems))
for _, ai := range additionalItems {
p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name)
}
pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc)
if err != nil {
return nil, nil, "", nil, errors.WithStack(err)
}
return &unstructured.Unstructured{Object: pvcMap},
additionalItems, operationID, itemToUpdate, nil
}
```
## Implementation
This design proposal is targeted for velero 1.16.
The implementation of this proposed design is targeted for velero 1.17.
**Note:**
- VGS support isn't a requirement on restore. The design does not have any VGS related elements/considerations in the restore workflow.
## Requirements and Assumptions
- Kubernetes Version:
- Minimum: v1.32.0 or later, since the VolumeGroupSnapshot API goes beta in 1.32.
- Assumption: CRDs for `VolumeGroupSnapshot`, `VolumeGroupSnapshotClass`, and `VolumeGroupSnapshotContent` are already installed.
- VolumeGroupSnapshot API Availability:
- If the VGS API group (`groupsnapshot.storage.k8s.io/v1beta1`) is not present, Velero backup will fail.
- CSI Driver Compatibility
- Only CSI drivers that implement the VolumeGroupSnapshot admission and controller support this feature.
- Upon VGS creation, we assume the driver will atomically snapshot all matching PVCs; if it does not, the plugin may time out.
## Performance Considerations
- Use VGS if you have many similar volumes that must be snapped together and you want to minimize API/server load.
- Use individual VS if you have only a few volumes, or want one‐volume failures to be isolated.
## Testing Strategy
- Unit tests: We will add targeted unit tests to cover all new code paths—including existing-VS detection, VGS creation, legacy VS fallback, and error scenarios.
- E2E tests: For E2E we would need, a Kind cluster with a CSI driver that supports group snapshots, deploy an application with multiple PVCs, execute a Velero backup and restore, and verify that VGS is created, all underlying VS objects reach ReadyToUse, and every PVC is restored successfully.
hack/fix_restic_cve.txt
+28 -28
View File
@@ -1,8 +1,8 @@
diff --git a/go.mod b/go.mod
index 5f939c481..5c5db077f 100644
index 5f939c481..6ae17f4a1 100644
--- a/go.mod
+++ b/go.mod
@@ -24,32 +24,32 @@ require (
@@ -24,32 +24,31 @@ require (
github.com/restic/chunker v0.4.0
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
@@ -16,7 +16,7 @@ index 5f939c481..5c5db077f 100644
- google.golang.org/api v0.106.0
+ golang.org/x/crypto v0.36.0
+ golang.org/x/net v0.38.0
+ golang.org/x/oauth2 v0.7.0
+ golang.org/x/oauth2 v0.28.0
+ golang.org/x/sync v0.12.0
+ golang.org/x/sys v0.31.0
+ golang.org/x/term v0.30.0
@@ -27,10 +27,10 @@ index 5f939c481..5c5db077f 100644
require (
- cloud.google.com/go v0.108.0 // indirect
- cloud.google.com/go/compute v1.15.1 // indirect
+ cloud.google.com/go v0.110.0 // indirect
+ cloud.google.com/go/compute v1.19.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
- cloud.google.com/go/compute/metadata v0.2.3 // indirect
- cloud.google.com/go/iam v0.10.0 // indirect
+ cloud.google.com/go v0.110.0 // indirect
+ cloud.google.com/go/compute/metadata v0.3.0 // indirect
+ cloud.google.com/go/iam v0.13.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@@ -49,7 +49,7 @@ index 5f939c481..5c5db077f 100644
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
@@ -63,11 +63,13 @@ require (
@@ -63,11 +62,13 @@ require (
go.opencensus.io v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
@@ -68,24 +68,24 @@ index 5f939c481..5c5db077f 100644
+
+toolchain go1.23.7
diff --git a/go.sum b/go.sum
index 026e1d2fa..836a9b274 100644
index 026e1d2fa..805792055 100644
--- a/go.sum
+++ b/go.sum
@@ -1,23 +1,26 @@
@@ -1,23 +1,24 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk=
-cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q=
-cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE=
-cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
+cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
+cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
+cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
+cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI=
-cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
-cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
+cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
+cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
+cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
+cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
+cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
+cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
@@ -105,7 +105,7 @@ index 026e1d2fa..836a9b274 100644
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do=
@@ -54,6 +57,7 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
@@ -54,6 +55,7 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
@@ -113,7 +113,7 @@ index 026e1d2fa..836a9b274 100644
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
@@ -70,8 +74,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
@@ -70,8 +72,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -124,7 +124,7 @@ index 026e1d2fa..836a9b274 100644
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -82,17 +86,18 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
@@ -82,17 +84,18 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -148,7 +148,7 @@ index 026e1d2fa..836a9b274 100644
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -114,6 +119,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -114,6 +117,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs=
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -156,7 +156,7 @@ index 026e1d2fa..836a9b274 100644
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.46 h1:Vo3tNmNXuj7ME5qrvN4iadO7b4mzu/RSFdUkUhaPldk=
@@ -129,6 +135,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P
@@ -129,6 +133,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P
github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
@@ -164,7 +164,7 @@ index 026e1d2fa..836a9b274 100644
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
@@ -172,8 +179,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
@@ -172,8 +177,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -175,7 +175,7 @@ index 026e1d2fa..836a9b274 100644
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -189,17 +196,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
@@ -189,17 +194,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -186,8 +186,8 @@ index 026e1d2fa..836a9b274 100644
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
-golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
+golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc=
+golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -199,7 +199,7 @@ index 026e1d2fa..836a9b274 100644
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -214,17 +221,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
@@ -214,17 +219,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -223,7 +223,7 @@ index 026e1d2fa..836a9b274 100644
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -237,8 +244,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
@@ -237,8 +242,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
@@ -234,7 +234,7 @@ index 026e1d2fa..836a9b274 100644
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
@@ -246,15 +253,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
@@ -246,15 +251,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
@@ -254,7 +254,7 @@ index 026e1d2fa..836a9b274 100644
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -266,14 +273,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
@@ -266,14 +271,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
internal/credentials/local.go
+7
View File
@@ -0,0 +1,7 @@
package credentials
import "os"
func DefaultStoreDirectory() string {
return os.TempDir() + "/credentials"
}
pkg/apis/velero/v1/labels_annotations.go
+9
View File
@@ -101,6 +101,15 @@ const (
// ExcludeFromBackupLabel is the label to exclude k8s resource from backup,
// even if the resource contains a matching selector label.
ExcludeFromBackupLabel = "velero.io/exclude-from-backup"
// defaultVGSLabelKey is the default label key used to group PVCs under a VolumeGroupSnapshot
DefaultVGSLabelKey = "velero.io/volume-group"
// PVBLabel is the label key used to identify the pvb for pvb pod
PVBLabel = "velero.io/pod-volume-backup"
// PVRLabel is the label key used to identify the pvb for pvr pod
PVRLabel = "velero.io/pod-volume-restore"
)
type AsyncOperationIDPrefix string
pkg/apis/velero/v1/pod_volume_backup_types.go
+23 -8
View File
@@ -57,15 +57,23 @@ type PodVolumeBackupSpec struct {
// +optional
// +nullable
UploaderSettings map[string]string `json:"uploaderSettings,omitempty"`
// Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set
// when the PodVolumeBackup is in InProgress phase
Cancel bool `json:"cancel,omitempty"`
}
// PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup.
// +kubebuilder:validation:Enum=New;InProgress;Completed;Failed
// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed
type PodVolumeBackupPhase string
const (
PodVolumeBackupPhaseNew PodVolumeBackupPhase = "New"
PodVolumeBackupPhaseAccepted PodVolumeBackupPhase = "Accepted"
PodVolumeBackupPhasePrepared PodVolumeBackupPhase = "Prepared"
PodVolumeBackupPhaseInProgress PodVolumeBackupPhase = "InProgress"
PodVolumeBackupPhaseCanceling PodVolumeBackupPhase = "Canceling"
PodVolumeBackupPhaseCanceled PodVolumeBackupPhase = "Canceled"
PodVolumeBackupPhaseCompleted PodVolumeBackupPhase = "Completed"
PodVolumeBackupPhaseFailed PodVolumeBackupPhase = "Failed"
)
@@ -109,20 +117,27 @@ type PodVolumeBackupStatus struct {
// about the backup operation.
// +optional
Progress shared.DataMoveOperationProgress `json:"progress,omitempty"`
// AcceptedTimestamp records the time the pod volume backup is to be prepared.
// The server's time is used for AcceptedTimestamp
// +optional
// +nullable
AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"`
}
// TODO(2.0) After converting all resources to use the runttime-controller client,
// the genclient and k8s:deepcopy markers will no longer be needed and should be removed.
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Pod Volume Backup status such as New/InProgress"
// +kubebuilder:printcolumn:name="Created",type="date",JSONPath=".status.startTimestamp",description="Time when this backup was started"
// +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.pod.namespace",description="Namespace of the pod containing the volume to be backed up"
// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.pod.name",description="Name of the pod containing the volume to be backed up"
// +kubebuilder:printcolumn:name="Volume",type="string",JSONPath=".spec.volume",description="Name of the volume to be backed up"
// +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer"
// +kubebuilder:storageversion
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeBackup status such as New/InProgress"
// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeBackup was started"
// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes"
// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes"
// +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeBackup was created"
// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeBackup is processed"
// +kubebuilder:printcolumn:name="Uploader",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer"
// +kubebuilder:object:root=true
// +kubebuilder:object:generate=true
pkg/apis/velero/v1/pod_volume_restore_type.go
+26 -8
View File
@@ -54,15 +54,23 @@ type PodVolumeRestoreSpec struct {
// +optional
// +nullable
UploaderSettings map[string]string `json:"uploaderSettings,omitempty"`
// Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
// when the PodVolumeRestore is in InProgress phase
Cancel bool `json:"cancel,omitempty"`
}
// PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore.
// +kubebuilder:validation:Enum=New;InProgress;Completed;Failed
// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed
type PodVolumeRestorePhase string
const (
PodVolumeRestorePhaseNew PodVolumeRestorePhase = "New"
PodVolumeRestorePhaseAccepted PodVolumeRestorePhase = "Accepted"
PodVolumeRestorePhasePrepared PodVolumeRestorePhase = "Prepared"
PodVolumeRestorePhaseInProgress PodVolumeRestorePhase = "InProgress"
PodVolumeRestorePhaseCanceling PodVolumeRestorePhase = "Canceling"
PodVolumeRestorePhaseCanceled PodVolumeRestorePhase = "Canceled"
PodVolumeRestorePhaseCompleted PodVolumeRestorePhase = "Completed"
PodVolumeRestorePhaseFailed PodVolumeRestorePhase = "Failed"
)
@@ -95,6 +103,16 @@ type PodVolumeRestoreStatus struct {
// about the restore operation.
// +optional
Progress shared.DataMoveOperationProgress `json:"progress,omitempty"`
// AcceptedTimestamp records the time the pod volume restore is to be prepared.
// The server's time is used for AcceptedTimestamp
// +optional
// +nullable
AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"`
// Node is name of the node where the pod volume restore is processed.
// +optional
Node string `json:"node,omitempty"`
}
// TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed.
@@ -103,14 +121,14 @@ type PodVolumeRestoreStatus struct {
// +kubebuilder:object:generate=true
// +kubebuilder:object:root=true
// +kubebuilder:storageversion
// +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.pod.namespace",description="Namespace of the pod containing the volume to be restored"
// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.pod.name",description="Name of the pod containing the volume to be restored"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeRestore status such as New/InProgress"
// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeRestore was started"
// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes"
// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes"
// +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeRestore was created"
// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeRestore is processed"
// +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer"
// +kubebuilder:printcolumn:name="Volume",type="string",JSONPath=".spec.volume",description="Name of the volume to be restored"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Pod Volume Restore status such as New/InProgress"
// +kubebuilder:printcolumn:name="TotalBytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Pod Volume Restore status such as New/InProgress"
// +kubebuilder:printcolumn:name="BytesDone",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Pod Volume Restore status such as New/InProgress"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
type PodVolumeRestore struct {
metav1.TypeMeta `json:",inline"`
pkg/apis/velero/v1/zz_generated.deepcopy.go
+8
View File
@@ -1043,6 +1043,10 @@ func (in *PodVolumeBackupStatus) DeepCopyInto(out *PodVolumeBackupStatus) {
*out = (*in).DeepCopy()
}
out.Progress = in.Progress
if in.AcceptedTimestamp != nil {
in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp
*out = (*in).DeepCopy()
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupStatus.
@@ -1149,6 +1153,10 @@ func (in *PodVolumeRestoreStatus) DeepCopyInto(out *PodVolumeRestoreStatus) {
*out = (*in).DeepCopy()
}
out.Progress = in.Progress
if in.AcceptedTimestamp != nil {
in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp
*out = (*in).DeepCopy()
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestoreStatus.
pkg/backup/actions/csi/pvc_action.go
-1
View File
@@ -285,7 +285,6 @@ func (p *pvcBackupItemAction) Execute(
vs,
p.crClient,
p.log,
true,
backup.Spec.CSISnapshotTimeout.Duration,
)
if err != nil {
pkg/backup/actions/csi/volumesnapshot_action.go
+45 -66
View File
@@ -96,22 +96,6 @@ func (p *volumeSnapshotBackupItemAction) Execute(
},
}
// determine if we are backing up a VolumeSnapshot that was created by velero while
// performing backup of a CSI backed PVC.
// For VolumeSnapshots that were created during the backup of a CSI backed PVC,
// we will wait for the VolumeSnapshotContents to be available.
// For VolumeSnapshots created outside of velero, we expect the VolumeSnapshotContent
// to be available prior to backing up the VolumeSnapshot. In case of a failure,
// backup should be re-attempted after the CSI driver has reconciled the VolumeSnapshot.
// existence of the velerov1api.BackupNameLabel indicates that the VolumeSnapshot was
// created while backing up a CSI backed PVC.
// We want to await reconciliation of only those VolumeSnapshots created during the
// ongoing backup. For this we will wait only if the backup label exists on the
// VolumeSnapshot object and the backup name is the same as that of the value of the
// backup label.
backupOngoing := vs.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backup.Name)
p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s",
vs.Namespace, vs.Name)
@@ -119,7 +103,6 @@ func (p *volumeSnapshotBackupItemAction) Execute(
vs,
p.crClient,
p.log,
backupOngoing,
backup.Spec.CSISnapshotTimeout.Duration,
)
if err != nil {
@@ -171,42 +154,40 @@ func (p *volumeSnapshotBackupItemAction) Execute(
}
}
if backupOngoing {
p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel",
vsc.Name)
// If we created the VolumeSnapshotContent object during this ongoing backup,
// we would have created it with a DeletionPolicy of Retain.
// But, we want to retain these VolumeSnapshotContent ONLY for the lifetime
// of the backup. To that effect, during velero backup
// deletion, we will update the DeletionPolicy of the VolumeSnapshotContent
// and then delete the VolumeSnapshot object which will cascade delete the
// VolumeSnapshotContent and the associated snapshot in the storage
// provider (handled by the CSI driver and the CSI common controller).
// However, in the event that the VolumeSnapshot object is deleted outside
// of the backup deletion process, it is possible that the dynamically created
// VolumeSnapshotContent object will be left as an orphaned and non-discoverable
// resource in the cluster as well as in the storage provider. To avoid piling
// up of such orphaned resources, we will want to discover and delete the
// dynamically created VolumeSnapshotContent. We do that by adding
// the "velero.io/backup-name" label on the VolumeSnapshotContent.
// Further, we want to add this label only on VolumeSnapshotContents that
// were created during an ongoing velero backup.
originVSC := vsc.DeepCopy()
kubeutil.AddLabels(
&vsc.ObjectMeta,
map[string]string{
velerov1api.BackupNameLabel: label.GetValidName(backup.Name),
},
)
p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel",
vsc.Name)
// If we created the VolumeSnapshotContent object during this ongoing backup,
// we would have created it with a DeletionPolicy of Retain.
// But, we want to retain these VolumeSnapshotContent ONLY for the lifetime
// of the backup. To that effect, during velero backup
// deletion, we will update the DeletionPolicy of the VolumeSnapshotContent
// and then delete the VolumeSnapshot object which will cascade delete the
// VolumeSnapshotContent and the associated snapshot in the storage
// provider (handled by the CSI driver and the CSI common controller).
// However, in the event that the VolumeSnapshot object is deleted outside
// of the backup deletion process, it is possible that the dynamically created
// VolumeSnapshotContent object will be left as an orphaned and non-discoverable
// resource in the cluster as well as in the storage provider. To avoid piling
// up of such orphaned resources, we will want to discover and delete the
// dynamically created VolumeSnapshotContent. We do that by adding
// the "velero.io/backup-name" label on the VolumeSnapshotContent.
// Further, we want to add this label only on VolumeSnapshotContents that
// were created during an ongoing velero backup.
originVSC := vsc.DeepCopy()
kubeutil.AddLabels(
&vsc.ObjectMeta,
map[string]string{
velerov1api.BackupNameLabel: label.GetValidName(backup.Name),
},
)
if vscPatchError := p.crClient.Patch(
context.TODO(),
vsc,
crclient.MergeFrom(originVSC),
); vscPatchError != nil {
p.log.Warnf("Failed to patch VolumeSnapshotContent %s: %v",
vsc.Name, vscPatchError)
}
if vscPatchError := p.crClient.Patch(
context.TODO(),
vsc,
crclient.MergeFrom(originVSC),
); vscPatchError != nil {
p.log.Warnf("Failed to patch VolumeSnapshotContent %s: %v",
vsc.Name, vscPatchError)
}
}
@@ -244,20 +225,18 @@ func (p *volumeSnapshotBackupItemAction) Execute(
var itemToUpdate []velero.ResourceIdentifier
// Only return Async operation for VSC created for this backup.
if backupOngoing {
// The operationID is of the form <namespace>/<volumesnapshot-name>/<started-time>
operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339)
itemToUpdate = []velero.ResourceIdentifier{
{
GroupResource: kuberesource.VolumeSnapshots,
Namespace: vs.Namespace,
Name: vs.Name,
},
{
GroupResource: kuberesource.VolumeSnapshotContents,
Name: vsc.Name,
},
}
// The operationID is of the form <namespace>/<volumesnapshot-name>/<started-time>
operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339)
itemToUpdate = []velero.ResourceIdentifier{
{
GroupResource: kuberesource.VolumeSnapshots,
Namespace: vs.Namespace,
Name: vs.Name,
},
{
GroupResource: kuberesource.VolumeSnapshotContents,
Name: vsc.Name,
},
}
return &unstructured.Unstructured{Object: vsMap},
pkg/backup/actions/csi/volumesnapshot_action_test.go
-17
View File
@@ -49,23 +49,6 @@ func TestVSExecute(t *testing.T) {
expectedAdditionalItems []velero.ResourceIdentifier
expectedItemToUpdate []velero.ResourceIdentifier
}{
{
name: "VS not created by backup, has no status. Backup is finalizing",
backup: builder.ForBackup("velero", "backup").
Phase(velerov1api.BackupPhaseFinalizing).Result(),
vs: builder.ForVolumeSnapshot("velero", "vs").
VolumeSnapshotClass("class").Result(),
expectedErr: "",
},
{
name: "VS is not created by the backup, associated VSC not exists",
backup: builder.ForBackup("velero", "backup").
Phase(velerov1api.BackupPhaseInProgress).Result(),
vs: builder.ForVolumeSnapshot("velero", "vs").
VolumeSnapshotClass("class").Status().
BoundVolumeSnapshotContentName("vsc").Result(),
expectedErr: `error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io "vsc" not found`,
},
{
name: "Normal case",
backup: builder.ForBackup("velero", "backup").
pkg/builder/pod_builder.go
+5
View File
@@ -88,6 +88,11 @@ func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuil
return b
}
func (b *PodBuilder) InitContainerState(state corev1api.ContainerState) *PodBuilder {
b.object.Status.InitContainerStatuses = append(b.object.Status.InitContainerStatuses, corev1api.ContainerStatus{State: state})
return b
}
func (b *PodBuilder) Containers(containers ...*corev1api.Container) *PodBuilder {
for _, c := range containers {
b.object.Spec.Containers = append(b.object.Spec.Containers, *c)
pkg/builder/pod_volume_backup_builder.go
+36
View File
@@ -113,3 +113,39 @@ func (b *PodVolumeBackupBuilder) UploaderType(uploaderType string) *PodVolumeBac
b.object.Spec.UploaderType = uploaderType
return b
}
// Annotations sets the PodVolumeBackup's Annotations.
func (b *PodVolumeBackupBuilder) Annotations(annotations map[string]string) *PodVolumeBackupBuilder {
b.object.Annotations = annotations
return b
}
// Cancel sets the PodVolumeBackup's Cancel.
func (b *PodVolumeBackupBuilder) Cancel(cancel bool) *PodVolumeBackupBuilder {
b.object.Spec.Cancel = cancel
return b
}
// AcceptedTimestamp sets the PodVolumeBackup's AcceptedTimestamp.
func (b *PodVolumeBackupBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeBackupBuilder {
b.object.Status.AcceptedTimestamp = acceptedTimestamp
return b
}
// Finalizers sets the PodVolumeBackup's Finalizers.
func (b *PodVolumeBackupBuilder) Finalizers(finalizers []string) *PodVolumeBackupBuilder {
b.object.Finalizers = finalizers
return b
}
// Message sets the PodVolumeBackup's Message.
func (b *PodVolumeBackupBuilder) Message(msg string) *PodVolumeBackupBuilder {
b.object.Status.Message = msg
return b
}
// OwnerReference sets the PodVolumeBackup's OwnerReference.
func (b *PodVolumeBackupBuilder) OwnerReference(ref metav1.OwnerReference) *PodVolumeBackupBuilder {
b.object.OwnerReferences = append(b.object.OwnerReferences, ref)
return b
}
pkg/builder/pod_volume_restore_builder.go
+36
View File
@@ -97,3 +97,39 @@ func (b *PodVolumeRestoreBuilder) UploaderType(uploaderType string) *PodVolumeRe
b.object.Spec.UploaderType = uploaderType
return b
}
// OwnerReference sets the OwnerReference for this PodVolumeRestore.
func (b *PodVolumeRestoreBuilder) OwnerReference(ownerRef []metav1.OwnerReference) *PodVolumeRestoreBuilder {
b.object.OwnerReferences = ownerRef
return b
}
// Cancel sets the DataDownload's Cancel.
func (b *PodVolumeRestoreBuilder) Cancel(cancel bool) *PodVolumeRestoreBuilder {
b.object.Spec.Cancel = cancel
return b
}
// AcceptedTimestamp sets the PodVolumeRestore's AcceptedTimestamp.
func (b *PodVolumeRestoreBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeRestoreBuilder {
b.object.Status.AcceptedTimestamp = acceptedTimestamp
return b
}
// Finalizers sets the PodVolumeRestore's Finalizers.
func (b *PodVolumeRestoreBuilder) Finalizers(finalizers []string) *PodVolumeRestoreBuilder {
b.object.Finalizers = finalizers
return b
}
// Message sets the PodVolumeRestore's Message.
func (b *PodVolumeRestoreBuilder) Message(msg string) *PodVolumeRestoreBuilder {
b.object.Status.Message = msg
return b
}
// Message sets the PodVolumeRestore's Node.
func (b *PodVolumeRestoreBuilder) Node(node string) *PodVolumeRestoreBuilder {
b.object.Status.Node = node
return b
}
pkg/cmd/cli/datamover/backup.go
+4 -9
View File
@@ -39,6 +39,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
ctrl "sigs.k8s.io/controller-runtime"
@@ -78,7 +79,7 @@ func NewBackupCommand(f client.Factory) *cobra.Command {
f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()))
s, err := newdataMoverBackup(logger, f, config)
if err != nil {
exitWithMessage(logger, false, "Failed to create data mover backup, %v", err)
kube.ExitPodWithMessage(logger, false, "Failed to create data mover backup, %v", err)
}
s.run()
@@ -100,12 +101,6 @@ func NewBackupCommand(f client.Factory) *cobra.Command {
return command
}
const (
// defaultCredentialsDirectory is the path on disk where credential
// files will be written to
defaultCredentialsDirectory = "/tmp/credentials"
)
type dataMoverBackup struct {
logger logrus.FieldLogger
ctx context.Context
@@ -215,7 +210,7 @@ func newdataMoverBackup(logger logrus.FieldLogger, factory client.Factory, confi
return s, nil
}
var funcExitWithMessage = exitWithMessage
var funcExitWithMessage = kube.ExitPodWithMessage
var funcCreateDataPathService = (*dataMoverBackup).createDataPathService
func (s *dataMoverBackup) run() {
@@ -277,7 +272,7 @@ func (s *dataMoverBackup) createDataPathService() (dataPathService, error) {
credentialFileStore, err := funcNewCredentialFileStore(
s.client,
s.namespace,
defaultCredentialsDirectory,
credentials.DefaultStoreDirectory(),
filesystem.NewFileSystem(),
)
if err != nil {
pkg/cmd/cli/datamover/data_mover.go
-30
View File
@@ -15,10 +15,7 @@ package datamover
import (
"context"
"fmt"
"os"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vmware-tanzu/velero/pkg/client"
@@ -45,30 +42,3 @@ type dataPathService interface {
RunCancelableDataPath(context.Context) (string, error)
Shutdown()
}
var funcExit = os.Exit
var funcCreateFile = os.Create
func exitWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) {
exitCode := 0
if !succeed {
exitCode = 1
}
toWrite := fmt.Sprintf(message, a...)
podFile, err := funcCreateFile("/dev/termination-log")
if err != nil {
logger.WithError(err).Error("Failed to create termination log file")
exitCode = 1
} else {
if _, err := podFile.WriteString(toWrite); err != nil {
logger.WithError(err).Error("Failed to write error to termination log file")
exitCode = 1
}
podFile.Close()
}
funcExit(exitCode)
}
pkg/cmd/cli/datamover/data_mover_test.go
-131
View File
@@ -1,131 +0,0 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package datamover
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
)
type exitWithMessageMock struct {
createErr error
writeFail bool
filePath string
exitCode int
}
func (em *exitWithMessageMock) Exit(code int) {
em.exitCode = code
}
func (em *exitWithMessageMock) CreateFile(name string) (*os.File, error) {
if em.createErr != nil {
return nil, em.createErr
}
if em.writeFail {
return os.OpenFile(em.filePath, os.O_CREATE|os.O_RDONLY, 0500)
} else {
return os.Create(em.filePath)
}
}
func TestExitWithMessage(t *testing.T) {
tests := []struct {
name string
message string
succeed bool
args []any
createErr error
writeFail bool
expectedExitCode int
expectedMessage string
}{
{
name: "create pod file failed",
createErr: errors.New("fake-create-file-error"),
succeed: true,
expectedExitCode: 1,
},
{
name: "write pod file failed",
writeFail: true,
succeed: true,
expectedExitCode: 1,
},
{
name: "not succeed",
message: "fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v",
args: []any{
"arg-1-1",
10,
false,
},
expectedExitCode: 1,
expectedMessage: fmt.Sprintf("fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-1", 10, false),
},
{
name: "not succeed",
message: "fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v",
args: []any{
"arg-1-2",
20,
true,
},
succeed: true,
expectedMessage: fmt.Sprintf("fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-2", 20, true),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
podFile := filepath.Join(os.TempDir(), uuid.NewString())
em := exitWithMessageMock{
createErr: test.createErr,
writeFail: test.writeFail,
filePath: podFile,
}
funcExit = em.Exit
funcCreateFile = em.CreateFile
exitWithMessage(velerotest.NewLogger(), test.succeed, test.message, test.args...)
assert.Equal(t, test.expectedExitCode, em.exitCode)
if test.createErr == nil && !test.writeFail {
reader, err := os.Open(podFile)
require.NoError(t, err)
message, err := io.ReadAll(reader)
require.NoError(t, err)
reader.Close()
assert.Equal(t, test.expectedMessage, string(message))
}
})
}
}
pkg/cmd/cli/datamover/restore.go
+3 -2
View File
@@ -42,6 +42,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
ctlcache "sigs.k8s.io/controller-runtime/pkg/cache"
@@ -76,7 +77,7 @@ func NewRestoreCommand(f client.Factory) *cobra.Command {
f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()))
s, err := newdataMoverRestore(logger, f, config)
if err != nil {
exitWithMessage(logger, false, "Failed to create data mover restore, %v", err)
kube.ExitPodWithMessage(logger, false, "Failed to create data mover restore, %v", err)
}
s.run()
@@ -263,7 +264,7 @@ func (s *dataMoverRestore) createDataPathService() (dataPathService, error) {
credentialFileStore, err := funcNewCredentialFileStore(
s.client,
s.namespace,
defaultCredentialsDirectory,
credentials.DefaultStoreDirectory(),
filesystem.NewFileSystem(),
)
if err != nil {
pkg/cmd/cli/nodeagent/server.go
+15 -42
View File
@@ -48,7 +48,6 @@ import (
snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
"github.com/vmware-tanzu/velero/pkg/buildinfo"
@@ -60,7 +59,6 @@ import (
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/metrics"
"github.com/vmware-tanzu/velero/pkg/nodeagent"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -76,12 +74,6 @@ const (
// the port where prometheus metrics are exposed
defaultMetricsAddress = ":8085"
// defaultCredentialsDirectory is the path on disk where credential
// files will be written to
defaultCredentialsDirectory = "/tmp/credentials"
defaultHostPodsPath = "/host_pods"
defaultResourceTimeout = 10 * time.Minute
defaultDataMoverPrepareTimeout = 30 * time.Minute
defaultDataPathConcurrentNum = 1
@@ -288,34 +280,6 @@ func (s *nodeAgentServer) run() {
s.logger.Info("Starting controllers")
credentialFileStore, err := credentials.NewNamespacedFileStore(
s.mgr.GetClient(),
s.namespace,
defaultCredentialsDirectory,
filesystem.NewFileSystem(),
)
if err != nil {
s.logger.Fatalf("Failed to create credentials file store: %v", err)
}
credSecretStore, err := credentials.NewNamespacedSecretStore(s.mgr.GetClient(), s.namespace)
if err != nil {
s.logger.Fatalf("Failed to create secret file store: %v", err)
}
credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
repoEnsurer := repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.resourceTimeout)
pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer,
credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger)
if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup)
}
if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
}
var loadAffinity *kube.LoadAffinity
if s.dataPathConfigs != nil && len(s.dataPathConfigs.LoadAffinity) > 0 {
loadAffinity = s.dataPathConfigs.LoadAffinity[0]
@@ -338,6 +302,15 @@ func (s *nodeAgentServer) run() {
}
}
pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.metrics, s.logger)
if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup)
}
if err := controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.logger).SetupWithManager(s.mgr); err != nil {
s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
}
dataUploadReconciler := controller.NewDataUploadReconciler(
s.mgr.GetClient(),
s.mgr,
@@ -353,7 +326,7 @@ func (s *nodeAgentServer) run() {
s.logger,
s.metrics,
)
if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
if err := dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
s.logger.WithError(err).Fatal("Unable to create the data upload controller")
}
@@ -364,7 +337,7 @@ func (s *nodeAgentServer) run() {
}
dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, restorePVCConfig, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
if err := dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
s.logger.WithError(err).Fatal("Unable to create the data download controller")
}
@@ -416,10 +389,10 @@ func (s *nodeAgentServer) waitCacheForResume() error {
// validatePodVolumesHostPath validates that the pod volumes path contains a
// directory for each Pod running on this node
func (s *nodeAgentServer) validatePodVolumesHostPath(client kubernetes.Interface) error {
files, err := s.fileSystem.ReadDir(defaultHostPodsPath)
files, err := s.fileSystem.ReadDir(nodeagent.HostPodVolumeMountPath())
if err != nil {
if errors.Is(err, os.ErrNotExist) {
s.logger.Warnf("Pod volumes host path [%s] doesn't exist, fs-backup is disabled", defaultHostPodsPath)
s.logger.Warnf("Pod volumes host path [%s] doesn't exist, fs-backup is disabled", nodeagent.HostPodVolumeMountPath())
return nil
}
return errors.Wrap(err, "could not read pod volumes host path")
@@ -452,7 +425,7 @@ func (s *nodeAgentServer) validatePodVolumesHostPath(client kubernetes.Interface
valid = false
s.logger.WithFields(logrus.Fields{
"pod": fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName()),
"path": defaultHostPodsPath + "/" + dirName,
"path": nodeagent.HostPodVolumeMountPath() + "/" + dirName,
}).Debug("could not find volumes for pod in host path")
}
}
@@ -531,7 +504,7 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) {
continue
}
if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i],
if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i], errors.New("cannot survive from node-agent restart"),
fmt.Sprintf("get a podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, velerov1api.PodVolumeRestorePhaseFailed),
time.Now(), s.logger); err != nil {
s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName())
pkg/cmd/cli/nodeagent/server_test.go
+1 -1
View File
@@ -99,7 +99,7 @@ func Test_validatePodVolumesHostPath(t *testing.T) {
for _, dir := range tt.dirs {
if tt.createDir {
err := fs.MkdirAll(filepath.Join(defaultHostPodsPath, dir), os.ModePerm)
err := fs.MkdirAll(filepath.Join(nodeagent.HostPodVolumeMountPath(), dir), os.ModePerm)
if err != nil {
t.Error(err)
}
pkg/cmd/cli/podvolume/backup.go
+291
View File
@@ -0,0 +1,291 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/bombsimon/logrusr/v3"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
corev1api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/pkg/buildinfo"
"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/podvolume"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
ctrl "sigs.k8s.io/controller-runtime"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
ctlcache "sigs.k8s.io/controller-runtime/pkg/cache"
ctlclient "sigs.k8s.io/controller-runtime/pkg/client"
)
type podVolumeBackupConfig struct {
volumePath string
pvbName string
resourceTimeout time.Duration
}
func NewBackupCommand(f client.Factory) *cobra.Command {
config := podVolumeBackupConfig{}
logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel)
formatFlag := logging.NewFormatFlag()
command := &cobra.Command{
Use: "backup",
Short: "Run the velero pod volume backup",
Long: "Run the velero pod volume backup",
Hidden: true,
Run: func(c *cobra.Command, args []string) {
logLevel := logLevelFlag.Parse()
logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String()))
logger := logging.DefaultLogger(logLevel, formatFlag.Parse())
logger.Infof("Starting Velero pod volume backup %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA())
f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()))
s, err := newPodVolumeBackup(logger, f, config)
if err != nil {
kube.ExitPodWithMessage(logger, false, "Failed to create pod volume backup, %v", err)
}
s.run()
},
}
command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", ")))
command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", ")))
command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be backed up")
command.Flags().StringVar(&config.pvbName, "pod-volume-backup", config.pvbName, "The PVB name")
command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.")
_ = command.MarkFlagRequired("volume-path")
_ = command.MarkFlagRequired("pod-volume-backup")
_ = command.MarkFlagRequired("resource-timeout")
return command
}
type podVolumeBackup struct {
logger logrus.FieldLogger
ctx context.Context
cancelFunc context.CancelFunc
client ctlclient.Client
cache ctlcache.Cache
namespace string
nodeName string
config podVolumeBackupConfig
kubeClient kubernetes.Interface
dataPathMgr *datapath.Manager
}
func newPodVolumeBackup(logger logrus.FieldLogger, factory client.Factory, config podVolumeBackupConfig) (*podVolumeBackup, error) {
ctx, cancelFunc := context.WithCancel(context.Background())
clientConfig, err := factory.ClientConfig()
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client config")
}
ctrl.SetLogger(logrusr.New(logger))
klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go
scheme := runtime.NewScheme()
if err := velerov1api.AddToScheme(scheme); err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to add velero v1 scheme")
}
if err := corev1api.AddToScheme(scheme); err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to add core v1 scheme")
}
nodeName := os.Getenv("NODE_NAME")
// use a field selector to filter to only pods scheduled on this node.
cacheOption := ctlcache.Options{
Scheme: scheme,
ByObject: map[ctlclient.Object]ctlcache.ByObject{
&corev1api.Pod{}: {
Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(),
},
&velerov1api.PodVolumeBackup{}: {
Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
},
},
}
cli, err := ctlclient.New(clientConfig, ctlclient.Options{
Scheme: scheme,
})
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client")
}
var cache ctlcache.Cache
retry := 10
for {
cache, err = ctlcache.New(clientConfig, cacheOption)
if err == nil {
break
}
retry--
if retry == 0 {
break
}
logger.WithError(err).Warn("Failed to create client cache, need retry")
time.Sleep(time.Second)
}
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client cache")
}
s := &podVolumeBackup{
logger: logger,
ctx: ctx,
cancelFunc: cancelFunc,
client: cli,
cache: cache,
config: config,
namespace: factory.Namespace(),
nodeName: nodeName,
}
s.kubeClient, err = factory.KubeClient()
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create kube client")
}
s.dataPathMgr = datapath.NewManager(1)
return s, nil
}
var funcExitWithMessage = kube.ExitPodWithMessage
var funcCreateDataPathService = (*podVolumeBackup).createDataPathService
func (s *podVolumeBackup) run() {
signals.CancelOnShutdown(s.cancelFunc, s.logger)
go func() {
if err := s.cache.Start(s.ctx); err != nil {
s.logger.WithError(err).Warn("error starting cache")
}
}()
s.runDataPath()
}
func (s *podVolumeBackup) runDataPath() {
s.logger.Infof("Starting micro service in node %s for PVB %s", s.nodeName, s.config.pvbName)
dpService, err := funcCreateDataPathService(s)
if err != nil {
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to create data path service for PVB %s: %v", s.config.pvbName, err)
return
}
s.logger.Infof("Starting data path service %s", s.config.pvbName)
err = dpService.Init()
if err != nil {
dpService.Shutdown()
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to init data path service for PVB %s: %v", s.config.pvbName, err)
return
}
s.logger.Infof("Running data path service %s", s.config.pvbName)
result, err := dpService.RunCancelableDataPath(s.ctx)
if err != nil {
dpService.Shutdown()
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to run data path service for PVB %s: %v", s.config.pvbName, err)
return
}
s.logger.WithField("PVB", s.config.pvbName).Info("Data path service completed")
dpService.Shutdown()
s.logger.WithField("PVB", s.config.pvbName).Info("Data path service is shut down")
s.cancelFunc()
funcExitWithMessage(s.logger, true, result)
}
var funcNewCredentialFileStore = credentials.NewNamespacedFileStore
var funcNewCredentialSecretStore = credentials.NewNamespacedSecretStore
func (s *podVolumeBackup) createDataPathService() (dataPathService, error) {
credentialFileStore, err := funcNewCredentialFileStore(
s.client,
s.namespace,
credentials.DefaultStoreDirectory(),
filesystem.NewFileSystem(),
)
if err != nil {
return nil, errors.Wrapf(err, "error to create credential file store")
}
credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace)
if err != nil {
return nil, errors.Wrapf(err, "error to create credential secret store")
}
credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
pvbInformer, err := s.cache.GetInformer(s.ctx, &velerov1api.PodVolumeBackup{})
if err != nil {
return nil, errors.Wrap(err, "error to get controller-runtime informer from manager")
}
repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout)
return podvolume.NewBackupMicroService(s.ctx, s.client, s.kubeClient, s.config.pvbName, s.namespace, s.nodeName, datapath.AccessPoint{
ByPath: s.config.volumePath,
VolMode: uploader.PersistentVolumeFilesystem,
}, s.dataPathMgr, repoEnsurer, credGetter, pvbInformer, s.logger), nil
}
pkg/cmd/cli/podvolume/backup_test.go
+216
View File
@@ -0,0 +1,216 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"errors"
"fmt"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
ctlclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/internal/credentials"
cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
)
func fakeCreateDataPathServiceWithErr(_ *podVolumeBackup) (dataPathService, error) {
return nil, errors.New("fake-create-data-path-error")
}
var frHelper *fakeRunHelper
func fakeCreateDataPathService(_ *podVolumeBackup) (dataPathService, error) {
return frHelper, nil
}
type fakeRunHelper struct {
initErr error
runCancelableDataPathErr error
runCancelableDataPathResult string
exitMessage string
succeed bool
}
func (fr *fakeRunHelper) Init() error {
return fr.initErr
}
func (fr *fakeRunHelper) RunCancelableDataPath(_ context.Context) (string, error) {
if fr.runCancelableDataPathErr != nil {
return "", fr.runCancelableDataPathErr
} else {
return fr.runCancelableDataPathResult, nil
}
}
func (fr *fakeRunHelper) Shutdown() {
}
func (fr *fakeRunHelper) ExitWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) {
fr.succeed = succeed
fr.exitMessage = fmt.Sprintf(message, a...)
}
func TestRunDataPath(t *testing.T) {
tests := []struct {
name string
pvbName string
createDataPathFail bool
initDataPathErr error
runCancelableDataPathErr error
runCancelableDataPathResult string
expectedMessage string
expectedSucceed bool
}{
{
name: "create data path failed",
pvbName: "fake-name",
createDataPathFail: true,
expectedMessage: "Failed to create data path service for PVB fake-name: fake-create-data-path-error",
},
{
name: "init data path failed",
pvbName: "fake-name",
initDataPathErr: errors.New("fake-init-data-path-error"),
expectedMessage: "Failed to init data path service for PVB fake-name: fake-init-data-path-error",
},
{
name: "run data path failed",
pvbName: "fake-name",
runCancelableDataPathErr: errors.New("fake-run-data-path-error"),
expectedMessage: "Failed to run data path service for PVB fake-name: fake-run-data-path-error",
},
{
name: "succeed",
pvbName: "fake-name",
runCancelableDataPathResult: "fake-run-data-path-result",
expectedMessage: "fake-run-data-path-result",
expectedSucceed: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
frHelper = &fakeRunHelper{
initErr: test.initDataPathErr,
runCancelableDataPathErr: test.runCancelableDataPathErr,
runCancelableDataPathResult: test.runCancelableDataPathResult,
}
if test.createDataPathFail {
funcCreateDataPathService = fakeCreateDataPathServiceWithErr
} else {
funcCreateDataPathService = fakeCreateDataPathService
}
funcExitWithMessage = frHelper.ExitWithMessage
s := &podVolumeBackup{
logger: velerotest.NewLogger(),
cancelFunc: func() {},
config: podVolumeBackupConfig{
pvbName: test.pvbName,
},
}
s.runDataPath()
assert.Equal(t, test.expectedMessage, frHelper.exitMessage)
assert.Equal(t, test.expectedSucceed, frHelper.succeed)
})
}
}
type fakeCreateDataPathServiceHelper struct {
fileStoreErr error
secretStoreErr error
}
func (fc *fakeCreateDataPathServiceHelper) NewNamespacedFileStore(_ ctlclient.Client, _ string, _ string, _ filesystem.Interface) (credentials.FileStore, error) {
return nil, fc.fileStoreErr
}
func (fc *fakeCreateDataPathServiceHelper) NewNamespacedSecretStore(_ ctlclient.Client, _ string) (credentials.SecretStore, error) {
return nil, fc.secretStoreErr
}
func TestCreateDataPathService(t *testing.T) {
tests := []struct {
name string
fileStoreErr error
secretStoreErr error
mockGetInformer bool
getInformerErr error
expectedError string
}{
{
name: "create credential file store error",
fileStoreErr: errors.New("fake-file-store-error"),
expectedError: "error to create credential file store: fake-file-store-error",
},
{
name: "create credential secret store",
secretStoreErr: errors.New("fake-secret-store-error"),
expectedError: "error to create credential secret store: fake-secret-store-error",
},
{
name: "get informer error",
mockGetInformer: true,
getInformerErr: errors.New("fake-get-informer-error"),
expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error",
},
{
name: "succeed",
mockGetInformer: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fcHelper := &fakeCreateDataPathServiceHelper{
fileStoreErr: test.fileStoreErr,
secretStoreErr: test.secretStoreErr,
}
funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore
funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore
cache := cacheMock.NewCache(t)
if test.mockGetInformer {
cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr)
}
funcExitWithMessage = frHelper.ExitWithMessage
s := &podVolumeBackup{
cache: cache,
}
_, err := s.createDataPathService()
if test.expectedError != "" {
assert.EqualError(t, err, test.expectedError)
} else {
assert.NoError(t, err)
}
})
}
}
pkg/cmd/cli/podvolume/podvolume.go
+44
View File
@@ -0,0 +1,44 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"github.com/spf13/cobra"
"github.com/vmware-tanzu/velero/pkg/client"
)
func NewCommand(f client.Factory) *cobra.Command {
command := &cobra.Command{
Use: "pod-volume",
Short: "Run the velero pod volume backup/restore",
Long: "Run the velero pod volume backup/restore",
Hidden: true,
}
command.AddCommand(
NewBackupCommand(f),
NewRestoreCommand(f),
)
return command
}
type dataPathService interface {
Init() error
RunCancelableDataPath(context.Context) (string, error)
Shutdown()
}
pkg/cmd/cli/podvolume/restore.go
+298
View File
@@ -0,0 +1,298 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/bombsimon/logrusr/v3"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
corev1api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/buildinfo"
"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/podvolume"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
ctlcache "sigs.k8s.io/controller-runtime/pkg/cache"
ctlclient "sigs.k8s.io/controller-runtime/pkg/client"
)
type podVolumeRestoreConfig struct {
volumePath string
pvrName string
resourceTimeout time.Duration
}
func NewRestoreCommand(f client.Factory) *cobra.Command {
logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel)
formatFlag := logging.NewFormatFlag()
config := podVolumeRestoreConfig{}
command := &cobra.Command{
Use: "restore",
Short: "Run the velero pod volume restore",
Long: "Run the velero pod volume restore",
Hidden: true,
Run: func(c *cobra.Command, args []string) {
logLevel := logLevelFlag.Parse()
logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String()))
logger := logging.DefaultLogger(logLevel, formatFlag.Parse())
logger.Infof("Starting Velero pod volume restore %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA())
f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()))
s, err := newPodVolumeRestore(logger, f, config)
if err != nil {
kube.ExitPodWithMessage(logger, false, "Failed to create pod volume restore, %v", err)
}
s.run()
},
}
command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", ")))
command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", ")))
command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be restored")
command.Flags().StringVar(&config.pvrName, "pod-volume-restore", config.pvrName, "The PVR name")
command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.")
_ = command.MarkFlagRequired("volume-path")
_ = command.MarkFlagRequired("pod-volume-restore")
_ = command.MarkFlagRequired("resource-timeout")
command.PreRunE = func(cmd *cobra.Command, args []string) error {
if config.resourceTimeout <= 0 {
return errors.New("resource-timeout must be greater than 0")
}
if config.volumePath == "" {
return errors.New("volume-path cannot be empty")
}
if config.pvrName == "" {
return errors.New("pod-volume-restore name cannot be empty")
}
return nil
}
return command
}
type podVolumeRestore struct {
logger logrus.FieldLogger
ctx context.Context
cancelFunc context.CancelFunc
client ctlclient.Client
cache ctlcache.Cache
namespace string
nodeName string
config podVolumeRestoreConfig
kubeClient kubernetes.Interface
dataPathMgr *datapath.Manager
}
func newPodVolumeRestore(logger logrus.FieldLogger, factory client.Factory, config podVolumeRestoreConfig) (*podVolumeRestore, error) {
ctx, cancelFunc := context.WithCancel(context.Background())
clientConfig, err := factory.ClientConfig()
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client config")
}
ctrl.SetLogger(logrusr.New(logger))
klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go
scheme := runtime.NewScheme()
if err := velerov1api.AddToScheme(scheme); err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to add velero v1 scheme")
}
if err := corev1api.AddToScheme(scheme); err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to add core v1 scheme")
}
nodeName := os.Getenv("NODE_NAME")
// use a field selector to filter to only pods scheduled on this node.
cacheOption := ctlcache.Options{
Scheme: scheme,
ByObject: map[ctlclient.Object]ctlcache.ByObject{
&corev1api.Pod{}: {
Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(),
},
&velerov1api.PodVolumeRestore{}: {
Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
},
},
}
cli, err := ctlclient.New(clientConfig, ctlclient.Options{
Scheme: scheme,
})
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client")
}
var cache ctlcache.Cache
retry := 10
for {
cache, err = ctlcache.New(clientConfig, cacheOption)
if err == nil {
break
}
retry--
if retry == 0 {
break
}
logger.WithError(err).Warn("Failed to create client cache, need retry")
time.Sleep(time.Second)
}
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create client cache")
}
s := &podVolumeRestore{
logger: logger,
ctx: ctx,
cancelFunc: cancelFunc,
client: cli,
cache: cache,
config: config,
namespace: factory.Namespace(),
nodeName: nodeName,
}
s.kubeClient, err = factory.KubeClient()
if err != nil {
cancelFunc()
return nil, errors.Wrap(err, "error to create kube client")
}
s.dataPathMgr = datapath.NewManager(1)
return s, nil
}
var funcCreateDataPathRestore = (*podVolumeRestore).createDataPathService
func (s *podVolumeRestore) run() {
signals.CancelOnShutdown(s.cancelFunc, s.logger)
go func() {
if err := s.cache.Start(s.ctx); err != nil {
s.logger.WithError(err).Warn("error starting cache")
}
}()
s.runDataPath()
}
func (s *podVolumeRestore) runDataPath() {
s.logger.Infof("Starting micro service in node %s for PVR %s", s.nodeName, s.config.pvrName)
dpService, err := funcCreateDataPathRestore(s)
if err != nil {
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to create data path service for PVR %s: %v", s.config.pvrName, err)
return
}
s.logger.Infof("Starting data path service %s", s.config.pvrName)
err = dpService.Init()
if err != nil {
dpService.Shutdown()
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to init data path service for PVR %s: %v", s.config.pvrName, err)
return
}
s.logger.Infof("Running data path service %s", s.config.pvrName)
result, err := dpService.RunCancelableDataPath(s.ctx)
if err != nil {
dpService.Shutdown()
s.cancelFunc()
funcExitWithMessage(s.logger, false, "Failed to run data path service for PVR %s: %v", s.config.pvrName, err)
return
}
s.logger.WithField("PVR", s.config.pvrName).Info("Data path service completed")
dpService.Shutdown()
s.logger.WithField("PVR", s.config.pvrName).Info("Data path service is shut down")
s.cancelFunc()
funcExitWithMessage(s.logger, true, result)
}
func (s *podVolumeRestore) createDataPathService() (dataPathService, error) {
credentialFileStore, err := funcNewCredentialFileStore(
s.client,
s.namespace,
credentials.DefaultStoreDirectory(),
filesystem.NewFileSystem(),
)
if err != nil {
return nil, errors.Wrapf(err, "error to create credential file store")
}
credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace)
if err != nil {
return nil, errors.Wrapf(err, "error to create credential secret store")
}
credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
pvrInformer, err := s.cache.GetInformer(s.ctx, &velerov1api.PodVolumeRestore{})
if err != nil {
return nil, errors.Wrap(err, "error to get controller-runtime informer from manager")
}
repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout)
return podvolume.NewRestoreMicroService(s.ctx, s.client, s.kubeClient, s.config.pvrName, s.namespace, s.nodeName, datapath.AccessPoint{
ByPath: s.config.volumePath,
VolMode: uploader.PersistentVolumeFilesystem,
}, s.dataPathMgr, repoEnsurer, credGetter, pvrInformer, s.logger), nil
}
pkg/cmd/cli/podvolume/restore_test.go
+166
View File
@@ -0,0 +1,166 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
)
func fakeCreateRestoreDataPathServiceWithErr(_ *podVolumeRestore) (dataPathService, error) {
return nil, errors.New("fake-create-data-path-error")
}
func fakeCreateRestoreDataPathService(_ *podVolumeRestore) (dataPathService, error) {
return frHelper, nil
}
func TestRunRestoreDataPath(t *testing.T) {
tests := []struct {
name string
pvrName string
createDataPathFail bool
initDataPathErr error
runCancelableDataPathErr error
runCancelableDataPathResult string
expectedMessage string
expectedSucceed bool
}{
{
name: "create data path failed",
pvrName: "fake-name",
createDataPathFail: true,
expectedMessage: "Failed to create data path service for PVR fake-name: fake-create-data-path-error",
},
{
name: "init data path failed",
pvrName: "fake-name",
initDataPathErr: errors.New("fake-init-data-path-error"),
expectedMessage: "Failed to init data path service for PVR fake-name: fake-init-data-path-error",
},
{
name: "run data path failed",
pvrName: "fake-name",
runCancelableDataPathErr: errors.New("fake-run-data-path-error"),
expectedMessage: "Failed to run data path service for PVR fake-name: fake-run-data-path-error",
},
{
name: "succeed",
pvrName: "fake-name",
runCancelableDataPathResult: "fake-run-data-path-result",
expectedMessage: "fake-run-data-path-result",
expectedSucceed: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
frHelper = &fakeRunHelper{
initErr: test.initDataPathErr,
runCancelableDataPathErr: test.runCancelableDataPathErr,
runCancelableDataPathResult: test.runCancelableDataPathResult,
}
if test.createDataPathFail {
funcCreateDataPathRestore = fakeCreateRestoreDataPathServiceWithErr
} else {
funcCreateDataPathRestore = fakeCreateRestoreDataPathService
}
funcExitWithMessage = frHelper.ExitWithMessage
s := &podVolumeRestore{
logger: velerotest.NewLogger(),
cancelFunc: func() {},
config: podVolumeRestoreConfig{
pvrName: test.pvrName,
},
}
s.runDataPath()
assert.Equal(t, test.expectedMessage, frHelper.exitMessage)
assert.Equal(t, test.expectedSucceed, frHelper.succeed)
})
}
}
func TestCreateRestoreDataPathService(t *testing.T) {
tests := []struct {
name string
fileStoreErr error
secretStoreErr error
mockGetInformer bool
getInformerErr error
expectedError string
}{
{
name: "create credential file store error",
fileStoreErr: errors.New("fake-file-store-error"),
expectedError: "error to create credential file store: fake-file-store-error",
},
{
name: "create credential secret store",
secretStoreErr: errors.New("fake-secret-store-error"),
expectedError: "error to create credential secret store: fake-secret-store-error",
},
{
name: "get informer error",
mockGetInformer: true,
getInformerErr: errors.New("fake-get-informer-error"),
expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error",
},
{
name: "succeed",
mockGetInformer: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fcHelper := &fakeCreateDataPathServiceHelper{
fileStoreErr: test.fileStoreErr,
secretStoreErr: test.secretStoreErr,
}
funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore
funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore
cache := cacheMock.NewCache(t)
if test.mockGetInformer {
cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr)
}
funcExitWithMessage = frHelper.ExitWithMessage
s := &podVolumeRestore{
cache: cache,
}
_, err := s.createDataPathService()
if test.expectedError != "" {
assert.EqualError(t, err, test.expectedError)
} else {
assert.NoError(t, err)
}
})
}
}
pkg/cmd/cli/repomantenance/maintenance.go
+1 -1
View File
@@ -121,7 +121,7 @@ func initRepoManager(namespace string, cli client.Client, kubeClient kubernetes.
credentialFileStore, err := credentials.NewNamespacedFileStore(
cli,
namespace,
"/tmp/credentials",
credentials.DefaultStoreDirectory(),
filesystem.NewFileSystem(),
)
if err != nil {
pkg/cmd/server/config/config.go
+6 -10
View File
@@ -5,9 +5,12 @@ import (
"strings"
"time"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
"github.com/vmware-tanzu/velero/pkg/constant"
podvolumeconfigs "github.com/vmware-tanzu/velero/pkg/podvolume/configs"
@@ -36,9 +39,6 @@ const (
// the default TTL for a backup
defaultBackupTTL = 30 * 24 * time.Hour
// defaultVGSLabelKey is the default label key used to group PVCs under a VolumeGroupSnapshot
defaultVGSLabelKey = "velero.io/volume-group-snapshot"
defaultCSISnapshotTimeout = 10 * time.Minute
defaultItemOperationTimeout = 4 * time.Hour
@@ -47,10 +47,6 @@ const (
defaultMaxConcurrentK8SConnections = 30
defaultDisableInformerCache = false
// defaultCredentialsDirectory is the path on disk where credential
// files will be written to
defaultCredentialsDirectory = "/tmp/credentials"
DefaultKeepLatestMaintenanceJobs = 3
DefaultMaintenanceJobCPURequest = "0"
DefaultMaintenanceJobCPULimit = "0"
@@ -196,7 +192,7 @@ func GetDefaultConfig() *Config {
DefaultVolumeSnapshotLocations: flag.NewMap().WithKeyValueDelimiter(':'),
BackupSyncPeriod: defaultBackupSyncPeriod,
DefaultBackupTTL: defaultBackupTTL,
DefaultVGSLabelKey: defaultVGSLabelKey,
DefaultVGSLabelKey: velerov1api.DefaultVGSLabelKey,
DefaultCSISnapshotTimeout: defaultCSISnapshotTimeout,
DefaultItemOperationTimeout: defaultItemOperationTimeout,
ResourceTimeout: resourceTimeout,
@@ -216,7 +212,7 @@ func GetDefaultConfig() *Config {
DefaultSnapshotMoveData: false,
DisableInformerCache: defaultDisableInformerCache,
ScheduleSkipImmediately: false,
CredentialsDirectory: defaultCredentialsDirectory,
CredentialsDirectory: credentials.DefaultStoreDirectory(),
PodResources: kube.PodResources{
CPURequest: DefaultMaintenanceJobCPULimit,
CPULimit: DefaultMaintenanceJobCPURequest,
@@ -248,7 +244,7 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) {
flags.StringVar(&c.ProfilerAddress, "profiler-address", c.ProfilerAddress, "The address to expose the pprof profiler.")
flags.DurationVar(&c.ResourceTerminatingTimeout, "terminating-resource-timeout", c.ResourceTerminatingTimeout, "How long to wait on persistent volumes and namespaces to terminate during a restore before timing out.")
flags.DurationVar(&c.DefaultBackupTTL, "default-backup-ttl", c.DefaultBackupTTL, "How long to wait by default before backups can be garbage collected.")
flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot. Default value is 'velero.io/volume-group-snapshot'")
flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot. Default value is 'velero.io/volume-group'")
flags.DurationVar(&c.RepoMaintenanceFrequency, "default-repo-maintain-frequency", c.RepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default.")
flags.DurationVar(&c.GarbageCollectionFrequency, "garbage-collection-frequency", c.GarbageCollectionFrequency, "How often garbage collection is run for expired backups.")
flags.DurationVar(&c.ItemOperationSyncFrequency, "item-operation-sync-frequency", c.ItemOperationSyncFrequency, "How often to check status on backup/restore operations after backup/restore processing. Default is 10 seconds")
pkg/cmd/velero/velero.go
+2
View File
@@ -26,6 +26,7 @@ import (
"k8s.io/klog/v2"
"github.com/vmware-tanzu/velero/pkg/cmd/cli/debug"
"github.com/vmware-tanzu/velero/pkg/cmd/cli/podvolume"
"github.com/vmware-tanzu/velero/pkg/cmd/cli/repomantenance"
"github.com/vmware-tanzu/velero/pkg/client"
@@ -126,6 +127,7 @@ operations can also be performed as 'velero backup get' and 'velero schedule cre
debug.NewCommand(f),
repomantenance.NewCommand(f),
datamover.NewCommand(f),
podvolume.NewCommand(f),
)
// init and add the klog flags
pkg/controller/backup_controller.go
+90 -12
View File
@@ -21,6 +21,7 @@ import (
"context"
"fmt"
"os"
"slices"
"time"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
@@ -31,6 +32,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/utils/clock"
ctrl "sigs.k8s.io/controller-runtime"
@@ -56,12 +58,27 @@ import (
kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
"github.com/vmware-tanzu/velero/pkg/util/results"
veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero"
)
const (
backupResyncPeriod = time.Minute
)
var autoExcludeNamespaceScopedResources = []string{
// CSI VolumeSnapshot and VolumeSnapshotContent are intermediate resources.
// Velero only handle the VS and VSC created during backup,
// not during resource collecting.
"volumesnapshots.snapshot.storage.k8s.io",
}
var autoExcludeClusterScopedResources = []string{
// CSI VolumeSnapshot and VolumeSnapshotContent are intermediate resources.
// Velero only handle the VS and VSC created during backup,
// not during resource collecting.
"volumesnapshotcontents.snapshot.storage.k8s.io",
}
type backupReconciler struct {
ctx context.Context
logger logrus.FieldLogger
@@ -424,6 +441,13 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg
request.Status.ValidationErrors = append(request.Status.ValidationErrors,
fmt.Sprintf("backup can't be created because backup storage location %s is currently in read-only mode", request.StorageLocation.Name))
}
if !veleroutil.BSLIsAvailable(*request.StorageLocation) {
request.Status.ValidationErrors = append(
request.Status.ValidationErrors,
fmt.Sprintf("backup can't be created because BackupStorageLocation %s is in Unavailable status.", request.StorageLocation.Name),
)
}
}
// add the storage location as a label for easy filtering later.
@@ -473,19 +497,51 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg
request.Status.ValidationErrors = append(request.Status.ValidationErrors, validatedError)
}
// validate the included/excluded resources
for _, err := range collections.ValidateIncludesExcludes(request.Spec.IncludedResources, request.Spec.ExcludedResources) {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err))
}
if collections.UseOldResourceFilters(request.Spec) {
// validate the included/excluded resources
ieErr := collections.ValidateIncludesExcludes(request.Spec.IncludedResources, request.Spec.ExcludedResources)
if len(ieErr) > 0 {
for _, err := range ieErr {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err))
}
} else {
request.Spec.IncludedResources, request.Spec.ExcludedResources =
modifyResourceIncludeExclude(
request.Spec.IncludedResources,
request.Spec.ExcludedResources,
append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
)
}
} else {
// validate the cluster-scoped included/excluded resources
clusterErr := collections.ValidateScopedIncludesExcludes(request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources)
if len(clusterErr) > 0 {
for _, err := range clusterErr {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid cluster-scoped included/excluded resource lists: %s", err))
}
} else {
request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources =
modifyResourceIncludeExclude(
request.Spec.IncludedClusterScopedResources,
request.Spec.ExcludedClusterScopedResources,
autoExcludeClusterScopedResources,
)
}
// validate the cluster-scoped included/excluded resources
for _, err := range collections.ValidateScopedIncludesExcludes(request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources) {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid cluster-scoped included/excluded resource lists: %s", err))
}
// validate the namespace-scoped included/excluded resources
for _, err := range collections.ValidateScopedIncludesExcludes(request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources) {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid namespace-scoped included/excluded resource lists: %s", err))
// validate the namespace-scoped included/excluded resources
namespaceErr := collections.ValidateScopedIncludesExcludes(request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources)
if len(namespaceErr) > 0 {
for _, err := range namespaceErr {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid namespace-scoped included/excluded resource lists: %s", err))
}
} else {
request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources =
modifyResourceIncludeExclude(
request.Spec.IncludedNamespaceScopedResources,
request.Spec.ExcludedNamespaceScopedResources,
autoExcludeNamespaceScopedResources,
)
}
}
// validate the included/excluded namespaces
@@ -924,3 +980,25 @@ func oldAndNewFilterParametersUsedTogether(backupSpec velerov1api.BackupSpec) bo
return haveOldResourceFilterParameters && haveNewResourceFilterParameters
}
func modifyResourceIncludeExclude(include, exclude, addedExclude []string) (modifiedInclude, modifiedExclude []string) {
modifiedInclude = include
modifiedExclude = exclude
excludeStrSet := sets.NewString(exclude...)
for _, ex := range addedExclude {
if !excludeStrSet.Has(ex) {
modifiedExclude = append(modifiedExclude, ex)
}
}
for _, exElem := range modifiedExclude {
for inIndex, inElem := range modifiedInclude {
if inElem == exElem {
modifiedInclude = slices.Delete(modifiedInclude, inIndex, inIndex+1)
}
}
}
return modifiedInclude, modifiedExclude
}
pkg/controller/backup_controller_test.go
+142 -8
View File
@@ -155,7 +155,7 @@ func TestProcessBackupNonProcessedItems(t *testing.T) {
}
func TestProcessBackupValidationFailures(t *testing.T) {
defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Result()
defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
tests := []struct {
name string
@@ -183,7 +183,7 @@ func TestProcessBackupValidationFailures(t *testing.T) {
{
name: "backup for read-only backup location fails validation",
backup: defaultBackup().StorageLocation("read-only").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(),
expectedErrs: []string{"backup can't be created because backup storage location read-only is currently in read-only mode"},
},
{
@@ -203,6 +203,12 @@ func TestProcessBackupValidationFailures(t *testing.T) {
backupLocation: defaultBackupLocation,
expectedErrs: []string{"include-resources, exclude-resources and include-cluster-resources are old filter parameters.\ninclude-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\nThey cannot be used together"},
},
{
name: "BSL in unavailable state",
backup: defaultBackup().StorageLocation("unavailable").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(),
expectedErrs: []string{"backup can't be created because BackupStorageLocation unavailable is in Unavailable status."},
},
}
for _, test := range tests {
@@ -484,8 +490,6 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) {
require.NoError(t, err)
now = now.Local()
defaultVGSLabelKey := "velero.io/volume-group-snapshot"
tests := []struct {
name string
backup *velerov1api.Backup
@@ -509,8 +513,8 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) {
{
name: "backup with no spec or server flag, uses default",
backup: builder.ForBackup("velero", "backup-3").Result(),
serverFlagKey: defaultVGSLabelKey,
expectedLabelKey: defaultVGSLabelKey,
serverFlagKey: velerov1api.DefaultVGSLabelKey,
expectedLabelKey: velerov1api.DefaultVGSLabelKey,
},
}
@@ -655,7 +659,7 @@ func TestDefaultVolumesToResticDeprecation(t *testing.T) {
}
func TestProcessBackupCompletions(t *testing.T) {
defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Default(true).Bucket("store-1").Result()
defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Default(true).Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
now, err := time.Parse(time.RFC1123Z, time.RFC1123Z)
require.NoError(t, err)
@@ -702,6 +706,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -715,7 +720,7 @@ func TestProcessBackupCompletions(t *testing.T) {
{
name: "backup with a specific backup location keeps it",
backup: defaultBackup().StorageLocation("alt-loc").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "alt-loc").Bucket("store-1").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "alt-loc").Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(),
defaultVolumesToFsBackup: false,
expectedResult: &velerov1api.Backup{
TypeMeta: metav1.TypeMeta{
@@ -739,6 +744,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: "alt-loc",
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -755,6 +761,7 @@ func TestProcessBackupCompletions(t *testing.T) {
backupLocation: builder.ForBackupStorageLocation("velero", "read-write").
Bucket("store-1").
AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).
Phase(velerov1api.BackupStorageLocationPhaseAvailable).
Result(),
defaultVolumesToFsBackup: true,
expectedResult: &velerov1api.Backup{
@@ -779,6 +786,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: "read-write",
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -817,6 +825,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -855,6 +864,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -894,6 +904,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -933,6 +944,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -972,6 +984,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1011,6 +1024,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1051,6 +1065,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFailed,
@@ -1091,6 +1106,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFailed,
@@ -1131,6 +1147,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1172,6 +1189,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1213,6 +1231,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1254,6 +1273,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1296,6 +1316,7 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1337,6 +1358,105 @@ func TestProcessBackupCompletions(t *testing.T) {
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedResources: append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
Version: 1,
FormatVersion: "1.1.0",
StartTimestamp: &timestamp,
Expiration: &timestamp,
CSIVolumeSnapshotsAttempted: 0,
CSIVolumeSnapshotsCompleted: 0,
},
},
volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
},
{
name: "backup with namespace-scoped and cluster-scoped resource filters",
backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").
ExcludedClusterScopedResources("clusterroles").
IncludedClusterScopedResources("storageclasses").
ExcludedNamespaceScopedResources("secrets").
IncludedNamespaceScopedResources("pods").Result(),
backupLocation: defaultBackupLocation,
defaultVolumesToFsBackup: false,
defaultSnapshotMoveData: true,
expectedResult: &velerov1api.Backup{
TypeMeta: metav1.TypeMeta{
Kind: "Backup",
APIVersion: "velero.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "backup-1",
Annotations: map[string]string{
"velero.io/source-cluster-k8s-major-version": "1",
"velero.io/source-cluster-k8s-minor-version": "16",
"velero.io/source-cluster-k8s-gitversion": "v1.16.4",
"velero.io/resource-timeout": "0s",
},
Labels: map[string]string{
"velero.io/storage-location": "loc-1",
},
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
IncludedClusterScopedResources: []string{"storageclasses"},
ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...),
IncludedNamespaceScopedResources: []string{"pods"},
ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
Version: 1,
FormatVersion: "1.1.0",
StartTimestamp: &timestamp,
Expiration: &timestamp,
CSIVolumeSnapshotsAttempted: 0,
CSIVolumeSnapshotsCompleted: 0,
},
},
volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
},
{
name: "backup's include filter overlap with default exclude resources",
backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").
ExcludedClusterScopedResources("clusterroles").
IncludedClusterScopedResources("storageclasses", "volumesnapshotcontents.snapshot.storage.k8s.io").
ExcludedNamespaceScopedResources("secrets").
IncludedNamespaceScopedResources("pods", "volumesnapshots.snapshot.storage.k8s.io").Result(),
backupLocation: defaultBackupLocation,
defaultVolumesToFsBackup: false,
defaultSnapshotMoveData: true,
expectedResult: &velerov1api.Backup{
TypeMeta: metav1.TypeMeta{
Kind: "Backup",
APIVersion: "velero.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "backup-1",
Annotations: map[string]string{
"velero.io/source-cluster-k8s-major-version": "1",
"velero.io/source-cluster-k8s-minor-version": "16",
"velero.io/source-cluster-k8s-gitversion": "v1.16.4",
"velero.io/resource-timeout": "0s",
},
Labels: map[string]string{
"velero.io/storage-location": "loc-1",
},
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
IncludedClusterScopedResources: []string{"storageclasses"},
ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...),
IncludedNamespaceScopedResources: []string{"pods"},
ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...),
},
Status: velerov1api.BackupStatus{
Phase: velerov1api.BackupPhaseFinalizing,
@@ -1477,11 +1597,13 @@ func TestProcessBackupCompletions(t *testing.T) {
}
func TestValidateAndGetSnapshotLocations(t *testing.T) {
defaultBSL := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "bsl").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
tests := []struct {
name string
backup *velerov1api.Backup
locations []*velerov1api.VolumeSnapshotLocation
defaultLocations map[string]string
bsl velerov1api.BackupStorageLocation
expectedVolumeSnapshotLocationNames []string // adding these in the expected order will allow to test with better msgs in case of a test failure
expectedErrors string
expectedSuccess bool
@@ -1495,6 +1617,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(),
},
expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false,
bsl: *defaultBSL,
},
{
name: "duplicate locationName per provider: should filter out dups",
@@ -1505,6 +1628,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "multiple non-dupe location names per provider should error",
@@ -1516,6 +1640,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedErrors: "more than one VolumeSnapshotLocation name specified for provider aws: aws-us-west-1; unexpected name was aws-us-east-1",
expectedSuccess: false,
bsl: *defaultBSL,
},
{
name: "no location name for the provider exists, only one VSL for the provider: use it",
@@ -1525,6 +1650,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "no location name for the provider exists, no default, more than one VSL for the provider: error",
@@ -1534,6 +1660,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(),
},
expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default",
bsl: *defaultBSL,
},
{
name: "no location name for the provider exists, more than one VSL for the provider: the provider's default should be added",
@@ -1545,11 +1672,13 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "no existing location name and no default location name given",
backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(),
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "multiple location names for a provider, default location name for another provider",
@@ -1561,6 +1690,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1", "some-name"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "location name does not correspond to any existing location and snapshotvolume disabled; should return error",
@@ -1572,6 +1702,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: nil,
expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false,
bsl: *defaultBSL,
},
{
name: "duplicate locationName per provider and snapshotvolume disabled; should return only one BSL",
@@ -1582,6 +1713,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "no location name for the provider exists, only one VSL created and snapshotvolume disabled; should return the VSL",
@@ -1591,6 +1723,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"},
expectedSuccess: true,
bsl: *defaultBSL,
},
{
name: "multiple location names for a provider, no default location and backup has no location defined, but snapshotvolume disabled, should return error",
@@ -1601,6 +1734,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
},
expectedVolumeSnapshotLocationNames: nil,
expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default",
bsl: *defaultBSL,
},
}
pkg/controller/backup_deletion_controller.go
+8 -4
View File
@@ -22,9 +22,8 @@ import (
"fmt"
"time"
"github.com/vmware-tanzu/velero/pkg/util/csi"
jsonpatch "github.com/evanphx/json-patch/v5"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
@@ -37,8 +36,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/internal/delete"
"github.com/vmware-tanzu/velero/internal/volume"
@@ -56,8 +53,10 @@ import (
repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager"
repotypes "github.com/vmware-tanzu/velero/pkg/repository/types"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
"github.com/vmware-tanzu/velero/pkg/util/csi"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero"
)
const (
@@ -202,6 +201,11 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque
return ctrl.Result{}, err
}
if !veleroutil.BSLIsAvailable(*location) {
err := r.patchDeleteBackupRequestWithError(ctx, dbr, fmt.Errorf("cannot delete backup because backup storage location %s is currently in Unavailable state", location.Name))
return ctrl.Result{}, err
}
// if the request object has no labels defined, initialize an empty map since
// we will be updating labels
if dbr.Labels == nil {
pkg/controller/backup_deletion_controller_test.go
+34 -1
View File
@@ -126,6 +126,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
dbr := defaultTestDbr()
td := setupBackupDeletionControllerTest(t, dbr, location, backup)
@@ -254,7 +257,7 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
t.Run("backup storage location is in read-only mode", func(t *testing.T) {
backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result()
location := builder.ForBackupStorageLocation("velero", "default").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result()
location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result()
td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup)
@@ -268,6 +271,24 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
assert.Len(t, res.Status.Errors, 1)
assert.Equal(t, "cannot delete backup because backup storage location default is currently in read-only mode", res.Status.Errors[0])
})
t.Run("backup storage location is in unavailable state", func(t *testing.T) {
backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result()
location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result()
td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup)
_, err := td.controller.Reconcile(context.TODO(), td.req)
require.NoError(t, err)
res := &velerov1api.DeleteBackupRequest{}
err = td.fakeClient.Get(ctx, td.req.NamespacedName, res)
require.NoError(t, err)
assert.Equal(t, "Processed", string(res.Status.Phase))
assert.Len(t, res.Status.Errors, 1)
assert.Equal(t, "cannot delete backup because backup storage location default is currently in Unavailable state", res.Status.Errors[0])
})
t.Run("full delete, no errors", func(t *testing.T) {
input := defaultTestDbr()
@@ -297,6 +318,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
snapshotLocation := &velerov1api.VolumeSnapshotLocation{
@@ -416,6 +440,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
snapshotLocation := &velerov1api.VolumeSnapshotLocation{
@@ -518,6 +545,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
snapshotLocation := &velerov1api.VolumeSnapshotLocation{
@@ -600,6 +630,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
snapshotLocation := &velerov1api.VolumeSnapshotLocation{
pkg/controller/backup_sync_controller.go
+5
View File
@@ -41,6 +41,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/persistence"
"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
"github.com/vmware-tanzu/velero/pkg/util/kube"
veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -92,6 +93,10 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
return ctrl.Result{}, errors.Wrapf(err, "error getting BackupStorageLocation %s", req.String())
}
if !veleroutil.BSLIsAvailable(*location) {
log.Errorf("BackupStorageLocation is in unavailable state, skip syncing backup from it.")
return ctrl.Result{}, nil
}
pluginManager := b.newPluginManager(log)
defer pluginManager.CleanupClients()
pkg/controller/backup_sync_controller_test.go
+21
View File
@@ -62,6 +62,9 @@ func defaultLocation(namespace string) *velerov1api.BackupStorageLocation {
},
Default: true,
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
}
@@ -141,6 +144,9 @@ func defaultLocationWithLongerLocationName(namespace string) *velerov1api.Backup
},
},
},
Status: velerov1api.BackupStorageLocationStatus{
Phase: velerov1api.BackupStorageLocationPhaseAvailable,
},
}
}
@@ -177,6 +183,21 @@ var _ = Describe("Backup Sync Reconciler", func() {
namespace: "ns-1",
location: defaultLocation("ns-1"),
},
{
name: "unavailable BSL",
namespace: "ns-1",
location: builder.ForBackupStorageLocation("ns-1", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(),
cloudBackups: []*cloudBackupData{
{
backup: builder.ForBackup("ns-1", "backup-1").Result(),
backupShouldSkipSync: true,
},
{
backup: builder.ForBackup("ns-1", "backup-2").Result(),
backupShouldSkipSync: true,
},
},
},
{
name: "normal case",
namespace: "ns-1",
pkg/controller/data_upload_controller_test.go
+7 -9
View File
@@ -312,30 +312,29 @@ func (f *fakeSnapshotExposer) DiagnoseExpose(context.Context, corev1api.ObjectRe
func (f *fakeSnapshotExposer) CleanUp(context.Context, corev1api.ObjectReference, string, string) {
}
type fakeDataUploadFSBR struct {
du *velerov2alpha1api.DataUpload
type fakeFSBR struct {
kubeClient kbclient.Client
clock clock.WithTickerAndDelayedExecution
initErr error
startErr error
}
func (f *fakeDataUploadFSBR) Init(ctx context.Context, param any) error {
func (f *fakeFSBR) Init(ctx context.Context, param any) error {
return f.initErr
}
func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error {
func (f *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error {
return f.startErr
}
func (f *fakeDataUploadFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
return nil
}
func (b *fakeDataUploadFSBR) Cancel() {
func (b *fakeFSBR) Cancel() {
}
func (b *fakeDataUploadFSBR) Close(ctx context.Context) {
func (b *fakeFSBR) Close(ctx context.Context) {
}
func TestReconcile(t *testing.T) {
@@ -651,8 +650,7 @@ func TestReconcile(t *testing.T) {
}
datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
return &fakeDataUploadFSBR{
du: test.du,
return &fakeFSBR{
kubeClient: r.client,
clock: r.Clock,
initErr: test.fsBRInitErr,
pkg/controller/gc_controller.go
+9
View File
@@ -18,6 +18,7 @@ package controller
import (
"context"
"fmt"
"time"
"github.com/pkg/errors"
@@ -36,6 +37,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/constant"
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/vmware-tanzu/velero/pkg/util/kube"
veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero"
)
const (
@@ -44,6 +46,7 @@ const (
gcFailureBSLNotFound = "BSLNotFound"
gcFailureBSLCannotGet = "BSLCannotGet"
gcFailureBSLReadOnly = "BSLReadOnly"
gcFailureBSLUnavailable = "BSLUnavailable"
)
// gcReconciler creates DeleteBackupRequests for expired backups.
@@ -144,12 +147,18 @@ func (c *gcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re
} else {
backup.Labels[garbageCollectionFailure] = gcFailureBSLCannotGet
}
if err := c.Update(ctx, backup); err != nil {
log.WithError(err).Error("error updating backup labels")
}
return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location")
}
if !veleroutil.BSLIsAvailable(*loc) {
log.Infof("BSL %s is unavailable, cannot gc backup", loc.Name)
return ctrl.Result{}, fmt.Errorf("bsl %s is unavailable, cannot gc backup", loc.Name)
}
if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly {
log.Infof("Backup cannot be garbage-collected because backup storage location %s is currently in read-only mode", loc.Name)
backup.Labels[garbageCollectionFailure] = gcFailureBSLReadOnly
pkg/controller/gc_controller_test.go
+9 -3
View File
@@ -46,7 +46,7 @@ func mockGCReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeCloc
func TestGCReconcile(t *testing.T) {
fakeClock := testclocks.NewFakeClock(time.Now())
defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result()
defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
tests := []struct {
name string
@@ -66,12 +66,12 @@ func TestGCReconcile(t *testing.T) {
{
name: "expired backup in read-only storage location is not deleted",
backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-only").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(),
},
{
name: "expired backup in read-write storage location is deleted",
backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-write").Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Result(),
backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(),
},
{
name: "expired backup with no pending deletion requests is deleted",
@@ -118,6 +118,12 @@ func TestGCReconcile(t *testing.T) {
},
},
},
{
name: "BSL is unavailable",
backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(),
backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(),
expectError: true,
},
}
for _, test := range tests {
pkg/controller/pod_volume_backup_controller.go
+681 -227
View File
File diff suppressed because it is too large Load Diff
pkg/controller/pod_volume_backup_controller_test.go
+842 -309
View File
File diff suppressed because it is too large Load Diff
pkg/controller/pod_volume_restore_controller.go
+710 -184
View File
File diff suppressed because it is too large Load Diff
pkg/controller/pod_volume_restore_controller_legacy.go
+10
View File
@@ -0,0 +1,10 @@
package controller
import (
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/uploader"
)
func isLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool {
return pvr.Spec.UploaderType == uploader.ResticType
}
pkg/controller/pod_volume_restore_controller_test.go
+926 -7
View File
@@ -18,21 +18,43 @@ package controller
import (
"context"
"fmt"
"testing"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
clientgofake "k8s.io/client-go/kubernetes/fake"
clocks "k8s.io/utils/clock"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/datapath"
datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks"
"github.com/vmware-tanzu/velero/pkg/exposer"
exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks"
"github.com/vmware-tanzu/velero/pkg/restorehelper"
"github.com/vmware-tanzu/velero/pkg/test"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
func TestShouldProcess(t *testing.T) {
@@ -195,11 +217,11 @@ func TestShouldProcess(t *testing.T) {
c := &PodVolumeRestoreReconciler{
logger: logrus.New(),
Client: cli,
client: cli,
clock: &clocks.RealClock{},
}
shouldProcess, _, _ := c.shouldProcess(ctx, c.logger, ts.obj)
shouldProcess, _, _ := shouldProcess(ctx, c.client, c.logger, ts.obj)
require.Equal(t, ts.shouldProcessed, shouldProcess)
})
}
@@ -478,7 +500,7 @@ func TestGetInitContainerIndex(t *testing.T) {
}
}
func TestFindVolumeRestoresForPod(t *testing.T) {
func TestFindPVRForTargetPod(t *testing.T) {
pod := &corev1api.Pod{}
pod.UID = "uid"
@@ -488,14 +510,14 @@ func TestFindVolumeRestoresForPod(t *testing.T) {
// no matching PVR
reconciler := &PodVolumeRestoreReconciler{
Client: clientBuilder.Build(),
client: clientBuilder.Build(),
logger: logrus.New(),
}
requests := reconciler.findVolumeRestoresForPod(context.Background(), pod)
requests := reconciler.findPVRForTargetPod(context.Background(), pod)
assert.Empty(t, requests)
// contain one matching PVR
reconciler.Client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{
reconciler.client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{
Items: []velerov1api.PodVolumeRestore{
{
ObjectMeta: metav1.ObjectMeta{
@@ -515,6 +537,903 @@ func TestFindVolumeRestoresForPod(t *testing.T) {
},
},
}).Build()
requests = reconciler.findVolumeRestoresForPod(context.Background(), pod)
requests = reconciler.findPVRForTargetPod(context.Background(), pod)
assert.Len(t, requests, 1)
}
const pvrName string = "pvr-1"
func pvrBuilder() *builder.PodVolumeRestoreBuilder {
return builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).
BackupStorageLocation("bsl-loc").
SnapshotID("test-snapshot-id")
}
func initPodVolumeRestoreReconciler(objects []runtime.Object, cliObj []client.Object, needError ...bool) (*PodVolumeRestoreReconciler, error) {
var errs = make([]error, 6)
for k, isError := range needError {
if k == 0 && isError {
errs[0] = fmt.Errorf("Get error")
} else if k == 1 && isError {
errs[1] = fmt.Errorf("Create error")
} else if k == 2 && isError {
errs[2] = fmt.Errorf("Update error")
} else if k == 3 && isError {
errs[3] = fmt.Errorf("Patch error")
} else if k == 4 && isError {
errs[4] = apierrors.NewConflict(velerov1api.Resource("podvolumerestore"), pvrName, errors.New("conflict"))
} else if k == 5 && isError {
errs[5] = fmt.Errorf("List error")
}
}
return initPodVolumeRestoreReconcilerWithError(objects, cliObj, errs...)
}
func initPodVolumeRestoreReconcilerWithError(objects []runtime.Object, cliObj []client.Object, needError ...error) (*PodVolumeRestoreReconciler, error) {
scheme := runtime.NewScheme()
err := velerov1api.AddToScheme(scheme)
if err != nil {
return nil, err
}
err = corev1api.AddToScheme(scheme)
if err != nil {
return nil, err
}
fakeClient := &FakeClient{
Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cliObj...).Build(),
}
for k := range needError {
if k == 0 {
fakeClient.getError = needError[0]
} else if k == 1 {
fakeClient.createError = needError[1]
} else if k == 2 {
fakeClient.updateError = needError[2]
} else if k == 3 {
fakeClient.patchError = needError[3]
} else if k == 4 {
fakeClient.updateConflict = needError[4]
} else if k == 5 {
fakeClient.listError = needError[5]
}
}
var fakeKubeClient *clientgofake.Clientset
if len(objects) != 0 {
fakeKubeClient = clientgofake.NewSimpleClientset(objects...)
} else {
fakeKubeClient = clientgofake.NewSimpleClientset()
}
fakeFS := velerotest.NewFakeFileSystem()
pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "test-uid", "test-pvc")
_, err = fakeFS.Create(pathGlob)
if err != nil {
return nil, err
}
dataPathMgr := datapath.NewManager(1)
return NewPodVolumeRestoreReconciler(fakeClient, nil, fakeKubeClient, dataPathMgr, "test-node", time.Minute*5, time.Minute, corev1api.ResourceRequirements{}, velerotest.NewLogger()), nil
}
func TestPodVolumeRestoreReconcile(t *testing.T) {
daemonSet := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "velero",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
APIVersion: appsv1api.SchemeGroupVersion.String(),
},
Spec: appsv1api.DaemonSetSpec{
Template: corev1api.PodTemplateSpec{
Spec: corev1api.PodSpec{
Containers: []corev1api.Container{
{
Image: "fake-image",
},
},
},
},
},
}
node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result()
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
notCreatePVR bool
targetPod *corev1api.Pod
dataMgr *datapath.Manager
needErrs []bool
needCreateFSBR bool
needDelete bool
sportTime *metav1.Time
mockExposeErr *bool
isGetExposeErr bool
isGetExposeNil bool
isPeekExposeErr bool
isNilExposer bool
notNilExpose bool
notMockCleanUp bool
mockInit bool
mockInitErr error
mockStart bool
mockStartErr error
mockCancel bool
mockClose bool
needExclusiveUpdateError error
expected *velerov1api.PodVolumeRestore
expectDeleted bool
expectCancelRecord bool
expectedResult *ctrl.Result
expectedErr string
expectDataPath bool
}{
{
name: "pvr not found",
pvr: pvrBuilder().Result(),
notCreatePVR: true,
},
{
name: "pvr not created in velero default namespace",
pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(),
},
{
name: "get dd fail",
pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(),
needErrs: []bool{true, false, false, false},
expectedErr: "Get error",
},
{
name: "add finalizer to pvr",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "add finalizer to pvr failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(),
needErrs: []bool{false, false, true, false},
expectedErr: "error updating PVR velero/pvr-1: Update error",
},
{
name: "pvr is under deletion",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(),
needDelete: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(),
},
{
name: "pvr is under deletion but cancel failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(),
needErrs: []bool{false, false, true, false},
needDelete: true,
expectedErr: "error updating PVR velero/pvr-1: Update error",
},
{
name: "pvr is under deletion and in terminal state",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(),
sportTime: &metav1.Time{Time: time.Now()},
needDelete: true,
expectDeleted: true,
},
{
name: "pvr is under deletion and in terminal state, but remove finalizer failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(),
needErrs: []bool{false, false, true, false},
needDelete: true,
expectedErr: "error updating PVR velero/pvr-1: Update error",
},
{
name: "delay cancel negative for others",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
sportTime: &metav1.Time{Time: time.Now()},
expectCancelRecord: true,
},
{
name: "delay cancel negative for inProgress",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)},
expectCancelRecord: true,
},
{
name: "delay cancel affirmative for others",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)},
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(),
},
{
name: "delay cancel affirmative for inProgress",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)},
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(),
},
{
name: "delay cancel failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
needErrs: []bool{false, false, true, false},
sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)},
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
expectCancelRecord: true,
},
{
name: "Unknown pvr status",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase("Unknown").Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "new pvr but accept failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).PodNamespace("test-ns").PodName("test-pod").Result(),
targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(),
needErrs: []bool{false, false, true, false},
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(),
expectedErr: "error accepting PVR pvr-1: error updating PVR velero/pvr-1: Update error",
},
{
name: "pvr is cancel on accepted",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(),
expectCancelRecord: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(),
},
{
name: "pvr expose failed",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(),
targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(),
mockExposeErr: boolptr.True(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("error to expose PVR").Result(),
expectedErr: "Error to expose restore exposer",
},
{
name: "pvr succeeds for accepted",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(),
mockExposeErr: boolptr.False(),
notMockCleanUp: true,
targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
},
{
name: "prepare timeout on accepted",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("timeout on preparing PVR").Result(),
},
{
name: "peek error on accepted",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).Result(),
isPeekExposeErr: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Message("found a PVR velero/pvr-1 with expose error: fake-peek-error. mark it as cancel").Result(),
},
{
name: "cancel on pvr",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Node("test-node").Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(),
},
{
name: "Failed to get restore expose on prepared",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
isGetExposeErr: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(),
expectedErr: "Error to get PVR exposer",
},
{
name: "Get nil restore expose on prepared",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
isGetExposeNil: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(),
expectedErr: "no expose result is available for the current node",
},
{
name: "Error in data path is concurrent limited",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
dataMgr: datapath.NewManager(0),
notNilExpose: true,
notMockCleanUp: true,
expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5},
},
{
name: "data path init error",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
mockInit: true,
mockInitErr: errors.New("fake-data-path-init-error"),
mockClose: true,
notNilExpose: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error initializing data path").Result(),
expectedErr: "error initializing asyncBR: fake-data-path-init-error",
},
{
name: "Unable to update status to in progress for pvr",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
needErrs: []bool{false, false, true, false},
mockInit: true,
mockClose: true,
notNilExpose: true,
notMockCleanUp: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "data path start error",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
mockInit: true,
mockStart: true,
mockStartErr: errors.New("fake-data-path-start-error"),
mockClose: true,
notNilExpose: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error starting data path").Result(),
expectedErr: "error starting async restore for pod test-name, volume test-pvc: fake-data-path-start-error",
},
{
name: "Prepare succeeds",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
mockInit: true,
mockStart: true,
notNilExpose: true,
notMockCleanUp: true,
expectDataPath: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "In progress pvr is not handled by the current node",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "In progress pvr is not set as cancel",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "Cancel pvr in progress with empty FSBR",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(),
},
{
name: "Cancel pvr in progress and patch pvr error",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
needErrs: []bool{false, false, true, false},
needCreateFSBR: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(),
expectedErr: "error updating PVR velero/pvr-1: Update error",
expectCancelRecord: true,
expectDataPath: true,
},
{
name: "Cancel pvr in progress succeeds",
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(),
needCreateFSBR: true,
mockCancel: true,
expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceling).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(),
expectDataPath: true,
expectCancelRecord: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objs := []runtime.Object{daemonSet, node}
ctlObj := []client.Object{}
if test.targetPod != nil {
ctlObj = append(ctlObj, test.targetPod)
}
r, err := initPodVolumeRestoreReconciler(objs, ctlObj, test.needErrs...)
require.NoError(t, err)
if !test.notCreatePVR {
err = r.client.Create(context.Background(), test.pvr)
require.NoError(t, err)
}
if test.needDelete {
err = r.client.Delete(context.Background(), test.pvr)
require.NoError(t, err)
}
if test.dataMgr != nil {
r.dataPathMgr = test.dataMgr
} else {
r.dataPathMgr = datapath.NewManager(1)
}
if test.sportTime != nil {
r.cancelledPVR[test.pvr.Name] = test.sportTime.Time
}
funcExclusiveUpdatePodVolumeRestore = exclusiveUpdatePodVolumeRestore
if test.needExclusiveUpdateError != nil {
funcExclusiveUpdatePodVolumeRestore = func(context.Context, kbclient.Client, *velerov1api.PodVolumeRestore, func(*velerov1api.PodVolumeRestore)) (bool, error) {
return false, test.needExclusiveUpdateError
}
}
datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string,
string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
asyncBR := datapathmockes.NewAsyncBR(t)
if test.mockInit {
asyncBR.On("Init", mock.Anything, mock.Anything).Return(test.mockInitErr)
}
if test.mockStart {
asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr)
}
if test.mockCancel {
asyncBR.On("Cancel").Return()
}
if test.mockClose {
asyncBR.On("Close", mock.Anything).Return()
}
return asyncBR
}
if test.mockExposeErr != nil || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose {
if test.isNilExposer {
r.exposer = nil
} else {
r.exposer = func() exposer.PodVolumeExposer {
ep := exposermockes.NewPodVolumeExposer(t)
if test.mockExposeErr != nil {
if boolptr.IsSetToTrue(test.mockExposeErr) {
ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer"))
} else {
ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
}
} else if test.notNilExpose {
hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result()
hostingPod.ObjectMeta.SetUID("test-uid")
ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, VolumeName: "test-pvc"}}, nil)
} else if test.isGetExposeErr {
ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("Error to get PVR exposer"))
} else if test.isGetExposeNil {
ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
} else if test.isPeekExposeErr {
ep.On("PeekExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("fake-peek-error"))
}
if !test.notMockCleanUp {
ep.On("CleanUp", mock.Anything, mock.Anything).Return()
}
return ep
}()
}
}
if test.needCreateFSBR {
if fsBR := r.dataPathMgr.GetAsyncBR(test.pvr.Name); fsBR == nil {
_, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeRestore, test.pvr.Name, pVBRRequestor,
velerov1api.DefaultNamespace, "", "", datapath.Callbacks{OnCancelled: r.OnDataPathCancelled}, false, velerotest.NewLogger())
require.NoError(t, err)
}
}
actualResult, err := r.Reconcile(ctx, ctrl.Request{
NamespacedName: types.NamespacedName{
Namespace: velerov1api.DefaultNamespace,
Name: test.pvr.Name,
},
})
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
}
if test.expectedResult != nil {
assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue)
assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter)
}
if test.expected != nil || test.expectDeleted {
pvr := velerov1api.PodVolumeRestore{}
err = r.client.Get(ctx, kbclient.ObjectKey{
Name: test.pvr.Name,
Namespace: test.pvr.Namespace,
}, &pvr)
if test.expectDeleted {
assert.True(t, apierrors.IsNotFound(err))
} else {
require.NoError(t, err)
assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase)
assert.Contains(t, pvr.Status.Message, test.expected.Status.Message)
assert.Equal(t, test.expected.Finalizers, pvr.Finalizers)
assert.Equal(t, test.expected.Spec.Cancel, pvr.Spec.Cancel)
}
}
if !test.expectDataPath {
assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name))
} else {
assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name))
}
if test.expectCancelRecord {
assert.Contains(t, r.cancelledPVR, test.pvr.Name)
} else {
assert.Empty(t, r.cancelledPVR)
}
})
}
}
func TestOnPodVolumeRestoreFailed(t *testing.T) {
for _, getErr := range []bool{true, false} {
ctx := context.TODO()
needErrs := []bool{getErr, false, false, false}
r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...)
require.NoError(t, err)
pvr := pvrBuilder().Result()
namespace := pvr.Namespace
pvrName := pvr.Name
assert.NoError(t, r.client.Create(ctx, pvr))
r.OnDataPathFailed(ctx, namespace, pvrName, fmt.Errorf("Failed to handle %v", pvrName))
updatedPVR := &velerov1api.PodVolumeRestore{}
if getErr {
assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase)
assert.True(t, updatedPVR.Status.StartTimestamp.IsZero())
} else {
assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.Equal(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase)
assert.True(t, updatedPVR.Status.StartTimestamp.IsZero())
}
}
}
func TestOnPodVolumeRestoreCancelled(t *testing.T) {
for _, getErr := range []bool{true, false} {
ctx := context.TODO()
needErrs := []bool{getErr, false, false, false}
r, err := initPodVolumeRestoreReconciler(nil, nil, needErrs...)
require.NoError(t, err)
pvr := pvrBuilder().Result()
namespace := pvr.Namespace
pvrName := pvr.Name
assert.NoError(t, r.client.Create(ctx, pvr))
r.OnDataPathCancelled(ctx, namespace, pvrName)
updatedPVR := &velerov1api.PodVolumeRestore{}
if getErr {
assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase)
assert.True(t, updatedPVR.Status.StartTimestamp.IsZero())
} else {
assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.Equal(t, velerov1api.PodVolumeRestorePhaseCanceled, updatedPVR.Status.Phase)
assert.False(t, updatedPVR.Status.StartTimestamp.IsZero())
assert.False(t, updatedPVR.Status.CompletionTimestamp.IsZero())
}
}
}
func TestOnPodVolumeRestoreCompleted(t *testing.T) {
tests := []struct {
name string
emptyFSBR bool
isGetErr bool
rebindVolumeErr bool
}{
{
name: "PVR complete",
emptyFSBR: false,
isGetErr: false,
rebindVolumeErr: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.TODO()
needErrs := []bool{test.isGetErr, false, false, false}
r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...)
r.exposer = func() exposer.PodVolumeExposer {
ep := exposermockes.NewPodVolumeExposer(t)
ep.On("CleanUp", mock.Anything, mock.Anything).Return()
return ep
}()
require.NoError(t, err)
pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result()
namespace := pvr.Namespace
ddName := pvr.Name
assert.NoError(t, r.client.Create(ctx, pvr))
r.OnDataPathCompleted(ctx, namespace, ddName, datapath.Result{})
updatedDD := &velerov1api.PodVolumeRestore{}
if test.isGetErr {
assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD))
assert.Equal(t, velerov1api.PodVolumeRestorePhase(""), updatedDD.Status.Phase)
assert.True(t, updatedDD.Status.CompletionTimestamp.IsZero())
} else {
assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD))
assert.Equal(t, velerov1api.PodVolumeRestorePhaseCompleted, updatedDD.Status.Phase)
assert.False(t, updatedDD.Status.CompletionTimestamp.IsZero())
}
})
}
}
func TestOnPodVolumeRestoreProgress(t *testing.T) {
totalBytes := int64(1024)
bytesDone := int64(512)
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
progress uploader.Progress
needErrs []bool
}{
{
name: "patch in progress phase success",
pvr: pvrBuilder().Result(),
progress: uploader.Progress{
TotalBytes: totalBytes,
BytesDone: bytesDone,
},
},
{
name: "failed to get pvr",
pvr: pvrBuilder().Result(),
needErrs: []bool{true, false, false, false},
},
{
name: "failed to patch pvr",
pvr: pvrBuilder().Result(),
needErrs: []bool{false, false, true, false},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.TODO()
r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, test.needErrs...)
require.NoError(t, err)
defer func() {
r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{})
}()
pvr := pvrBuilder().Result()
namespace := pvr.Namespace
pvrName := pvr.Name
assert.NoError(t, r.client.Create(context.Background(), pvr))
// Create a Progress object
progress := &uploader.Progress{
TotalBytes: totalBytes,
BytesDone: bytesDone,
}
r.OnDataPathProgress(ctx, namespace, pvrName, progress)
if len(test.needErrs) != 0 && !test.needErrs[0] {
updatedPVR := &velerov1api.PodVolumeRestore{}
assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.Equal(t, test.progress.TotalBytes, updatedPVR.Status.Progress.TotalBytes)
assert.Equal(t, test.progress.BytesDone, updatedPVR.Status.Progress.BytesDone)
}
})
}
}
func TestFindPVBForRestorePod(t *testing.T) {
needErrs := []bool{false, false, false, false}
r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...)
require.NoError(t, err)
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
pod *corev1api.Pod
checkFunc func(*velerov1api.PodVolumeRestore, []reconcile.Request)
}{
{
name: "find pvr for pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
// Assert that the function returns a single request
assert.Len(t, requests, 1)
// Assert that the request contains the correct namespaced name
assert.Equal(t, pvr.Namespace, requests[0].Namespace)
assert.Equal(t, pvr.Name, requests[0].Name)
},
}, {
name: "no selected label found for pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
// Assert that the function returns a single request
assert.Empty(t, requests)
},
}, {
name: "no matched pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: "non-existing-pvr"}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
assert.Empty(t, requests)
},
},
{
name: "pvr not accept",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
assert.Empty(t, requests)
},
},
}
for _, test := range tests {
ctx := context.Background()
assert.NoError(t, r.client.Create(ctx, test.pod))
assert.NoError(t, r.client.Create(ctx, test.pvr))
// Call the findSnapshotRestoreForPod function
requests := r.findPVRForRestorePod(context.Background(), test.pod)
test.checkFunc(test.pvr, requests)
r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{})
if test.pod != nil {
r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{})
}
}
}
func TestOnPVRPrepareTimeout(t *testing.T) {
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
needErrs []error
expected *velerov1api.PodVolumeRestore
}{
{
name: "update fail",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil},
expected: pvrBuilder().Result(),
},
{
name: "update interrupted",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil},
expected: pvrBuilder().Result(),
},
{
name: "succeed",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, nil, nil},
expected: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(),
},
}
for _, test := range tests {
ctx := context.Background()
r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...)
require.NoError(t, err)
err = r.client.Create(ctx, test.pvr)
require.NoError(t, err)
r.onPrepareTimeout(ctx, test.pvr)
pvr := velerov1api.PodVolumeRestore{}
_ = r.client.Get(ctx, kbclient.ObjectKey{
Name: test.pvr.Name,
Namespace: test.pvr.Namespace,
}, &pvr)
assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase)
}
}
func TestTryCancelPVR(t *testing.T) {
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
needErrs []error
succeeded bool
expectedErr string
}{
{
name: "update fail",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil},
},
{
name: "cancel by others",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil},
},
{
name: "succeed",
pvr: pvrBuilder().Result(),
needErrs: []error{nil, nil, nil, nil},
succeeded: true,
},
}
for _, test := range tests {
ctx := context.Background()
r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...)
require.NoError(t, err)
err = r.client.Create(ctx, test.pvr)
require.NoError(t, err)
r.tryCancelPodVolumeRestore(ctx, test.pvr, "")
if test.expectedErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, test.expectedErr)
}
}
}
func TestUpdatePVRWithRetry(t *testing.T) {
namespacedName := types.NamespacedName{
Name: pvrName,
Namespace: "velero",
}
// Define test cases
testCases := []struct {
Name string
needErrs []bool
noChange bool
ExpectErr bool
}{
{
Name: "SuccessOnFirstAttempt",
},
{
Name: "Error get",
needErrs: []bool{true, false, false, false, false},
ExpectErr: true,
},
{
Name: "Error update",
needErrs: []bool{false, false, true, false, false},
ExpectErr: true,
},
{
Name: "no change",
noChange: true,
needErrs: []bool{false, false, true, false, false},
},
{
Name: "Conflict with error timeout",
needErrs: []bool{false, false, false, false, true},
ExpectErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.Name, func(t *testing.T) {
ctx, cancelFunc := context.WithTimeout(context.TODO(), time.Second*5)
defer cancelFunc()
r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, tc.needErrs...)
require.NoError(t, err)
err = r.client.Create(ctx, pvrBuilder().Result())
require.NoError(t, err)
updateFunc := func(pvr *velerov1api.PodVolumeRestore) bool {
if tc.noChange {
return false
}
pvr.Spec.Cancel = true
return true
}
err = UpdatePVRWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc)
if tc.ExpectErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
pkg/controller/restore_controller.go
+10
View File
@@ -58,6 +58,7 @@ import (
kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
"github.com/vmware-tanzu/velero/pkg/util/results"
veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero"
pkgrestoreUtil "github.com/vmware-tanzu/velero/pkg/util/velero/restore"
)
@@ -393,6 +394,11 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf
return backupInfo{}, nil
}
if !veleroutil.BSLIsAvailable(*info.location) {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("The BSL %s is unavailable, cannot retrieve the backup", info.location.Name))
return backupInfo{}, nil
}
// Fill in the ScheduleName so it's easier to consume for metrics.
if restore.Spec.ScheduleName == "" {
restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel]
@@ -728,6 +734,10 @@ func (r *restoreReconciler) deleteExternalResources(restore *api.Restore) error
return errors.Wrap(err, fmt.Sprintf("can't get backup info, backup: %s", restore.Spec.BackupName))
}
if !veleroutil.BSLIsAvailable(*backupInfo.location) {
return fmt.Errorf("bsl %s is unavailable, cannot get the backup info", backupInfo.location.Name)
}
// delete restore files in object storage
pluginManager := r.newPluginManager(r.logger)
defer pluginManager.CleanupClients()
pkg/controller/restore_controller_test.go
+21 -5
View File
@@ -66,7 +66,7 @@ func TestFetchBackupInfo(t *testing.T) {
{
name: "lister has backup",
backupName: "backup-1",
informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()},
informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()},
informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()},
expectedRes: defaultBackup().StorageLocation("default").Result(),
},
@@ -74,7 +74,7 @@ func TestFetchBackupInfo(t *testing.T) {
name: "lister does not have a backup, but backupSvc does",
backupName: "backup-1",
backupStoreBackup: defaultBackup().StorageLocation("default").Result(),
informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()},
informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()},
informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()},
expectedRes: defaultBackup().StorageLocation("default").Result(),
},
@@ -211,7 +211,7 @@ func TestProcessQueueItemSkips(t *testing.T) {
}
func TestRestoreReconcile(t *testing.T) {
defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()
defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
now, err := time.Parse(time.RFC1123Z, time.RFC1123Z)
require.NoError(t, err)
@@ -464,6 +464,22 @@ func TestRestoreReconcile(t *testing.T) {
expectedCompletedTime: &timestamp,
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(),
},
{
name: "Restore creation is rejected when BSL is unavailable",
location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(),
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
backup: defaultBackup().StorageLocation("default").Result(),
expectedErr: false,
expectedPhase: string(velerov1api.RestorePhaseNew),
expectedValidationErrors: []string{"The BSL default is unavailable, cannot retrieve the backup"},
},
{
name: "Restore deletion is rejected when BSL is unavailable.",
location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(),
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).ObjectMeta(builder.WithFinalizers(ExternalResourcesFinalizer), builder.WithDeletionTimestamp(timestamp.Time)).Result(),
backup: defaultBackup().StorageLocation("default").Result(),
expectedErr: true,
},
}
formatFlag := logging.FormatText
@@ -738,7 +754,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) {
Result(),
))
location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()
location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
require.NoError(t, r.kbClient.Create(context.Background(), location))
restore = &velerov1api.Restore{
@@ -797,7 +813,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) {
},
}
location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()
location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
require.NoError(t, r.kbClient.Create(context.Background(), location))
require.NoError(t, r.kbClient.Create(
pkg/exposer/host_path.go
+26 -5
View File
@@ -19,13 +19,15 @@ package exposer
import (
"context"
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
"k8s.io/client-go/kubernetes"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/nodeagent"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
@@ -37,17 +39,17 @@ var singlePathMatch = kube.SinglePathMatch
// GetPodVolumeHostPath returns a path that can be accessed from the host for a given volume of a pod
func GetPodVolumeHostPath(ctx context.Context, pod *corev1api.Pod, volumeName string,
cli ctrlclient.Client, fs filesystem.Interface, log logrus.FieldLogger) (datapath.AccessPoint, error) {
kubeClient kubernetes.Interface, fs filesystem.Interface, log logrus.FieldLogger) (datapath.AccessPoint, error) {
logger := log.WithField("pod name", pod.Name).WithField("pod UID", pod.GetUID()).WithField("volume", volumeName)
volDir, err := getVolumeDirectory(ctx, logger, pod, volumeName, cli)
volDir, err := getVolumeDirectory(ctx, logger, pod, volumeName, kubeClient)
if err != nil {
return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume directory name for volume %s in pod %s", volumeName, pod.Name)
}
logger.WithField("volDir", volDir).Info("Got volume dir")
volMode, err := getVolumeMode(ctx, logger, pod, volumeName, cli)
volMode, err := getVolumeMode(ctx, logger, pod, volumeName, kubeClient)
if err != nil {
return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume mode for volume %s in pod %s", volumeName, pod.Name)
}
@@ -57,7 +59,7 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1api.Pod, volumeName st
volSubDir = "volumeDevices"
}
pathGlob := fmt.Sprintf("/host_pods/%s/%s/*/%s", string(pod.GetUID()), volSubDir, volDir)
pathGlob := fmt.Sprintf("%s/%s/%s/*/%s", nodeagent.HostPodVolumeMountPath(), string(pod.GetUID()), volSubDir, volDir)
logger.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob")
path, err := singlePathMatch(pathGlob, fs, logger)
@@ -72,3 +74,22 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1api.Pod, volumeName st
VolMode: volMode,
}, nil
}
var getHostPodPath = nodeagent.GetHostPodPath
func ExtractPodVolumeHostPath(ctx context.Context, path string, kubeClient kubernetes.Interface, veleroNamespace string, osType string) (string, error) {
podPath, err := getHostPodPath(ctx, kubeClient, veleroNamespace, osType)
if err != nil {
return "", errors.Wrap(err, "error getting host pod path from node-agent")
}
if osType == kube.NodeOSWindows {
podPath = strings.Replace(podPath, "/", "\\", -1)
}
if osType == kube.NodeOSWindows {
return strings.Replace(path, nodeagent.HostPodVolumeMountPathWin(), podPath, 1), nil
} else {
return strings.Replace(path, nodeagent.HostPodVolumeMountPath(), podPath, 1), nil
}
}
pkg/exposer/host_path_test.go
+65 -7
View File
@@ -18,26 +18,29 @@ package exposer
import (
"context"
"fmt"
"testing"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
corev1api "k8s.io/api/core/v1"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
"k8s.io/client-go/kubernetes"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/nodeagent"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
func TestGetPodVolumeHostPath(t *testing.T) {
tests := []struct {
name string
getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (string, error)
getVolumeModeFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (uploader.PersistentVolumeMode, error)
getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error)
getVolumeModeFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (uploader.PersistentVolumeMode, error)
pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error)
pod *corev1api.Pod
pvc string
@@ -45,7 +48,7 @@ func TestGetPodVolumeHostPath(t *testing.T) {
}{
{
name: "get volume dir fail",
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (string, error) {
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error) {
return "", errors.New("fake-error-1")
},
pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(),
@@ -54,10 +57,10 @@ func TestGetPodVolumeHostPath(t *testing.T) {
},
{
name: "single path match fail",
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (string, error) {
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error) {
return "", nil
},
getVolumeModeFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (uploader.PersistentVolumeMode, error) {
getVolumeModeFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (uploader.PersistentVolumeMode, error) {
return uploader.PersistentVolumeFilesystem, nil
},
pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) {
@@ -69,7 +72,7 @@ func TestGetPodVolumeHostPath(t *testing.T) {
},
{
name: "get block volume dir success",
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, ctrlclient.Client) (
getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (
string, error) {
return "fake-pvc-1", nil
},
@@ -102,3 +105,58 @@ func TestGetPodVolumeHostPath(t *testing.T) {
})
}
}
func TestExtractPodVolumeHostPath(t *testing.T) {
tests := []struct {
name string
getHostPodPathFunc func(context.Context, kubernetes.Interface, string, string) (string, error)
path string
osType string
expectedErr string
expected string
}{
{
name: "get host pod path error",
getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) {
return "", errors.New("fake-error-1")
},
expectedErr: "error getting host pod path from node-agent: fake-error-1",
},
{
name: "Windows os",
getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) {
return "/var/lib/kubelet/pods", nil
},
path: fmt.Sprintf("\\%s\\pod-id-xxx\\volumes\\kubernetes.io~csi\\pvc-id-xxx\\mount", nodeagent.HostPodVolumeMountPoint),
osType: kube.NodeOSWindows,
expected: "\\var\\lib\\kubelet\\pods\\pod-id-xxx\\volumes\\kubernetes.io~csi\\pvc-id-xxx\\mount",
},
{
name: "linux OS",
getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) {
return "/var/lib/kubelet/pods", nil
},
path: fmt.Sprintf("/%s/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nodeagent.HostPodVolumeMountPoint),
osType: kube.NodeOSLinux,
expected: "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.getHostPodPathFunc != nil {
getHostPodPath = test.getHostPodPathFunc
}
path, err := ExtractPodVolumeHostPath(context.Background(), test.path, nil, "", test.osType)
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
assert.Equal(t, test.expected, path)
}
})
}
}
pkg/exposer/mocks/PodVolumeExposer.go
+125
View File
@@ -0,0 +1,125 @@
// Code generated by mockery v2.39.1. DO NOT EDIT.
package mocks
import (
context "context"
client "sigs.k8s.io/controller-runtime/pkg/client"
exposer "github.com/vmware-tanzu/velero/pkg/exposer"
mock "github.com/stretchr/testify/mock"
time "time"
corev1api "k8s.io/api/core/v1"
)
// PodVolumeExposer is an autogenerated mock type for the PodVolumeExposer type
type PodVolumeExposer struct {
mock.Mock
}
// CleanUp provides a mock function with given fields: _a0, _a1
func (_m *PodVolumeExposer) CleanUp(_a0 context.Context, _a1 corev1api.ObjectReference) {
_m.Called(_a0, _a1)
}
// DiagnoseExpose provides a mock function with given fields: _a0, _a1
func (_m *PodVolumeExposer) DiagnoseExpose(_a0 context.Context, _a1 corev1api.ObjectReference) string {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for DiagnoseExpose")
}
var r0 string
if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference) string); ok {
r0 = rf(_a0, _a1)
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Expose provides a mock function with given fields: _a0, _a1, _a2
func (_m *PodVolumeExposer) Expose(_a0 context.Context, _a1 corev1api.ObjectReference, _a2 exposer.PodVolumeExposeParam) error {
ret := _m.Called(_a0, _a1, _a2)
if len(ret) == 0 {
panic("no return value specified for Expose")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, exposer.PodVolumeExposeParam) error); ok {
r0 = rf(_a0, _a1, _a2)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetExposed provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4
func (_m *PodVolumeExposer) GetExposed(_a0 context.Context, _a1 corev1api.ObjectReference, _a2 client.Client, _a3 string, _a4 time.Duration) (*exposer.ExposeResult, error) {
ret := _m.Called(_a0, _a1, _a2, _a3, _a4)
if len(ret) == 0 {
panic("no return value specified for GetExposed")
}
var r0 *exposer.ExposeResult
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error)); ok {
return rf(_a0, _a1, _a2, _a3, _a4)
}
if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) *exposer.ExposeResult); ok {
r0 = rf(_a0, _a1, _a2, _a3, _a4)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*exposer.ExposeResult)
}
}
if rf, ok := ret.Get(1).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) error); ok {
r1 = rf(_a0, _a1, _a2, _a3, _a4)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// PeekExposed provides a mock function with given fields: _a0, _a1
func (_m *PodVolumeExposer) PeekExposed(_a0 context.Context, _a1 corev1api.ObjectReference) error {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for PeekExposed")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference) error); ok {
r0 = rf(_a0, _a1)
} else {
r0 = ret.Error(0)
}
return r0
}
// NewPodVolumeExposer creates a new instance of PodVolumeExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewPodVolumeExposer(t interface {
mock.TestingT
Cleanup(func())
}) *PodVolumeExposer {
mock := &PodVolumeExposer{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
pkg/exposer/pod_volume.go
+392
View File
@@ -0,0 +1,392 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package exposer
import (
"context"
"fmt"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/pkg/nodeagent"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
const (
PodVolumeExposeTypeBackup = "pod-volume-backup"
PodVolumeExposeTypeRestore = "pod-volume-restore"
)
// PodVolumeExposeParam define the input param for pod volume Expose
type PodVolumeExposeParam struct {
// ClientPodName is the name of pod to be backed up or restored
ClientPodName string
// ClientNamespace is the namespace to be backed up or restored
ClientNamespace string
// ClientNamespace is the pod volume for the client PVC
ClientPodVolume string
// HostingPodLabels is the labels that are going to apply to the hosting pod
HostingPodLabels map[string]string
// HostingPodAnnotations is the annotations that are going to apply to the hosting pod
HostingPodAnnotations map[string]string
// Resources defines the resource requirements of the hosting pod
Resources corev1api.ResourceRequirements
// OperationTimeout specifies the time wait for resources operations in Expose
OperationTimeout time.Duration
// Type specifies the type of the expose, either backup or erstore
Type string
}
// PodVolumeExposer is the interfaces for a pod volume exposer
type PodVolumeExposer interface {
// Expose starts the process to a pod volume expose, the expose process may take long time
Expose(context.Context, corev1api.ObjectReference, PodVolumeExposeParam) error
// GetExposed polls the status of the expose.
// If the expose is accessible by the current caller, it waits the expose ready and returns the expose result.
// Otherwise, it returns nil as the expose result without an error.
GetExposed(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*ExposeResult, error)
// PeekExposed tests the status of the expose.
// If the expose is incomplete but not recoverable, it returns an error.
// Otherwise, it returns nil immediately.
PeekExposed(context.Context, corev1api.ObjectReference) error
// DiagnoseExpose generate the diagnostic info when the expose is not finished for a long time.
// If it finds any problem, it returns an string about the problem.
DiagnoseExpose(context.Context, corev1api.ObjectReference) string
// CleanUp cleans up any objects generated during the restore expose
CleanUp(context.Context, corev1api.ObjectReference)
}
// NewPodVolumeExposer creates a new instance of pod volume exposer
func NewPodVolumeExposer(kubeClient kubernetes.Interface, log logrus.FieldLogger) PodVolumeExposer {
return &podVolumeExposer{
kubeClient: kubeClient,
fs: filesystem.NewFileSystem(),
log: log,
}
}
type podVolumeExposer struct {
kubeClient kubernetes.Interface
fs filesystem.Interface
log logrus.FieldLogger
}
var getPodVolumeHostPath = GetPodVolumeHostPath
var extractPodVolumeHostPath = ExtractPodVolumeHostPath
func (e *podVolumeExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param PodVolumeExposeParam) error {
curLog := e.log.WithFields(logrus.Fields{
"owner": ownerObject.Name,
"client pod": param.ClientPodName,
"client pod volume": param.ClientPodVolume,
"client namespace": param.ClientNamespace,
"type": param.Type,
})
pod, err := e.kubeClient.CoreV1().Pods(param.ClientNamespace).Get(ctx, param.ClientPodName, metav1.GetOptions{})
if err != nil {
return errors.Wrapf(err, "error getting client pod %s", param.ClientPodName)
}
if pod.Spec.NodeName == "" {
return errors.Errorf("client pod %s doesn't have a node name", pod.Name)
}
nodeOS, err := kube.GetNodeOS(ctx, pod.Spec.NodeName, e.kubeClient.CoreV1())
if err != nil {
return errors.Wrapf(err, "error getting OS for node %s", pod.Spec.NodeName)
}
curLog.Infof("Client pod is running in node %s, os %s", pod.Spec.NodeName, nodeOS)
path, err := getPodVolumeHostPath(ctx, pod, param.ClientPodVolume, e.kubeClient, e.fs, e.log)
if err != nil {
return errors.Wrapf(err, "error to get pod volume path")
}
path.ByPath, err = extractPodVolumeHostPath(ctx, path.ByPath, e.kubeClient, ownerObject.Namespace, nodeOS)
if err != nil {
return errors.Wrapf(err, "error to extract pod volume path")
}
curLog.WithField("path", path).Infof("Host path is retrieved for pod %s, volume %s", param.ClientPodName, param.ClientPodVolume)
hostingPod, err := e.createHostingPod(ctx, ownerObject, param.Type, path.ByPath, param.OperationTimeout, param.HostingPodLabels, param.HostingPodAnnotations, pod.Spec.NodeName, param.Resources, nodeOS)
if err != nil {
return errors.Wrapf(err, "error to create hosting pod")
}
curLog.WithField("pod name", hostingPod.Name).Info("Hosting pod is created")
return nil
}
func (e *podVolumeExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, nodeClient client.Client, nodeName string, timeout time.Duration) (*ExposeResult, error) {
hostingPodName := ownerObject.Name
containerName := string(ownerObject.UID)
volumeName := string(ownerObject.UID)
curLog := e.log.WithFields(logrus.Fields{
"owner": ownerObject.Name,
"node": nodeName,
})
var updated *corev1api.Pod
err := wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
pod := &corev1api.Pod{}
err := nodeClient.Get(ctx, types.NamespacedName{
Namespace: ownerObject.Namespace,
Name: hostingPodName,
}, pod)
if err != nil {
return false, errors.Wrapf(err, "error to get pod %s/%s", ownerObject.Namespace, hostingPodName)
}
if pod.Status.Phase != corev1api.PodRunning {
return false, nil
}
updated = pod
return true, nil
})
if err != nil {
if apierrors.IsNotFound(err) {
curLog.WithField("hosting pod", hostingPodName).Debug("Hosting pod is not running in the current node")
return nil, nil
} else {
return nil, errors.Wrapf(err, "error to wait for rediness of pod %s", hostingPodName)
}
}
curLog.WithField("pod", updated.Name).Infof("Hosting pod is in running state in node %s", updated.Spec.NodeName)
return &ExposeResult{ByPod: ExposeByPod{
HostingPod: updated,
HostingContainer: containerName,
VolumeName: volumeName,
}}, nil
}
func (e *podVolumeExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error {
hostingPodName := ownerObject.Name
curLog := e.log.WithFields(logrus.Fields{
"owner": ownerObject.Name,
})
pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, hostingPodName, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
curLog.WithError(err).Warnf("error to peek hosting pod %s", hostingPodName)
return nil
}
if podFailed, message := kube.IsPodUnrecoverable(pod, curLog); podFailed {
return errors.New(message)
}
return nil
}
func (e *podVolumeExposer) DiagnoseExpose(ctx context.Context, ownerObject corev1api.ObjectReference) string {
hostingPodName := ownerObject.Name
diag := "begin diagnose pod volume exposer\n"
pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, hostingPodName, metav1.GetOptions{})
if err != nil {
pod = nil
diag += fmt.Sprintf("error getting hosting pod %s, err: %v\n", hostingPodName, err)
}
if pod != nil {
diag += kube.DiagnosePod(pod)
if pod.Spec.NodeName != "" {
if err := nodeagent.KbClientIsRunningInNode(ctx, ownerObject.Namespace, pod.Spec.NodeName, e.kubeClient); err != nil {
diag += fmt.Sprintf("node-agent is not running in node %s, err: %v\n", pod.Spec.NodeName, err)
}
}
}
diag += "end diagnose pod volume exposer"
return diag
}
func (e *podVolumeExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference) {
restorePodName := ownerObject.Name
kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log)
}
func (e *podVolumeExposer) createHostingPod(ctx context.Context, ownerObject corev1api.ObjectReference, exposeType string, hostPath string,
operationTimeout time.Duration, label map[string]string, annotation map[string]string, selectedNode string, resources corev1api.ResourceRequirements, nodeOS string) (*corev1api.Pod, error) {
hostingPodName := ownerObject.Name
containerName := string(ownerObject.UID)
clientVolumeName := string(ownerObject.UID)
clientVolumePath := "/" + clientVolumeName
podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS)
if err != nil {
return nil, errors.Wrap(err, "error to get inherited pod info from node-agent")
}
var gracePeriod int64
mountPropagation := corev1api.MountPropagationHostToContainer
volumeMounts := []corev1api.VolumeMount{{
Name: clientVolumeName,
MountPath: clientVolumePath,
MountPropagation: &mountPropagation,
}}
volumeMounts = append(volumeMounts, podInfo.volumeMounts...)
volumes := []corev1api.Volume{{
Name: clientVolumeName,
VolumeSource: corev1api.VolumeSource{
HostPath: &corev1api.HostPathVolumeSource{
Path: hostPath,
},
},
}}
volumes = append(volumes, podInfo.volumes...)
args := []string{
fmt.Sprintf("--volume-path=%s", clientVolumePath),
fmt.Sprintf("--resource-timeout=%s", operationTimeout.String()),
}
command := []string{
"/velero",
"pod-volume",
}
if exposeType == PodVolumeExposeTypeBackup {
args = append(args, fmt.Sprintf("--pod-volume-backup=%s", ownerObject.Name))
command = append(command, "backup")
} else {
args = append(args, fmt.Sprintf("--pod-volume-restore=%s", ownerObject.Name))
command = append(command, "restore")
}
args = append(args, podInfo.logFormatArgs...)
args = append(args, podInfo.logLevelArgs...)
var securityCtx *corev1api.PodSecurityContext
nodeSelector := map[string]string{}
podOS := corev1api.PodOS{}
toleration := []corev1api.Toleration{}
if nodeOS == kube.NodeOSWindows {
userID := "ContainerAdministrator"
securityCtx = &corev1api.PodSecurityContext{
WindowsOptions: &corev1api.WindowsSecurityContextOptions{
RunAsUserName: &userID,
},
}
nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows
podOS.Name = kube.NodeOSWindows
toleration = append(toleration, corev1api.Toleration{
Key: "os",
Operator: "Equal",
Effect: "NoSchedule",
Value: "windows",
})
} else {
userID := int64(0)
securityCtx = &corev1api.PodSecurityContext{
RunAsUser: &userID,
}
nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux
podOS.Name = kube.NodeOSLinux
}
pod := &corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: hostingPodName,
Namespace: ownerObject.Namespace,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: ownerObject.APIVersion,
Kind: ownerObject.Kind,
Name: ownerObject.Name,
UID: ownerObject.UID,
Controller: boolptr.True(),
},
},
Labels: label,
Annotations: annotation,
},
Spec: corev1api.PodSpec{
NodeSelector: nodeSelector,
OS: &podOS,
Containers: []corev1api.Container{
{
Name: containerName,
Image: podInfo.image,
ImagePullPolicy: corev1api.PullNever,
Command: command,
Args: args,
VolumeMounts: volumeMounts,
Env: podInfo.env,
EnvFrom: podInfo.envFrom,
Resources: resources,
},
},
ServiceAccountName: podInfo.serviceAccount,
TerminationGracePeriodSeconds: &gracePeriod,
Volumes: volumes,
NodeName: selectedNode,
RestartPolicy: corev1api.RestartPolicyNever,
SecurityContext: securityCtx,
Tolerations: toleration,
},
}
return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{})
}
pkg/exposer/pod_volume_test.go
+591
View File
@@ -0,0 +1,591 @@
package exposer
import (
"context"
"errors"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/datapath"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
)
func TestPodVolumeExpose(t *testing.T) {
backup := &velerov1.Backup{
TypeMeta: metav1.TypeMeta{
APIVersion: velerov1.SchemeGroupVersion.String(),
Kind: "Backup",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
UID: "fake-uid",
},
}
podWithNoNode := builder.ForPod("fake-ns", "fake-client-pod").Result()
podWithNode := builder.ForPod("fake-ns", "fake-client-pod").NodeName("fake-node").Result()
node := builder.ForNode("fake-node").Result()
daemonSet := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "velero",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
APIVersion: appsv1api.SchemeGroupVersion.String(),
},
Spec: appsv1api.DaemonSetSpec{
Template: corev1api.PodTemplateSpec{
Spec: corev1api.PodSpec{
Containers: []corev1api.Container{
{
Name: "node-agent",
},
},
},
},
},
}
tests := []struct {
name string
snapshotClientObj []runtime.Object
kubeClientObj []runtime.Object
ownerBackup *velerov1.Backup
exposeParam PodVolumeExposeParam
funcGetPodVolumeHostPath func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error)
funcExtractPodVolumeHostPath func(context.Context, string, kubernetes.Interface, string, string) (string, error)
err string
}{
{
name: "get client pod fail",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
},
err: "error getting client pod fake-client-pod: pods \"fake-client-pod\" not found",
},
{
name: "client pod with no node name",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
},
kubeClientObj: []runtime.Object{
podWithNoNode,
},
err: "client pod fake-client-pod doesn't have a node name",
},
{
name: "get node os fail",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
},
kubeClientObj: []runtime.Object{
podWithNode,
},
err: "error getting OS for node fake-node: error getting node fake-node: nodes \"fake-node\" not found",
},
{
name: "get pod volume path fail",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
ClientPodVolume: "fake-client-volume",
},
kubeClientObj: []runtime.Object{
podWithNode,
node,
},
funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) {
return datapath.AccessPoint{}, errors.New("fake-get-pod-volume-path-error")
},
err: "error to get pod volume path: fake-get-pod-volume-path-error",
},
{
name: "extract pod volume path fail",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
ClientPodVolume: "fake-client-volume",
},
kubeClientObj: []runtime.Object{
podWithNode,
node,
},
funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) {
return datapath.AccessPoint{
ByPath: "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount",
}, nil
},
funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) {
return "", errors.New("fake-extract-error")
},
err: "error to extract pod volume path: fake-extract-error",
},
{
name: "create hosting pod fail",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
ClientPodVolume: "fake-client-volume",
},
kubeClientObj: []runtime.Object{
podWithNode,
node,
},
funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) {
return datapath.AccessPoint{
ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount",
}, nil
},
funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) {
return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil
},
err: "error to create hosting pod: error to get inherited pod info from node-agent: error to get node-agent pod template: error to get node-agent daemonset: daemonsets.apps \"node-agent\" not found",
},
{
name: "succeed",
ownerBackup: backup,
exposeParam: PodVolumeExposeParam{
ClientNamespace: "fake-ns",
ClientPodName: "fake-client-pod",
ClientPodVolume: "fake-client-volume",
},
kubeClientObj: []runtime.Object{
podWithNode,
node,
daemonSet,
},
funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) {
return datapath.AccessPoint{
ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount",
}, nil
},
funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) {
return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
exposer := podVolumeExposer{
kubeClient: fakeKubeClient,
log: velerotest.NewLogger(),
}
var ownerObject corev1api.ObjectReference
if test.ownerBackup != nil {
ownerObject = corev1api.ObjectReference{
Kind: test.ownerBackup.Kind,
Namespace: test.ownerBackup.Namespace,
Name: test.ownerBackup.Name,
UID: test.ownerBackup.UID,
APIVersion: test.ownerBackup.APIVersion,
}
}
if test.funcGetPodVolumeHostPath != nil {
getPodVolumeHostPath = test.funcGetPodVolumeHostPath
}
if test.funcExtractPodVolumeHostPath != nil {
extractPodVolumeHostPath = test.funcExtractPodVolumeHostPath
}
err := exposer.Expose(context.Background(), ownerObject, test.exposeParam)
if err == nil {
assert.NoError(t, err)
_, err = exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(context.Background(), ownerObject.Name, metav1.GetOptions{})
assert.NoError(t, err)
} else {
assert.EqualError(t, err, test.err)
}
})
}
}
func TestGetPodVolumeExpose(t *testing.T) {
backup := &velerov1.Backup{
TypeMeta: metav1.TypeMeta{
APIVersion: velerov1.SchemeGroupVersion.String(),
Kind: "Backup",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
UID: "fake-uid",
},
}
backupPodNotRunning := builder.ForPod(backup.Namespace, backup.Name).Result()
backupPodRunning := builder.ForPod(backup.Namespace, backup.Name).Phase(corev1api.PodRunning).Result()
scheme := runtime.NewScheme()
corev1api.AddToScheme(scheme)
tests := []struct {
name string
kubeClientObj []runtime.Object
ownerBackup *velerov1.Backup
nodeName string
Timeout time.Duration
err string
expectedResult *ExposeResult
}{
{
name: "backup pod is not found",
ownerBackup: backup,
nodeName: "fake-node",
},
{
name: "wait backup pod running fail",
ownerBackup: backup,
nodeName: "fake-node",
kubeClientObj: []runtime.Object{
backupPodNotRunning,
},
Timeout: time.Second,
err: "error to wait for rediness of pod fake-backup: context deadline exceeded",
},
{
name: "succeed",
ownerBackup: backup,
nodeName: "fake-node",
kubeClientObj: []runtime.Object{
backupPodRunning,
},
Timeout: time.Second,
expectedResult: &ExposeResult{
ByPod: ExposeByPod{
HostingPod: backupPodRunning,
VolumeName: string(backup.UID),
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
fakeClientBuilder := clientFake.NewClientBuilder()
fakeClientBuilder = fakeClientBuilder.WithScheme(scheme)
fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
exposer := podVolumeExposer{
kubeClient: fakeKubeClient,
log: velerotest.NewLogger(),
}
var ownerObject corev1api.ObjectReference
if test.ownerBackup != nil {
ownerObject = corev1api.ObjectReference{
Kind: test.ownerBackup.Kind,
Namespace: test.ownerBackup.Namespace,
Name: test.ownerBackup.Name,
UID: test.ownerBackup.UID,
APIVersion: test.ownerBackup.APIVersion,
}
}
result, err := exposer.GetExposed(context.Background(), ownerObject, fakeClient, test.nodeName, test.Timeout)
if test.err == "" {
assert.NoError(t, err)
if test.expectedResult == nil {
assert.Nil(t, result)
} else {
assert.NoError(t, err)
assert.Equal(t, test.expectedResult.ByPod.VolumeName, result.ByPod.VolumeName)
assert.Equal(t, test.expectedResult.ByPod.HostingPod.Name, result.ByPod.HostingPod.Name)
}
} else {
assert.EqualError(t, err, test.err)
}
})
}
}
func TestPodVolumePeekExpose(t *testing.T) {
backup := &velerov1.Backup{
TypeMeta: metav1.TypeMeta{
APIVersion: velerov1.SchemeGroupVersion.String(),
Kind: "Backup",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
UID: "fake-uid",
},
}
backupPodUrecoverable := &corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: backup.Namespace,
Name: backup.Name,
},
Status: corev1api.PodStatus{
Phase: corev1api.PodFailed,
},
}
backupPod := &corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: backup.Namespace,
Name: backup.Name,
},
}
scheme := runtime.NewScheme()
corev1api.AddToScheme(scheme)
tests := []struct {
name string
kubeClientObj []runtime.Object
ownerBackup *velerov1.Backup
err string
}{
{
name: "backup pod is not found",
ownerBackup: backup,
},
{
name: "pod is unrecoverable",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
backupPodUrecoverable,
},
err: "Pod is in abnormal state [Failed], message []",
},
{
name: "succeed",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
backupPod,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
exposer := podVolumeExposer{
kubeClient: fakeKubeClient,
log: velerotest.NewLogger(),
}
var ownerObject corev1api.ObjectReference
if test.ownerBackup != nil {
ownerObject = corev1api.ObjectReference{
Kind: test.ownerBackup.Kind,
Namespace: test.ownerBackup.Namespace,
Name: test.ownerBackup.Name,
UID: test.ownerBackup.UID,
APIVersion: test.ownerBackup.APIVersion,
}
}
err := exposer.PeekExposed(context.Background(), ownerObject)
if test.err == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, test.err)
}
})
}
}
func TestPodVolumeDiagnoseExpose(t *testing.T) {
backup := &velerov1.Backup{
TypeMeta: metav1.TypeMeta{
APIVersion: velerov1.SchemeGroupVersion.String(),
Kind: "Backup",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
UID: "fake-uid",
},
}
backupPodWithoutNodeName := corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: backup.APIVersion,
Kind: backup.Kind,
Name: backup.Name,
UID: backup.UID,
},
},
},
Status: corev1api.PodStatus{
Phase: corev1api.PodPending,
Conditions: []corev1api.PodCondition{
{
Type: corev1api.PodInitialized,
Status: corev1api.ConditionTrue,
Message: "fake-pod-message",
},
},
},
}
backupPodWithNodeName := corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "fake-backup",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: backup.APIVersion,
Kind: backup.Kind,
Name: backup.Name,
UID: backup.UID,
},
},
},
Spec: corev1api.PodSpec{
NodeName: "fake-node",
},
Status: corev1api.PodStatus{
Phase: corev1api.PodPending,
Conditions: []corev1api.PodCondition{
{
Type: corev1api.PodInitialized,
Status: corev1api.ConditionTrue,
Message: "fake-pod-message",
},
},
},
}
nodeAgentPod := corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1.DefaultNamespace,
Name: "node-agent-pod-1",
Labels: map[string]string{"role": "node-agent"},
},
Spec: corev1api.PodSpec{
NodeName: "fake-node",
},
Status: corev1api.PodStatus{
Phase: corev1api.PodRunning,
},
}
tests := []struct {
name string
ownerBackup *velerov1.Backup
kubeClientObj []runtime.Object
snapshotClientObj []runtime.Object
expected string
}{
{
name: "no pod",
ownerBackup: backup,
expected: `begin diagnose pod volume exposer
error getting hosting pod fake-backup, err: pods "fake-backup" not found
end diagnose pod volume exposer`,
},
{
name: "pod without node name, pvc without volume name, vs without status",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
&backupPodWithoutNodeName,
},
expected: `begin diagnose pod volume exposer
Pod velero/fake-backup, phase Pending, node name
Pod condition Initialized, status True, reason , message fake-pod-message
end diagnose pod volume exposer`,
},
{
name: "pod without node name",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
&backupPodWithoutNodeName,
},
expected: `begin diagnose pod volume exposer
Pod velero/fake-backup, phase Pending, node name
Pod condition Initialized, status True, reason , message fake-pod-message
end diagnose pod volume exposer`,
},
{
name: "pod with node name, no node agent",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
&backupPodWithNodeName,
},
expected: `begin diagnose pod volume exposer
Pod velero/fake-backup, phase Pending, node name fake-node
Pod condition Initialized, status True, reason , message fake-pod-message
node-agent is not running in node fake-node, err: daemonset pod not found in running state in node fake-node
end diagnose pod volume exposer`,
},
{
name: "pod with node name, node agent is running",
ownerBackup: backup,
kubeClientObj: []runtime.Object{
&backupPodWithNodeName,
&nodeAgentPod,
},
expected: `begin diagnose pod volume exposer
Pod velero/fake-backup, phase Pending, node name fake-node
Pod condition Initialized, status True, reason , message fake-pod-message
end diagnose pod volume exposer`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(tt.kubeClientObj...)
e := &podVolumeExposer{
kubeClient: fakeKubeClient,
log: velerotest.NewLogger(),
}
var ownerObject corev1api.ObjectReference
if tt.ownerBackup != nil {
ownerObject = corev1api.ObjectReference{
Kind: tt.ownerBackup.Kind,
Namespace: tt.ownerBackup.Namespace,
Name: tt.ownerBackup.Name,
UID: tt.ownerBackup.UID,
APIVersion: tt.ownerBackup.APIVersion,
}
}
diag := e.DiagnoseExpose(context.Background(), ownerObject)
assert.Equal(t, tt.expected, diag)
})
}
}
pkg/install/daemonset.go
+3 -2
View File
@@ -25,6 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/vmware-tanzu/velero/internal/velero"
"github.com/vmware-tanzu/velero/pkg/nodeagent"
)
func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet {
@@ -126,8 +127,8 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet
},
VolumeMounts: []corev1api.VolumeMount{
{
Name: "host-pods",
MountPath: "/host_pods",
Name: nodeagent.HostPodVolumeMount,
MountPath: nodeagent.HostPodVolumeMountPath(),
MountPropagation: &mountPropagationMode,
},
{
pkg/itemblock/actions/pvc_action.go
+59 -1
View File
@@ -105,9 +105,67 @@ func (a *PVCAction) GetRelatedItems(item runtime.Unstructured, backup *v1.Backup
}
}
// Gather groupedPVCs based on VGS label provided in the backup
groupedPVCs, err := a.getGroupedPVCs(context.Background(), pvc, backup)
if err != nil {
return nil, err
}
// Add the groupedPVCs to relatedItems so that they processed in a single item block
relatedItems = append(relatedItems, groupedPVCs...)
return relatedItems, nil
}
func (a *PVCAction) Name() string {
return "PodItemBlockAction"
return "PVCItemBlockAction"
}
// getGroupedPVCs returns other PVCs in the same group based on the VGS label key in the Backup spec.
func (a *PVCAction) getGroupedPVCs(ctx context.Context, pvc *corev1api.PersistentVolumeClaim, backup *v1.Backup) ([]velero.ResourceIdentifier, error) {
var related []velero.ResourceIdentifier
vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey
if vgsLabelKey == "" {
a.log.Debug("No VolumeGroupSnapshotLabelKey provided in backup spec; skipping PVC grouping")
return nil, nil
}
groupID, ok := pvc.Labels[vgsLabelKey]
if !ok || groupID == "" {
// PVC does not belong to any VGS group or groupID has empty value
a.log.Debug("PVC does not belong to any PVC group or group label value is empty; skipping PVC grouping")
return nil, nil
}
pvcList := new(corev1api.PersistentVolumeClaimList)
if err := a.crClient.List(
ctx,
pvcList,
crclient.InNamespace(pvc.Namespace),
crclient.MatchingLabels{vgsLabelKey: groupID},
); err != nil {
return nil, errors.Wrapf(err, "failed to list PVCs for VGS grouping with label %s=%s in namespace %s", vgsLabelKey, groupID, pvc.Namespace)
}
if len(pvcList.Items) <= 1 {
// Only the current PVC exists in this group
return nil, nil
}
for _, groupPVC := range pvcList.Items {
if groupPVC.Name == pvc.Name {
continue
}
a.log.Infof("Adding grouped PVC %s (group %s) to relatedItems for PVC %s", groupPVC.Name, groupID, pvc.Name)
related = append(related, velero.ResourceIdentifier{
GroupResource: kuberesource.PersistentVolumeClaims,
Namespace: groupPVC.Namespace,
Name: groupPVC.Name,
})
}
return related, nil
}
pkg/itemblock/actions/pvc_action_test.go
+109
View File
@@ -124,6 +124,22 @@ func TestBackupPVAction(t *testing.T) {
{GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod2"},
},
},
{
name: "Test with PVC grouping via VGS label",
pvc: builder.ForPersistentVolumeClaim("velero", "testPVC-1").ObjectMeta(builder.WithLabels("velero.io/group", "db")).VolumeName("testPV-1").Phase(corev1api.ClaimBound).Result(),
pods: []*corev1api.Pod{
builder.ForPod("velero", "testPod-1").
Volumes(builder.ForVolume("testPV-1").PersistentVolumeClaimSource("testPVC-1").Result()).
NodeName("node").
Phase(corev1api.PodRunning).Result(),
},
expectedErr: nil,
expectedRelated: []velero.ResourceIdentifier{
{GroupResource: kuberesource.PersistentVolumes, Name: "testPV-1"},
{GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod-1"},
{GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "velero", Name: "groupedPVC"},
},
},
}
backup := &v1.Backup{}
@@ -152,6 +168,12 @@ func TestBackupPVAction(t *testing.T) {
require.NoError(t, crClient.Create(context.Background(), pod))
}
if tc.name == "Test with PVC grouping via VGS label" {
groupedPVC := builder.ForPersistentVolumeClaim("velero", "groupedPVC").ObjectMeta(builder.WithLabels("velero.io/group", "db")).VolumeName("groupedPV").Phase(corev1api.ClaimBound).Result()
require.NoError(t, crClient.Create(context.Background(), groupedPVC))
backup.Spec.VolumeGroupSnapshotLabelKey = "velero.io/group"
}
pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc)
require.NoError(t, err)
@@ -165,3 +187,90 @@ func TestBackupPVAction(t *testing.T) {
})
}
}
// Test_getGroupedPVCs verifies the PVC grouping logic for VolumeGroupSnapshots.
// This ensures only same-namespace PVCs with the same label key and value are included.
func Test_getGroupedPVCs(t *testing.T) {
tests := []struct {
name string
labelKey string
groupValue string
existingPVCs []*corev1api.PersistentVolumeClaim
targetPVC *corev1api.PersistentVolumeClaim
expectedRelated []velero.ResourceIdentifier
expectError bool
}{
{
name: "No label key in spec",
labelKey: "",
targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(),
expectError: false,
},
{
name: "No group value",
labelKey: "velero.io/group",
groupValue: "",
targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(),
expectError: false,
},
{
name: "Target PVC does not have the label",
labelKey: "velero.io/group",
targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(),
expectError: false,
},
{
name: "Target PVC has label, but no group matches",
labelKey: "velero.io/group",
groupValue: "group-1",
targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
existingPVCs: []*corev1api.PersistentVolumeClaim{
builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
},
expectError: false,
expectedRelated: nil,
},
{
name: "Multiple PVCs in the same group",
labelKey: "velero.io/group",
groupValue: "group-1",
targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
existingPVCs: []*corev1api.PersistentVolumeClaim{
builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
builder.ForPersistentVolumeClaim("ns", "pvc-2").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
builder.ForPersistentVolumeClaim("ns", "pvc-3").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(),
},
expectError: false,
expectedRelated: []velero.ResourceIdentifier{
{GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns", Name: "pvc-2"},
{GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns", Name: "pvc-3"},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
crClient := velerotest.NewFakeControllerRuntimeClient(t)
for _, pvc := range tc.existingPVCs {
require.NoError(t, crClient.Create(context.Background(), pvc))
}
logger := logrus.New()
a := &PVCAction{
log: logger,
crClient: crClient,
}
backup := builder.ForBackup("ns", "bkp").VolumeGroupSnapshotLabelKey(tc.labelKey).Result()
related, err := a.getGroupedPVCs(context.Background(), tc.targetPVC, backup)
if tc.expectError {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.ElementsMatch(t, tc.expectedRelated, related)
})
}
}
pkg/nodeagent/node_agent.go
+48
View File
@@ -41,6 +41,12 @@ const (
// nodeAgentRole marks pods with node-agent role on all nodes.
nodeAgentRole = "node-agent"
// HostPodVolumeMount is the name of the volume in node-agent for host-pod mount
HostPodVolumeMount = "host-pods"
// HostPodVolumeMountPoint is the mount point of the volume in node-agent for host-pod mount
HostPodVolumeMountPoint = "host_pods"
)
var (
@@ -249,3 +255,45 @@ func GetAnnotationValue(ctx context.Context, kubeClient kubernetes.Interface, na
return val, nil
}
func GetHostPodPath(ctx context.Context, kubeClient kubernetes.Interface, namespace string, osType string) (string, error) {
dsName := daemonSet
if osType == kube.NodeOSWindows {
dsName = daemonsetWindows
}
ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{})
if err != nil {
return "", errors.Wrapf(err, "error getting daemonset %s", dsName)
}
var volume *corev1api.Volume
for _, v := range ds.Spec.Template.Spec.Volumes {
if v.Name == HostPodVolumeMount {
volume = &v
break
}
}
if volume == nil {
return "", errors.New("host pod volume is not found")
}
if volume.HostPath == nil {
return "", errors.New("host pod volume is not a host path volume")
}
if volume.HostPath.Path == "" {
return "", errors.New("host pod volume path is empty")
}
return volume.HostPath.Path, nil
}
func HostPodVolumeMountPath() string {
return "/" + HostPodVolumeMountPoint
}
func HostPodVolumeMountPathWin() string {
return "\\" + HostPodVolumeMountPoint
}
pkg/nodeagent/node_agent_test.go
+161
View File
@@ -590,3 +590,164 @@ func TestGetAnnotationValue(t *testing.T) {
})
}
}
func TestGetHostPodPath(t *testing.T) {
daemonSet := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-ns",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
}
daemonSetWithHostPodVolume := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-ns",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
Spec: appsv1api.DaemonSetSpec{
Template: corev1api.PodTemplateSpec{
Spec: corev1api.PodSpec{
Volumes: []corev1api.Volume{
{
Name: HostPodVolumeMount,
},
},
},
},
},
}
daemonSetWithHostPodVolumeAndEmptyPath := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-ns",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
Spec: appsv1api.DaemonSetSpec{
Template: corev1api.PodTemplateSpec{
Spec: corev1api.PodSpec{
Volumes: []corev1api.Volume{
{
Name: HostPodVolumeMount,
VolumeSource: corev1api.VolumeSource{
HostPath: &corev1api.HostPathVolumeSource{},
},
},
},
},
},
},
}
daemonSetWithHostPodVolumeAndValidPath := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-ns",
Name: "node-agent",
},
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
Spec: appsv1api.DaemonSetSpec{
Template: corev1api.PodTemplateSpec{
Spec: corev1api.PodSpec{
Volumes: []corev1api.Volume{
{
Name: HostPodVolumeMount,
VolumeSource: corev1api.VolumeSource{
HostPath: &corev1api.HostPathVolumeSource{
Path: "/var/lib/kubelet/pods",
},
},
},
},
},
},
},
}
tests := []struct {
name string
kubeClientObj []runtime.Object
namespace string
osType string
expectedValue string
expectErr string
}{
{
name: "ds get error",
namespace: "fake-ns",
osType: kube.NodeOSWindows,
kubeClientObj: []runtime.Object{
daemonSet,
},
expectErr: "error getting daemonset node-agent-windows: daemonsets.apps \"node-agent-windows\" not found",
},
{
name: "no host pod volume",
namespace: "fake-ns",
osType: kube.NodeOSLinux,
kubeClientObj: []runtime.Object{
daemonSet,
},
expectErr: "host pod volume is not found",
},
{
name: "no host pod volume path",
namespace: "fake-ns",
osType: kube.NodeOSLinux,
kubeClientObj: []runtime.Object{
daemonSetWithHostPodVolume,
},
expectErr: "host pod volume is not a host path volume",
},
{
name: "empty host pod volume path",
namespace: "fake-ns",
osType: kube.NodeOSLinux,
kubeClientObj: []runtime.Object{
daemonSetWithHostPodVolumeAndEmptyPath,
},
expectErr: "host pod volume path is empty",
},
{
name: "succeed",
namespace: "fake-ns",
osType: kube.NodeOSLinux,
kubeClientObj: []runtime.Object{
daemonSetWithHostPodVolumeAndValidPath,
},
expectedValue: "/var/lib/kubelet/pods",
},
{
name: "succeed on empty os type",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
daemonSetWithHostPodVolumeAndValidPath,
},
expectedValue: "/var/lib/kubelet/pods",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
path, err := GetHostPodPath(context.TODO(), fakeKubeClient, test.namespace, test.osType)
if test.expectErr == "" {
assert.NoError(t, err)
assert.Equal(t, test.expectedValue, path)
} else {
assert.EqualError(t, err, test.expectErr)
}
})
}
}
pkg/podvolume/backup_micro_service.go
+313
View File
@@ -0,0 +1,313 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"encoding/json"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
cachetool "k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/cache"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/kube"
apierrors "k8s.io/apimachinery/pkg/api/errors"
)
const (
podVolumeRequestor = "snapshot-pod-volume"
)
// BackupMicroService process data mover backups inside the backup pod
type BackupMicroService struct {
ctx context.Context
client client.Client
kubeClient kubernetes.Interface
repoEnsurer *repository.Ensurer
credentialGetter *credentials.CredentialGetter
logger logrus.FieldLogger
dataPathMgr *datapath.Manager
eventRecorder kube.EventRecorder
namespace string
pvbName string
pvb *velerov1api.PodVolumeBackup
sourceTargetPath datapath.AccessPoint
resultSignal chan dataPathResult
pvbInformer cache.Informer
pvbHandler cachetool.ResourceEventHandlerRegistration
nodeName string
}
type dataPathResult struct {
err error
result string
}
func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, pvbName string, namespace string, nodeName string,
sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter,
pvbInformer cache.Informer, log logrus.FieldLogger) *BackupMicroService {
return &BackupMicroService{
ctx: ctx,
client: client,
kubeClient: kubeClient,
credentialGetter: cred,
logger: log,
repoEnsurer: repoEnsurer,
dataPathMgr: dataPathMgr,
namespace: namespace,
pvbName: pvbName,
sourceTargetPath: sourceTargetPath,
nodeName: nodeName,
resultSignal: make(chan dataPathResult),
pvbInformer: pvbInformer,
}
}
func (r *BackupMicroService) Init() error {
r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.pvbName, r.nodeName, r.logger)
handler, err := r.pvbInformer.AddEventHandler(
cachetool.ResourceEventHandlerFuncs{
UpdateFunc: func(oldObj any, newObj any) {
oldPvb := oldObj.(*velerov1api.PodVolumeBackup)
newPvb := newObj.(*velerov1api.PodVolumeBackup)
if newPvb.Name != r.pvbName {
return
}
if newPvb.Status.Phase != velerov1api.PodVolumeBackupPhaseInProgress {
return
}
if newPvb.Spec.Cancel && !oldPvb.Spec.Cancel {
r.cancelPodVolumeBackup(newPvb)
}
},
},
)
if err != nil {
return errors.Wrap(err, "error adding PVB handler")
}
r.pvbHandler = handler
return err
}
func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, error) {
log := r.logger.WithFields(logrus.Fields{
"PVB": r.pvbName,
})
pvb := &velerov1api.PodVolumeBackup{}
err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) {
err := r.client.Get(ctx, types.NamespacedName{
Namespace: r.namespace,
Name: r.pvbName,
}, pvb)
if apierrors.IsNotFound(err) {
return false, nil
}
if err != nil {
return true, errors.Wrapf(err, "error to get PVB %s", r.pvbName)
}
if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress {
return true, nil
} else {
return false, nil
}
})
if err != nil {
log.WithError(err).Error("Failed to wait PVB")
return "", errors.Wrap(err, "error waiting for PVB")
}
r.pvb = pvb
log.Info("Run cancelable PVB")
callbacks := datapath.Callbacks{
OnCompleted: r.OnDataPathCompleted,
OnFailed: r.OnDataPathFailed,
OnCancelled: r.OnDataPathCancelled,
OnProgress: r.OnDataPathProgress,
}
fsBackup, err := r.dataPathMgr.CreateFileSystemBR(pvb.Name, podVolumeRequestor, ctx, r.client, pvb.Namespace, callbacks, log)
if err != nil {
return "", errors.Wrap(err, "error to create data path")
}
log.Debug("Async fs br created")
if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{
BSLName: pvb.Spec.BackupStorageLocation,
SourceNamespace: pvb.Spec.Pod.Namespace,
UploaderType: pvb.Spec.UploaderType,
RepositoryType: velerov1api.BackupRepositoryTypeKopia,
RepoIdentifier: "",
RepositoryEnsurer: r.repoEnsurer,
CredentialGetter: r.credentialGetter,
}); err != nil {
return "", errors.Wrap(err, "error to initialize data path")
}
log.Info("Async fs br init")
tags := map[string]string{}
if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.FSBRStartParam{
RealSource: GetRealSource(pvb),
ParentSnapshot: "",
ForceFull: false,
Tags: tags,
}); err != nil {
return "", errors.Wrap(err, "error starting data path backup")
}
log.Info("Async fs backup data path started")
r.eventRecorder.Event(pvb, false, datapath.EventReasonStarted, "Data path for %s started", pvb.Name)
result := ""
select {
case <-ctx.Done():
err = errors.New("timed out waiting for fs backup to complete")
break
case res := <-r.resultSignal:
err = res.err
result = res.result
break
}
if err != nil {
log.WithError(err).Error("Async fs backup was not completed")
}
r.eventRecorder.EndingEvent(pvb, false, datapath.EventReasonStopped, "Data path for %s stopped", pvb.Name)
return result, err
}
func (r *BackupMicroService) Shutdown() {
r.eventRecorder.Shutdown()
r.closeDataPath(r.ctx, r.pvbName)
if r.pvbHandler != nil {
if err := r.pvbInformer.RemoveEventHandler(r.pvbHandler); err != nil {
r.logger.WithError(err).Warn("Failed to remove pod handler")
}
}
}
var funcMarshal = json.Marshal
func (r *BackupMicroService) OnDataPathCompleted(ctx context.Context, namespace string, pvbName string, result datapath.Result) {
log := r.logger.WithField("PVB", pvbName)
backupBytes, err := funcMarshal(result.Backup)
if err != nil {
log.WithError(err).Errorf("Failed to marshal backup result %v", result.Backup)
r.resultSignal <- dataPathResult{
err: errors.Wrapf(err, "Failed to marshal backup result %v", result.Backup),
}
} else {
r.eventRecorder.Event(r.pvb, false, datapath.EventReasonCompleted, string(backupBytes))
r.resultSignal <- dataPathResult{
result: string(backupBytes),
}
}
log.Info("Async fs backup completed")
}
func (r *BackupMicroService) OnDataPathFailed(ctx context.Context, namespace string, pvbName string, err error) {
log := r.logger.WithField("PVB", pvbName)
log.WithError(err).Error("Async fs backup data path failed")
r.eventRecorder.Event(r.pvb, false, datapath.EventReasonFailed, "Data path for PVB %s failed, error %v", r.pvbName, err)
r.resultSignal <- dataPathResult{
err: errors.Wrapf(err, "Data path for PVB %s failed", r.pvbName),
}
}
func (r *BackupMicroService) OnDataPathCancelled(ctx context.Context, namespace string, pvbName string) {
log := r.logger.WithField("PVB", pvbName)
log.Warn("Async fs backup data path canceled")
r.eventRecorder.Event(r.pvb, false, datapath.EventReasonCancelled, "Data path for PVB %s canceled", pvbName)
r.resultSignal <- dataPathResult{
err: errors.New(datapath.ErrCancelled),
}
}
func (r *BackupMicroService) OnDataPathProgress(ctx context.Context, namespace string, pvbName string, progress *uploader.Progress) {
log := r.logger.WithFields(logrus.Fields{
"PVB": pvbName,
})
progressBytes, err := funcMarshal(progress)
if err != nil {
log.WithError(err).Errorf("Failed to marshal progress %v", progress)
return
}
r.eventRecorder.Event(r.pvb, false, datapath.EventReasonProgress, string(progressBytes))
}
func (r *BackupMicroService) closeDataPath(ctx context.Context, duName string) {
fsBackup := r.dataPathMgr.GetAsyncBR(duName)
if fsBackup != nil {
fsBackup.Close(ctx)
}
r.dataPathMgr.RemoveAsyncBR(duName)
}
func (r *BackupMicroService) cancelPodVolumeBackup(pvb *velerov1api.PodVolumeBackup) {
r.logger.WithField("PVB", pvb.Name).Info("PVB is being canceled")
r.eventRecorder.Event(pvb, false, datapath.EventReasonCancelling, "Canceling for PVB %s", pvb.Name)
fsBackup := r.dataPathMgr.GetAsyncBR(pvb.Name)
if fsBackup == nil {
r.OnDataPathCancelled(r.ctx, pvb.GetNamespace(), pvb.GetName())
} else {
fsBackup.Cancel()
}
}
pkg/podvolume/backup_micro_service_test.go
+447
View File
@@ -0,0 +1,447 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"fmt"
"sync"
"testing"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"k8s.io/apimachinery/pkg/runtime"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/uploader"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks"
)
type backupMsTestHelper struct {
eventReason string
eventMsg string
marshalErr error
marshalBytes []byte
withEvent bool
eventLock sync.Mutex
}
func (bt *backupMsTestHelper) Event(_ runtime.Object, _ bool, reason string, message string, a ...any) {
bt.eventLock.Lock()
defer bt.eventLock.Unlock()
bt.withEvent = true
bt.eventReason = reason
bt.eventMsg = fmt.Sprintf(message, a...)
}
func (bt *backupMsTestHelper) EndingEvent(_ runtime.Object, _ bool, reason string, message string, a ...any) {
bt.eventLock.Lock()
defer bt.eventLock.Unlock()
bt.withEvent = true
bt.eventReason = reason
bt.eventMsg = fmt.Sprintf(message, a...)
}
func (bt *backupMsTestHelper) Shutdown() {}
func (bt *backupMsTestHelper) Marshal(v any) ([]byte, error) {
if bt.marshalErr != nil {
return nil, bt.marshalErr
}
return bt.marshalBytes, nil
}
func (bt *backupMsTestHelper) EventReason() string {
bt.eventLock.Lock()
defer bt.eventLock.Unlock()
return bt.eventReason
}
func (bt *backupMsTestHelper) EventMessage() string {
bt.eventLock.Lock()
defer bt.eventLock.Unlock()
return bt.eventMsg
}
func TestOnDataPathFailed(t *testing.T) {
pvbName := "fake-pvb"
bt := &backupMsTestHelper{}
bs := &BackupMicroService{
pvbName: pvbName,
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
expectedErr := "Data path for PVB fake-pvb failed: fake-error"
expectedEventReason := datapath.EventReasonFailed
expectedEventMsg := "Data path for PVB fake-pvb failed, error fake-error"
go bs.OnDataPathFailed(context.TODO(), velerov1api.DefaultNamespace, pvbName, errors.New("fake-error"))
result := <-bs.resultSignal
assert.EqualError(t, result.err, expectedErr)
assert.Equal(t, expectedEventReason, bt.EventReason())
assert.Equal(t, expectedEventMsg, bt.EventMessage())
}
func TestOnDataPathCancelled(t *testing.T) {
pvbName := "fake-pvb"
bt := &backupMsTestHelper{}
bs := &BackupMicroService{
pvbName: pvbName,
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
expectedErr := datapath.ErrCancelled
expectedEventReason := datapath.EventReasonCancelled
expectedEventMsg := "Data path for PVB fake-pvb canceled"
go bs.OnDataPathCancelled(context.TODO(), velerov1api.DefaultNamespace, pvbName)
result := <-bs.resultSignal
assert.EqualError(t, result.err, expectedErr)
assert.Equal(t, expectedEventReason, bt.EventReason())
assert.Equal(t, expectedEventMsg, bt.EventMessage())
}
func TestOnDataPathCompleted(t *testing.T) {
tests := []struct {
name string
expectedErr string
expectedEventReason string
expectedEventMsg string
marshalErr error
marshallStr string
}{
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result { false { } 0}: fake-marshal-error",
},
{
name: "succeed",
marshallStr: "fake-complete-string",
expectedEventReason: datapath.EventReasonCompleted,
expectedEventMsg: "fake-complete-string",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvbName := "fake-pvb"
bt := &backupMsTestHelper{
marshalErr: test.marshalErr,
marshalBytes: []byte(test.marshallStr),
}
bs := &BackupMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
funcMarshal = bt.Marshal
go bs.OnDataPathCompleted(context.TODO(), velerov1api.DefaultNamespace, pvbName, datapath.Result{})
result := <-bs.resultSignal
if test.marshalErr != nil {
assert.EqualError(t, result.err, test.expectedErr)
} else {
assert.NoError(t, result.err)
assert.Equal(t, test.expectedEventReason, bt.EventReason())
assert.Equal(t, test.expectedEventMsg, bt.EventMessage())
}
})
}
}
func TestOnDataPathProgress(t *testing.T) {
tests := []struct {
name string
expectedErr string
expectedEventReason string
expectedEventMsg string
marshalErr error
marshallStr string
}{
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result",
},
{
name: "succeed",
marshallStr: "fake-progress-string",
expectedEventReason: datapath.EventReasonProgress,
expectedEventMsg: "fake-progress-string",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvbName := "fake-pvb"
bt := &backupMsTestHelper{
marshalErr: test.marshalErr,
marshalBytes: []byte(test.marshallStr),
}
bs := &BackupMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
logger: velerotest.NewLogger(),
}
funcMarshal = bt.Marshal
bs.OnDataPathProgress(context.TODO(), velerov1api.DefaultNamespace, pvbName, &uploader.Progress{})
if test.marshalErr != nil {
assert.False(t, bt.withEvent)
} else {
assert.True(t, bt.withEvent)
assert.Equal(t, test.expectedEventReason, bt.EventReason())
assert.Equal(t, test.expectedEventMsg, bt.EventMessage())
}
})
}
}
func TestCancelPodVolumeBackup(t *testing.T) {
tests := []struct {
name string
expectedEventReason string
expectedEventMsg string
expectedErr string
}{
{
name: "no fs backup",
expectedEventReason: datapath.EventReasonCancelled,
expectedEventMsg: "Data path for PVB fake-pvb canceled",
expectedErr: datapath.ErrCancelled,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvbName := "fake-pvb"
pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Result()
bt := &backupMsTestHelper{}
bs := &BackupMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
go bs.cancelPodVolumeBackup(pvb)
result := <-bs.resultSignal
assert.EqualError(t, result.err, test.expectedErr)
assert.True(t, bt.withEvent)
assert.Equal(t, test.expectedEventReason, bt.EventReason())
assert.Equal(t, test.expectedEventMsg, bt.EventMessage())
})
}
}
func TestRunCancelableDataPath(t *testing.T) {
pvbName := "fake-pvb"
pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Phase(velerov1api.PodVolumeBackupPhaseNew).Result()
pvbInProgress := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result()
ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second)
tests := []struct {
name string
ctx context.Context
result *dataPathResult
dataPathMgr *datapath.Manager
kubeClientObj []runtime.Object
initErr error
startErr error
dataPathStarted bool
expectedEventMsg string
expectedErr string
}{
{
name: "no pvb",
ctx: ctxTimeout,
expectedErr: "error waiting for PVB: context deadline exceeded",
},
{
name: "pvb not in in-progress",
ctx: ctxTimeout,
kubeClientObj: []runtime.Object{pvb},
expectedErr: "error waiting for PVB: context deadline exceeded",
},
{
name: "create data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvbInProgress},
dataPathMgr: datapath.NewManager(0),
expectedErr: "error to create data path: Concurrent number exceeds",
},
{
name: "init data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvbInProgress},
initErr: errors.New("fake-init-error"),
expectedErr: "error to initialize data path: fake-init-error",
},
{
name: "start data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvbInProgress},
startErr: errors.New("fake-start-error"),
expectedErr: "error starting data path backup: fake-start-error",
},
{
name: "data path timeout",
ctx: ctxTimeout,
kubeClientObj: []runtime.Object{pvbInProgress},
dataPathStarted: true,
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName),
expectedErr: "timed out waiting for fs backup to complete",
},
{
name: "data path returns error",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvbInProgress},
dataPathStarted: true,
result: &dataPathResult{
err: errors.New("fake-data-path-error"),
},
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName),
expectedErr: "fake-data-path-error",
},
{
name: "succeed",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvbInProgress},
dataPathStarted: true,
result: &dataPathResult{
result: "fake-succeed-result",
},
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName),
},
}
scheme := runtime.NewScheme()
velerov1api.AddToScheme(scheme)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeClientBuilder := clientFake.NewClientBuilder()
fakeClientBuilder = fakeClientBuilder.WithScheme(scheme)
fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
bt := &backupMsTestHelper{}
bs := &BackupMicroService{
namespace: velerov1api.DefaultNamespace,
pvbName: pvbName,
ctx: context.Background(),
client: fakeClient,
dataPathMgr: datapath.NewManager(1),
eventRecorder: bt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
if test.ctx != nil {
bs.ctx = test.ctx
}
if test.dataPathMgr != nil {
bs.dataPathMgr = test.dataPathMgr
}
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
fsBR := datapathmockes.NewAsyncBR(t)
if test.initErr != nil {
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
}
if test.startErr != nil {
fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
}
if test.dataPathStarted {
fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(nil)
}
return fsBR
}
if test.result != nil {
go func() {
time.Sleep(time.Millisecond * 500)
bs.resultSignal <- *test.result
}()
}
result, err := bs.RunCancelableDataPath(test.ctx)
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
assert.Equal(t, test.result.result, result)
}
if test.expectedEventMsg != "" {
assert.True(t, bt.withEvent)
assert.Equal(t, test.expectedEventMsg, bt.EventMessage())
}
})
}
cancel()
}
pkg/podvolume/backupper.go
+5 -3
View File
@@ -169,7 +169,8 @@ func newBackupper(
}
if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted &&
pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed {
pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed &&
pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCanceled {
return
}
@@ -179,7 +180,8 @@ func newBackupper(
existPVB, ok := existObj.(*velerov1api.PodVolumeBackup)
// the PVB in the indexer is already in final status, no need to call WaitGroup.Done()
if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted ||
existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed) {
existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed ||
pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) {
statusChangedToFinal = false
}
}
@@ -428,7 +430,7 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero
continue
}
podVolumeBackups = append(podVolumeBackups, pvb)
if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed {
if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled {
log.Errorf("pod volume backup failed: %s", pvb.Status.Message)
}
}
pkg/podvolume/restore_micro_service.go
+345
View File
@@ -0,0 +1,345 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/kube"
cachetool "k8s.io/client-go/tools/cache"
)
// RestoreMicroService process data mover restores inside the restore pod
type RestoreMicroService struct {
ctx context.Context
client client.Client
kubeClient kubernetes.Interface
repoEnsurer *repository.Ensurer
credentialGetter *credentials.CredentialGetter
logger logrus.FieldLogger
dataPathMgr *datapath.Manager
eventRecorder kube.EventRecorder
namespace string
pvrName string
pvr *velerov1api.PodVolumeRestore
sourceTargetPath datapath.AccessPoint
resultSignal chan dataPathResult
pvrInformer cache.Informer
pvrHandler cachetool.ResourceEventHandlerRegistration
nodeName string
}
func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, pvrName string, namespace string, nodeName string,
sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter,
pvrInformer cache.Informer, log logrus.FieldLogger) *RestoreMicroService {
return &RestoreMicroService{
ctx: ctx,
client: client,
kubeClient: kubeClient,
credentialGetter: cred,
logger: log,
repoEnsurer: repoEnsurer,
dataPathMgr: dataPathMgr,
namespace: namespace,
pvrName: pvrName,
sourceTargetPath: sourceTargetPath,
nodeName: nodeName,
resultSignal: make(chan dataPathResult),
pvrInformer: pvrInformer,
}
}
func (r *RestoreMicroService) Init() error {
r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.pvrName, r.nodeName, r.logger)
handler, err := r.pvrInformer.AddEventHandler(
cachetool.ResourceEventHandlerFuncs{
UpdateFunc: func(oldObj any, newObj any) {
oldPvr := oldObj.(*velerov1api.PodVolumeRestore)
newPvr := newObj.(*velerov1api.PodVolumeRestore)
if newPvr.Name != r.pvrName {
return
}
if newPvr.Status.Phase != velerov1api.PodVolumeRestorePhaseInProgress {
return
}
if newPvr.Spec.Cancel && !oldPvr.Spec.Cancel {
r.cancelPodVolumeRestore(newPvr)
}
},
},
)
if err != nil {
return errors.Wrap(err, "error adding PVR handler")
}
r.pvrHandler = handler
return err
}
func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string, error) {
log := r.logger.WithFields(logrus.Fields{
"PVR": r.pvrName,
})
pvr := &velerov1api.PodVolumeRestore{}
err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) {
err := r.client.Get(ctx, types.NamespacedName{
Namespace: r.namespace,
Name: r.pvrName,
}, pvr)
if apierrors.IsNotFound(err) {
return false, nil
}
if err != nil {
return true, errors.Wrapf(err, "error to get PVR %s", r.pvrName)
}
if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress {
return true, nil
} else {
return false, nil
}
})
if err != nil {
log.WithError(err).Error("Failed to wait PVR")
return "", errors.Wrap(err, "error waiting for PVR")
}
r.pvr = pvr
log.Info("Run cancelable PVR")
callbacks := datapath.Callbacks{
OnCompleted: r.OnPvrCompleted,
OnFailed: r.OnPvrFailed,
OnCancelled: r.OnPvrCancelled,
OnProgress: r.OnPvrProgress,
}
fsRestore, err := r.dataPathMgr.CreateFileSystemBR(pvr.Name, podVolumeRequestor, ctx, r.client, pvr.Namespace, callbacks, log)
if err != nil {
return "", errors.Wrap(err, "error to create data path")
}
log.Debug("Async fs br created")
if err := fsRestore.Init(ctx,
&datapath.FSBRInitParam{
BSLName: pvr.Spec.BackupStorageLocation,
SourceNamespace: pvr.Spec.SourceNamespace,
UploaderType: pvr.Spec.UploaderType,
RepositoryType: velerov1api.BackupRepositoryTypeKopia,
RepoIdentifier: "",
RepositoryEnsurer: r.repoEnsurer,
CredentialGetter: r.credentialGetter,
}); err != nil {
return "", errors.Wrap(err, "error to initialize data path")
}
log.Info("Async fs br init")
if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings); err != nil {
return "", errors.Wrap(err, "error starting data path restore")
}
log.Info("Async fs restore data path started")
r.eventRecorder.Event(pvr, false, datapath.EventReasonStarted, "Data path for %s started", pvr.Name)
result := ""
select {
case <-ctx.Done():
err = errors.New("timed out waiting for fs restore to complete")
break
case res := <-r.resultSignal:
err = res.err
result = res.result
break
}
if err != nil {
log.WithError(err).Error("Async fs restore was not completed")
}
r.eventRecorder.EndingEvent(pvr, false, datapath.EventReasonStopped, "Data path for %s stopped", pvr.Name)
return result, err
}
func (r *RestoreMicroService) Shutdown() {
r.eventRecorder.Shutdown()
r.closeDataPath(r.ctx, r.pvrName)
if r.pvrHandler != nil {
if err := r.pvrInformer.RemoveEventHandler(r.pvrHandler); err != nil {
r.logger.WithError(err).Warn("Failed to remove pod handler")
}
}
}
var funcWriteCompletionMark = writeCompletionMark
func (r *RestoreMicroService) OnPvrCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) {
log := r.logger.WithField("PVR", pvrName)
err := funcWriteCompletionMark(r.pvr, result.Restore, log)
if err != nil {
log.WithError(err).Warnf("Failed to write completion mark, restored pod may failed to start")
}
restoreBytes, err := funcMarshal(result.Restore)
if err != nil {
log.WithError(err).Errorf("Failed to marshal restore result %v", result.Restore)
r.recordPvrFailed(fmt.Sprintf("error marshaling restore result %v", result.Restore), err)
} else {
r.eventRecorder.Event(r.pvr, false, datapath.EventReasonCompleted, string(restoreBytes))
r.resultSignal <- dataPathResult{
result: string(restoreBytes),
}
}
log.Info("Async fs restore data path completed")
}
func (r *RestoreMicroService) recordPvrFailed(msg string, err error) {
evtMsg := fmt.Sprintf("%s, error %v", msg, err)
r.eventRecorder.Event(r.pvr, false, datapath.EventReasonFailed, evtMsg)
r.resultSignal <- dataPathResult{
err: errors.Wrapf(err, msg),
}
}
func (r *RestoreMicroService) OnPvrFailed(ctx context.Context, namespace string, pvrName string, err error) {
log := r.logger.WithField("PVR", pvrName)
log.WithError(err).Error("Async fs restore data path failed")
r.recordPvrFailed(fmt.Sprintf("Data path for PVR %s failed", pvrName), err)
}
func (r *RestoreMicroService) OnPvrCancelled(ctx context.Context, namespace string, pvrName string) {
log := r.logger.WithField("PVR", pvrName)
log.Warn("Async fs restore data path canceled")
r.eventRecorder.Event(r.pvr, false, datapath.EventReasonCancelled, "Data path for PVR %s canceled", pvrName)
r.resultSignal <- dataPathResult{
err: errors.New(datapath.ErrCancelled),
}
}
func (r *RestoreMicroService) OnPvrProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) {
log := r.logger.WithFields(logrus.Fields{
"PVR": pvrName,
})
progressBytes, err := funcMarshal(progress)
if err != nil {
log.WithError(err).Errorf("Failed to marshal progress %v", progress)
return
}
r.eventRecorder.Event(r.pvr, false, datapath.EventReasonProgress, string(progressBytes))
}
func (r *RestoreMicroService) closeDataPath(ctx context.Context, pvrName string) {
fsRestore := r.dataPathMgr.GetAsyncBR(pvrName)
if fsRestore != nil {
fsRestore.Close(ctx)
}
r.dataPathMgr.RemoveAsyncBR(pvrName)
}
func (r *RestoreMicroService) cancelPodVolumeRestore(pvr *velerov1api.PodVolumeRestore) {
r.logger.WithField("PVR", pvr.Name).Info("PVR is being canceled")
r.eventRecorder.Event(pvr, false, datapath.EventReasonCancelling, "Canceling for PVR %s", pvr.Name)
fsBackup := r.dataPathMgr.GetAsyncBR(pvr.Name)
if fsBackup == nil {
r.OnPvrCancelled(r.ctx, pvr.GetNamespace(), pvr.GetName())
} else {
fsBackup.Cancel()
}
}
var funcRemoveAll = os.RemoveAll
var funcMkdirAll = os.MkdirAll
var funcWriteFile = os.WriteFile
func writeCompletionMark(pvr *velerov1api.PodVolumeRestore, result datapath.RestoreResult, log logrus.FieldLogger) error {
volumePath := result.Target.ByPath
if volumePath == "" {
return errors.New("target volume is empty in restore result")
}
// Remove the .velero directory from the restored volume (it may contain done files from previous restores
// of this volume, which we don't want to carry over). If this fails for any reason, log and continue, since
// this is non-essential cleanup (the done files are named based on restore UID and the init container looks
// for the one specific to the restore being executed).
if err := funcRemoveAll(filepath.Join(volumePath, ".velero")); err != nil {
log.WithError(err).Warnf("Failed to remove .velero directory from directory %s", volumePath)
}
if len(pvr.OwnerReferences) == 0 {
return errors.New("error finding restore UID")
}
restoreUID := pvr.OwnerReferences[0].UID
// Create the .velero directory within the volume dir so we can write a done file
// for this restore.
if err := funcMkdirAll(filepath.Join(volumePath, ".velero"), 0755); err != nil {
return errors.Wrapf(err, "error creating .velero directory for done file")
}
// Write a done file with name=<restore-uid> into the just-created .velero dir
// within the volume. The velero init container on the pod is waiting
// for this file to exist in each restored volume before completing.
if err := funcWriteFile(filepath.Join(volumePath, ".velero", string(restoreUID)), nil, 0644); err != nil {
return errors.Wrapf(err, "error writing done file")
}
return nil
}
pkg/podvolume/restore_micro_service_test.go
+622
View File
@@ -0,0 +1,622 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podvolume
import (
"context"
"fmt"
"os"
"sync"
"testing"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"k8s.io/apimachinery/pkg/runtime"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/uploader"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type restoreMsTestHelper struct {
eventReason string
eventMsg string
marshalErr error
marshalBytes []byte
withEvent bool
eventLock sync.Mutex
writeCompletionErr error
}
func (rt *restoreMsTestHelper) Event(_ runtime.Object, _ bool, reason string, message string, a ...any) {
rt.eventLock.Lock()
defer rt.eventLock.Unlock()
rt.withEvent = true
rt.eventReason = reason
rt.eventMsg = fmt.Sprintf(message, a...)
}
func (rt *restoreMsTestHelper) EndingEvent(_ runtime.Object, _ bool, reason string, message string, a ...any) {
rt.eventLock.Lock()
defer rt.eventLock.Unlock()
rt.withEvent = true
rt.eventReason = reason
rt.eventMsg = fmt.Sprintf(message, a...)
}
func (rt *restoreMsTestHelper) Shutdown() {}
func (rt *restoreMsTestHelper) Marshal(v any) ([]byte, error) {
if rt.marshalErr != nil {
return nil, rt.marshalErr
}
return rt.marshalBytes, nil
}
func (rt *restoreMsTestHelper) EventReason() string {
rt.eventLock.Lock()
defer rt.eventLock.Unlock()
return rt.eventReason
}
func (rt *restoreMsTestHelper) EventMessage() string {
rt.eventLock.Lock()
defer rt.eventLock.Unlock()
return rt.eventMsg
}
func (rt *restoreMsTestHelper) WriteCompletionMark(*velerov1api.PodVolumeRestore, datapath.RestoreResult, logrus.FieldLogger) error {
return rt.writeCompletionErr
}
func TestOnPvrFailed(t *testing.T) {
pvrName := "fake-pvr"
rt := &restoreMsTestHelper{}
rs := &RestoreMicroService{
pvrName: pvrName,
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
expectedErr := "Data path for PVR fake-pvr failed: fake-error"
expectedEventReason := datapath.EventReasonFailed
expectedEventMsg := "Data path for PVR fake-pvr failed, error fake-error"
go rs.OnPvrFailed(context.TODO(), velerov1api.DefaultNamespace, pvrName, errors.New("fake-error"))
result := <-rs.resultSignal
assert.EqualError(t, result.err, expectedErr)
assert.Equal(t, expectedEventReason, rt.EventReason())
assert.Equal(t, expectedEventMsg, rt.EventMessage())
}
func TestPvrCancelled(t *testing.T) {
pvrName := "fake-pvr"
rt := &restoreMsTestHelper{}
rs := RestoreMicroService{
pvrName: pvrName,
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
expectedErr := datapath.ErrCancelled
expectedEventReason := datapath.EventReasonCancelled
expectedEventMsg := "Data path for PVR fake-pvr canceled"
go rs.OnPvrCancelled(context.TODO(), velerov1api.DefaultNamespace, pvrName)
result := <-rs.resultSignal
assert.EqualError(t, result.err, expectedErr)
assert.Equal(t, expectedEventReason, rt.EventReason())
assert.Equal(t, expectedEventMsg, rt.EventMessage())
}
func TestOnPvrCompleted(t *testing.T) {
tests := []struct {
name string
expectedErr string
expectedEventReason string
expectedEventMsg string
marshalErr error
marshallStr string
writeCompletionErr error
expectedLog string
}{
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "error marshaling restore result {{ } 0}: fake-marshal-error",
},
{
name: "succeed",
marshallStr: "fake-complete-string",
expectedEventReason: datapath.EventReasonCompleted,
expectedEventMsg: "fake-complete-string",
},
{
name: "succeed but write completion mark fail",
marshallStr: "fake-complete-string",
writeCompletionErr: errors.New("fake-write-completion-error"),
expectedEventReason: datapath.EventReasonCompleted,
expectedEventMsg: "fake-complete-string",
expectedLog: "Failed to write completion mark, restored pod may failed to start",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvrName := "fake-pvr"
rt := &restoreMsTestHelper{
marshalErr: test.marshalErr,
marshalBytes: []byte(test.marshallStr),
writeCompletionErr: test.writeCompletionErr,
}
logBuffer := []string{}
rs := &RestoreMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewMultipleLogger(&logBuffer),
}
funcMarshal = rt.Marshal
funcWriteCompletionMark = rt.WriteCompletionMark
go rs.OnPvrCompleted(context.TODO(), velerov1api.DefaultNamespace, pvrName, datapath.Result{})
result := <-rs.resultSignal
if test.marshalErr != nil {
assert.EqualError(t, result.err, test.expectedErr)
} else {
assert.NoError(t, result.err)
assert.Equal(t, test.expectedEventReason, rt.EventReason())
assert.Equal(t, test.expectedEventMsg, rt.EventMessage())
if test.expectedLog != "" {
assert.Contains(t, logBuffer[0], test.expectedLog)
}
}
})
}
}
func TestOnPvrProgress(t *testing.T) {
tests := []struct {
name string
expectedErr string
expectedEventReason string
expectedEventMsg string
marshalErr error
marshallStr string
}{
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal restore result",
},
{
name: "succeed",
marshallStr: "fake-progress-string",
expectedEventReason: datapath.EventReasonProgress,
expectedEventMsg: "fake-progress-string",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvrName := "fake-pvr"
rt := &restoreMsTestHelper{
marshalErr: test.marshalErr,
marshalBytes: []byte(test.marshallStr),
}
rs := &RestoreMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
logger: velerotest.NewLogger(),
}
funcMarshal = rt.Marshal
rs.OnPvrProgress(context.TODO(), velerov1api.DefaultNamespace, pvrName, &uploader.Progress{})
if test.marshalErr != nil {
assert.False(t, rt.withEvent)
} else {
assert.True(t, rt.withEvent)
assert.Equal(t, test.expectedEventReason, rt.EventReason())
assert.Equal(t, test.expectedEventMsg, rt.EventMessage())
}
})
}
}
func TestCancelPodVolumeRestore(t *testing.T) {
tests := []struct {
name string
expectedEventReason string
expectedEventMsg string
expectedErr string
}{
{
name: "no fs restore",
expectedEventReason: datapath.EventReasonCancelled,
expectedEventMsg: "Data path for PVR fake-pvr canceled",
expectedErr: datapath.ErrCancelled,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pvrName := "fake-pvr"
pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result()
rt := &restoreMsTestHelper{}
rs := &RestoreMicroService{
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
go rs.cancelPodVolumeRestore(pvr)
result := <-rs.resultSignal
assert.EqualError(t, result.err, test.expectedErr)
assert.True(t, rt.withEvent)
assert.Equal(t, test.expectedEventReason, rt.EventReason())
assert.Equal(t, test.expectedEventMsg, rt.EventMessage())
})
}
}
func TestRunCancelableDataPathRestore(t *testing.T) {
pvrName := "fake-pvr"
pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseNew).Result()
pvrInProgress := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result()
ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second)
tests := []struct {
name string
ctx context.Context
result *dataPathResult
dataPathMgr *datapath.Manager
kubeClientObj []runtime.Object
initErr error
startErr error
dataPathStarted bool
expectedEventMsg string
expectedErr string
}{
{
name: "no pvr",
ctx: ctxTimeout,
expectedErr: "error waiting for PVR: context deadline exceeded",
},
{
name: "pvr not in in-progress",
ctx: ctxTimeout,
kubeClientObj: []runtime.Object{pvr},
expectedErr: "error waiting for PVR: context deadline exceeded",
},
{
name: "create data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvrInProgress},
dataPathMgr: datapath.NewManager(0),
expectedErr: "error to create data path: Concurrent number exceeds",
},
{
name: "init data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvrInProgress},
initErr: errors.New("fake-init-error"),
expectedErr: "error to initialize data path: fake-init-error",
},
{
name: "start data path fail",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvrInProgress},
startErr: errors.New("fake-start-error"),
expectedErr: "error starting data path restore: fake-start-error",
},
{
name: "data path timeout",
ctx: ctxTimeout,
kubeClientObj: []runtime.Object{pvrInProgress},
dataPathStarted: true,
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName),
expectedErr: "timed out waiting for fs restore to complete",
},
{
name: "data path returns error",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvrInProgress},
dataPathStarted: true,
result: &dataPathResult{
err: errors.New("fake-data-path-error"),
},
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName),
expectedErr: "fake-data-path-error",
},
{
name: "succeed",
ctx: context.Background(),
kubeClientObj: []runtime.Object{pvrInProgress},
dataPathStarted: true,
result: &dataPathResult{
result: "fake-succeed-result",
},
expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName),
},
}
scheme := runtime.NewScheme()
velerov1api.AddToScheme(scheme)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeClientBuilder := clientFake.NewClientBuilder()
fakeClientBuilder = fakeClientBuilder.WithScheme(scheme)
fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
rt := &restoreMsTestHelper{}
rs := &RestoreMicroService{
namespace: velerov1api.DefaultNamespace,
pvrName: pvrName,
ctx: context.Background(),
client: fakeClient,
dataPathMgr: datapath.NewManager(1),
eventRecorder: rt,
resultSignal: make(chan dataPathResult),
logger: velerotest.NewLogger(),
}
if test.ctx != nil {
rs.ctx = test.ctx
}
if test.dataPathMgr != nil {
rs.dataPathMgr = test.dataPathMgr
}
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
fsBR := datapathmockes.NewAsyncBR(t)
if test.initErr != nil {
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
}
if test.startErr != nil {
fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
}
if test.dataPathStarted {
fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil)
}
return fsBR
}
if test.result != nil {
go func() {
time.Sleep(time.Millisecond * 500)
rs.resultSignal <- *test.result
}()
}
result, err := rs.RunCancelableDataPath(test.ctx)
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
assert.Equal(t, test.result.result, result)
}
if test.expectedEventMsg != "" {
assert.True(t, rt.withEvent)
assert.Equal(t, test.expectedEventMsg, rt.EventMessage())
}
})
}
cancel()
}
func TestWriteCompletionMark(t *testing.T) {
tests := []struct {
name string
pvr *velerov1api.PodVolumeRestore
result datapath.RestoreResult
funcRemoveAll func(string) error
funcMkdirAll func(string, os.FileMode) error
funcWriteFile func(string, []byte, os.FileMode) error
expectedErr string
expectedLog string
}{
{
name: "no volume path",
result: datapath.RestoreResult{},
expectedErr: "target volume is empty in restore result",
},
{
name: "no owner reference",
result: datapath.RestoreResult{
Target: datapath.AccessPoint{
ByPath: "fake-volume-path",
},
},
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").Result(),
funcRemoveAll: func(string) error {
return nil
},
expectedErr: "error finding restore UID",
},
{
name: "mkdir fail",
result: datapath.RestoreResult{
Target: datapath.AccessPoint{
ByPath: "fake-volume-path",
},
},
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{
{
UID: "fake-uid",
},
}).Result(),
funcRemoveAll: func(string) error {
return nil
},
funcMkdirAll: func(string, os.FileMode) error {
return errors.New("fake-mk-dir-error")
},
expectedErr: "error creating .velero directory for done file: fake-mk-dir-error",
},
{
name: "write file fail",
result: datapath.RestoreResult{
Target: datapath.AccessPoint{
ByPath: "fake-volume-path",
},
},
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{
{
UID: "fake-uid",
},
}).Result(),
funcRemoveAll: func(string) error {
return nil
},
funcMkdirAll: func(string, os.FileMode) error {
return nil
},
funcWriteFile: func(string, []byte, os.FileMode) error {
return errors.New("fake-write-file-error")
},
expectedErr: "error writing done file: fake-write-file-error",
},
{
name: "succeed",
result: datapath.RestoreResult{
Target: datapath.AccessPoint{
ByPath: "fake-volume-path",
},
},
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{
{
UID: "fake-uid",
},
}).Result(),
funcRemoveAll: func(string) error {
return nil
},
funcMkdirAll: func(string, os.FileMode) error {
return nil
},
funcWriteFile: func(string, []byte, os.FileMode) error {
return nil
},
},
{
name: "succeed but previous dir is not removed",
result: datapath.RestoreResult{
Target: datapath.AccessPoint{
ByPath: "fake-volume-path",
},
},
pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{
{
UID: "fake-uid",
},
}).Result(),
funcRemoveAll: func(string) error {
return errors.New("fake-remove-dir-error")
},
funcMkdirAll: func(string, os.FileMode) error {
return nil
},
funcWriteFile: func(string, []byte, os.FileMode) error {
return nil
},
expectedLog: "Failed to remove .velero directory from directory fake-volume-path",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.funcRemoveAll != nil {
funcRemoveAll = test.funcRemoveAll
}
if test.funcMkdirAll != nil {
funcMkdirAll = test.funcMkdirAll
}
if test.funcWriteFile != nil {
funcWriteFile = test.funcWriteFile
}
logBuffer := ""
err := writeCompletionMark(test.pvr, test.result, velerotest.NewSingleLogger(&logBuffer))
if test.expectedErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, test.expectedErr)
}
if test.expectedLog != "" {
assert.Contains(t, logBuffer, test.expectedLog)
}
})
}
}
pkg/podvolume/restorer.go
+2 -2
View File
@@ -98,7 +98,7 @@ func newRestorer(
return
}
if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed {
if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled {
r.resultsLock.Lock()
defer r.resultsLock.Unlock()
@@ -234,7 +234,7 @@ ForEachVolume:
errs = append(errs, errors.New("timed out waiting for all PodVolumeRestores to complete"))
break ForEachVolume
case res := <-resultsChan:
if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed {
if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || res.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled {
errs = append(errs, errors.Errorf("pod volume restore failed: %s", res.Status.Message))
}
tracker.TrackPodVolume(res)
pkg/podvolume/util.go
+15
View File
@@ -17,12 +17,14 @@ limitations under the License.
package podvolume
import (
"fmt"
"strings"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/podvolume/configs"
repotypes "github.com/vmware-tanzu/velero/pkg/repository/types"
"github.com/vmware-tanzu/velero/pkg/uploader"
)
@@ -143,6 +145,19 @@ func GetSnapshotIdentifier(podVolumeBackups *velerov1api.PodVolumeBackupList) ma
return res
}
func GetRealSource(pvb *velerov1api.PodVolumeBackup) string {
pvcName := ""
if pvb.Annotations != nil {
pvcName = pvb.Annotations[configs.PVCNameAnnotation]
}
if pvcName != "" {
return fmt.Sprintf("%s/%s/%s", pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvcName)
} else {
return fmt.Sprintf("%s/%s/%s", pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvb.Spec.Volume)
}
}
func getUploaderTypeOrDefault(uploaderType string) string {
if uploaderType != "" {
return uploaderType
pkg/podvolume/util_test.go
+31
View File
@@ -300,3 +300,34 @@ func TestVolumeHasNonRestorableSource(t *testing.T) {
})
}
}
func TestGetRealSource(t *testing.T) {
testCases := []struct {
name string
pvb *velerov1api.PodVolumeBackup
expected string
}{
{
name: "pvb with empty annotation",
pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Result(),
expected: "fake-pod-ns/fake-pod-name/fake-volume",
},
{
name: "pvb without pvc name annotation",
pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Annotations(map[string]string{}).Result(),
expected: "fake-pod-ns/fake-pod-name/fake-volume",
},
{
name: "pvb with pvc name annotation",
pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Annotations(map[string]string{"velero.io/pvc-name": "fake-pvc-name"}).Result(),
expected: "fake-pod-ns/fake-pod-name/fake-pvc-name",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
actual := GetRealSource(tc.pvb)
assert.Equal(t, tc.expected, actual)
})
}
}
pkg/util/csi/volume_snapshot.go
-24
View File
@@ -588,32 +588,8 @@ func WaitUntilVSCHandleIsReady(
volSnap *snapshotv1api.VolumeSnapshot,
crClient crclient.Client,
log logrus.FieldLogger,
shouldWait bool,
csiSnapshotTimeout time.Duration,
) (*snapshotv1api.VolumeSnapshotContent, error) {
if !shouldWait {
if volSnap.Status == nil ||
volSnap.Status.BoundVolumeSnapshotContentName == nil {
// volumesnapshot hasn't been reconciled and we're
// not waiting for it.
return nil, nil
}
vsc := new(snapshotv1api.VolumeSnapshotContent)
err := crClient.Get(
context.TODO(),
crclient.ObjectKey{
Name: *volSnap.Status.BoundVolumeSnapshotContentName,
},
vsc,
)
if err != nil {
return nil,
errors.Wrap(err,
"error getting volume snapshot content from API")
}
return vsc, nil
}
// We'll wait 10m for the VSC to be reconciled polling
// every 5s unless backup's csiSnapshotTimeout is set
interval := 5 * time.Second
pkg/util/csi/volume_snapshot_test.go
+1 -40
View File
@@ -1646,26 +1646,22 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) {
name string
volSnap *snapshotv1api.VolumeSnapshot
exepctedVSC *snapshotv1api.VolumeSnapshotContent
wait bool
expectError bool
}{
{
name: "waitEnabled should find volumesnapshotcontent for volumesnapshot",
volSnap: validVS,
exepctedVSC: vscObj,
wait: true,
expectError: false,
},
{
name: "waitEnabled should not find volumesnapshotcontent for volumesnapshot with non-existing snapshotcontent name in status.BoundVolumeSnapshotContentName",
volSnap: vsWithVSCNotFound,
exepctedVSC: nil,
wait: true,
expectError: true,
},
{
name: "waitEnabled should not find volumesnapshotcontent for a non-existent volumesnapshot",
wait: true,
exepctedVSC: nil,
expectError: true,
volSnap: &snapshotv1api.VolumeSnapshot{
@@ -1678,46 +1674,11 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) {
},
},
},
{
name: "waitDisabled should not find volumesnapshotcontent when volumesnapshot status is nil",
wait: false,
expectError: false,
exepctedVSC: nil,
volSnap: vsWithNilStatus,
},
{
name: "waitDisabled should not find volumesnapshotcontent when volumesnapshot status.BoundVolumeSnapshotContentName is nil",
wait: false,
expectError: false,
exepctedVSC: nil,
volSnap: vsWithNilStatusField,
},
{
name: "waitDisabled should find volumesnapshotcontent when volumesnapshotcontent status is nil",
wait: false,
expectError: false,
exepctedVSC: vscWithNilStatus,
volSnap: vsForNilStatusVsc,
},
{
name: "waitDisabled should find volumesnapshotcontent when volumesnapshotcontent status.SnapshotHandle is nil",
wait: false,
expectError: false,
exepctedVSC: vscWithNilStatusField,
volSnap: vsForNilStatusFieldVsc,
},
{
name: "waitDisabled should not find a non-existent volumesnapshotcontent",
wait: false,
exepctedVSC: nil,
expectError: true,
volSnap: vsWithVSCNotFound,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
actualVSC, actualError := WaitUntilVSCHandleIsReady(tc.volSnap, fakeClient, logrus.New().WithField("fake", "test"), tc.wait, 0)
actualVSC, actualError := WaitUntilVSCHandleIsReady(tc.volSnap, fakeClient, logrus.New().WithField("fake", "test"), 0)
if tc.expectError && actualError == nil {
assert.Error(t, actualError)
assert.Nil(t, actualVSC)
pkg/util/kube/pod.go
+28
View File
@@ -19,6 +19,7 @@ import (
"context"
"fmt"
"io"
"os"
"time"
"github.com/pkg/errors"
@@ -273,3 +274,30 @@ func DiagnosePod(pod *corev1api.Pod) string {
return diag
}
var funcExit = os.Exit
var funcCreateFile = os.Create
func ExitPodWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) {
exitCode := 0
if !succeed {
exitCode = 1
}
toWrite := fmt.Sprintf(message, a...)
podFile, err := funcCreateFile("/dev/termination-log")
if err != nil {
logger.WithError(err).Error("Failed to create termination log file")
exitCode = 1
} else {
if _, err := podFile.WriteString(toWrite); err != nil {
logger.WithError(err).Error("Failed to write error to termination log file")
exitCode = 1
}
podFile.Close()
}
funcExit(exitCode)
}
pkg/util/kube/pod_test.go
+107
View File
@@ -18,14 +18,19 @@ package kube
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -932,3 +937,105 @@ func TestDiagnosePod(t *testing.T) {
})
}
}
type exitWithMessageMock struct {
createErr error
writeFail bool
filePath string
exitCode int
}
func (em *exitWithMessageMock) Exit(code int) {
em.exitCode = code
}
func (em *exitWithMessageMock) CreateFile(name string) (*os.File, error) {
if em.createErr != nil {
return nil, em.createErr
}
if em.writeFail {
return os.OpenFile(em.filePath, os.O_CREATE|os.O_RDONLY, 0500)
} else {
return os.Create(em.filePath)
}
}
func TestExitPodWithMessage(t *testing.T) {
tests := []struct {
name string
message string
succeed bool
args []any
createErr error
writeFail bool
expectedExitCode int
expectedMessage string
}{
{
name: "create pod file failed",
createErr: errors.New("fake-create-file-error"),
succeed: true,
expectedExitCode: 1,
},
{
name: "write pod file failed",
writeFail: true,
succeed: true,
expectedExitCode: 1,
},
{
name: "not succeed",
message: "fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v",
args: []any{
"arg-1-1",
10,
false,
},
expectedExitCode: 1,
expectedMessage: fmt.Sprintf("fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-1", 10, false),
},
{
name: "not succeed",
message: "fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v",
args: []any{
"arg-1-2",
20,
true,
},
succeed: true,
expectedMessage: fmt.Sprintf("fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-2", 20, true),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
podFile := filepath.Join(os.TempDir(), uuid.NewString())
em := exitWithMessageMock{
createErr: test.createErr,
writeFail: test.writeFail,
filePath: podFile,
}
funcExit = em.Exit
funcCreateFile = em.CreateFile
ExitPodWithMessage(velerotest.NewLogger(), test.succeed, test.message, test.args...)
assert.Equal(t, test.expectedExitCode, em.exitCode)
if test.createErr == nil && !test.writeFail {
reader, err := os.Open(podFile)
require.NoError(t, err)
message, err := io.ReadAll(reader)
require.NoError(t, err)
reader.Close()
assert.Equal(t, test.expectedMessage, string(message))
}
})
}
}
pkg/util/kube/utils.go
+13 -15
View File
@@ -25,7 +25,6 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
storagev1api "k8s.io/api/storage/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,8 +32,8 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/label"
@@ -148,8 +147,8 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core
// GetVolumeDirectory gets the name of the directory on the host, under /var/lib/kubelet/pods/<podUID>/volumes/,
// where the specified volume lives.
// For volumes with a CSIVolumeSource, append "/mount" to the directory name.
func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) (string, error) {
pvc, pv, volume, err := GetPodPVCVolume(ctx, log, pod, volumeName, cli)
func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) (string, error) {
pvc, pv, volume, err := GetPodPVCVolume(ctx, log, pod, volumeName, kubeClient)
if err != nil {
// This case implies the administrator created the PV and attached it directly, without PVC.
// Note that only one VolumeSource can be populated per Volume on a pod
@@ -164,7 +163,7 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1
// Most common case is that we have a PVC VolumeSource, and we need to check the PV it points to for a CSI source.
// PV's been created with a CSI source.
isProvisionedByCSI, err := isProvisionedByCSI(log, pv, cli)
isProvisionedByCSI, err := isProvisionedByCSI(log, pv, kubeClient)
if err != nil {
return "", errors.WithStack(err)
}
@@ -179,9 +178,9 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1
}
// GetVolumeMode gets the uploader.PersistentVolumeMode of the volume.
func GetVolumeMode(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) (
func GetVolumeMode(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) (
uploader.PersistentVolumeMode, error) {
_, pv, _, err := GetPodPVCVolume(ctx, log, pod, volumeName, cli)
_, pv, _, err := GetPodPVCVolume(ctx, log, pod, volumeName, kubeClient)
if err != nil {
if err == ErrorPodVolumeIsNotPVC {
@@ -198,7 +197,7 @@ func GetVolumeMode(ctx context.Context, log logrus.FieldLogger, pod *corev1api.P
// GetPodPVCVolume gets the PVC, PV and volume for a pod volume name.
// Returns pod volume in case of ErrorPodVolumeIsNotPVC error
func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) (
func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) (
*corev1api.PersistentVolumeClaim, *corev1api.PersistentVolume, *corev1api.Volume, error) {
var volume *corev1api.Volume
@@ -217,14 +216,12 @@ func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api
return nil, nil, volume, ErrorPodVolumeIsNotPVC // There is a pod volume but it is not a PVC
}
pvc := &corev1api.PersistentVolumeClaim{}
err := cli.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: volume.VolumeSource.PersistentVolumeClaim.ClaimName}, pvc)
pvc, err := kubeClient.CoreV1().PersistentVolumeClaims(pod.Namespace).Get(ctx, volume.VolumeSource.PersistentVolumeClaim.ClaimName, metav1.GetOptions{})
if err != nil {
return nil, nil, nil, errors.WithStack(err)
}
pv := &corev1api.PersistentVolume{}
err = cli.Get(ctx, client.ObjectKey{Name: pvc.Spec.VolumeName}, pv)
pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return nil, nil, nil, errors.WithStack(err)
}
@@ -235,7 +232,7 @@ func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api
// isProvisionedByCSI function checks whether this is a CSI PV by annotation.
// Either "pv.kubernetes.io/provisioned-by" or "pv.kubernetes.io/migrated-to" indicates
// PV is provisioned by CSI.
func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kbClient client.Client) (bool, error) {
func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kubeClient kubernetes.Interface) (bool, error) {
if pv.Spec.CSI != nil {
return true, nil
}
@@ -245,10 +242,11 @@ func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume,
driverName := pv.Annotations[KubeAnnDynamicallyProvisioned]
migratedDriver := pv.Annotations[KubeAnnMigratedTo]
if len(driverName) > 0 || len(migratedDriver) > 0 {
list := &storagev1api.CSIDriverList{}
if err := kbClient.List(context.TODO(), list); err != nil {
list, err := kubeClient.StorageV1().CSIDrivers().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return false, err
}
for _, driver := range list.Items {
if driverName == driver.Name || migratedDriver == driver.Name {
log.Debugf("the annotation %s or %s equals to %s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, KubeAnnMigratedTo, driver.Name)
pkg/util/kube/utils_test.go
+14 -11
View File
@@ -34,11 +34,12 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"github.com/vmware-tanzu/velero/pkg/builder"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/uploader"
"k8s.io/client-go/kubernetes/fake"
)
func TestNamespaceAndName(t *testing.T) {
@@ -216,17 +217,18 @@ func TestGetVolumeDirectorySuccess(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "csi.test.com"},
}
for _, tc := range tests {
clientBuilder := fake.NewClientBuilder().WithLists(&storagev1api.CSIDriverList{Items: []storagev1api.CSIDriver{csiDriver}})
objs := []runtime.Object{&csiDriver}
if tc.pvc != nil {
clientBuilder = clientBuilder.WithObjects(tc.pvc)
objs = append(objs, tc.pvc)
}
if tc.pv != nil {
clientBuilder = clientBuilder.WithObjects(tc.pv)
objs = append(objs, tc.pv)
}
fakeKubeClient := fake.NewSimpleClientset(objs...)
// Function under test
dir, err := GetVolumeDirectory(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, clientBuilder.Build())
dir, err := GetVolumeDirectory(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, fakeKubeClient)
require.NoError(t, err)
assert.Equal(t, tc.want, dir)
@@ -264,17 +266,18 @@ func TestGetVolumeModeSuccess(t *testing.T) {
}
for _, tc := range tests {
clientBuilder := fake.NewClientBuilder()
objs := []runtime.Object{}
if tc.pvc != nil {
clientBuilder = clientBuilder.WithObjects(tc.pvc)
objs = append(objs, tc.pvc)
}
if tc.pv != nil {
clientBuilder = clientBuilder.WithObjects(tc.pv)
objs = append(objs, tc.pv)
}
fakeKubeClient := fake.NewSimpleClientset(objs...)
// Function under test
mode, err := GetVolumeMode(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, clientBuilder.Build())
mode, err := GetVolumeMode(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, fakeKubeClient)
require.NoError(t, err)
assert.Equal(t, tc.want, mode)
pkg/util/velero/velero.go
+6
View File
@@ -19,6 +19,8 @@ package velero
import (
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
// GetNodeSelectorFromVeleroServer get the node selector from the Velero server deployment
@@ -105,3 +107,7 @@ func GetVeleroServerAnnotationValue(deployment *appsv1api.Deployment, key string
return deployment.Spec.Template.Annotations[key]
}
func BSLIsAvailable(bsl velerov1api.BackupStorageLocation) bool {
return bsl.Status.Phase == velerov1api.BackupStorageLocationPhaseAvailable
}
pkg/util/velero/velero_test.go
+25 -31
View File
@@ -21,9 +21,13 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
)
func TestGetNodeSelectorFromVeleroServer(t *testing.T) {
@@ -207,39 +211,21 @@ func TestGetAffinityFromVeleroServer(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got := GetAffinityFromVeleroServer(test.deploy)
if got == nil {
if test.want != nil {
t.Errorf("expected affinity to be %v, got nil", test.want)
if test.want != nil {
require.NotNilf(t, got, "expected affinity to be %v, got nil", test.want)
if test.want.NodeAffinity != nil {
require.NotNilf(t, got.NodeAffinity, "expected node affinity to be %v, got nil", test.want.NodeAffinity)
if test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
require.NotNilf(t, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "expected required during scheduling ignored during execution to be %v, got nil", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
assert.Truef(t, reflect.DeepEqual(got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution), "expected required during scheduling ignored during execution to be %v, got %v", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
} else {
assert.Nilf(t, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "expected required during scheduling ignored during execution to be nil, got %v", got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
}
} else {
assert.Nilf(t, got.NodeAffinity, "expected node affinity to be nil, got %v", got.NodeAffinity)
}
} else {
if test.want == nil {
t.Errorf("expected affinity to be nil, got %v", got)
} else {
if got.NodeAffinity == nil {
if test.want.NodeAffinity != nil {
t.Errorf("expected node affinity to be %v, got nil", test.want.NodeAffinity)
}
} else {
if test.want.NodeAffinity == nil {
t.Errorf("expected node affinity to be nil, got %v", got.NodeAffinity)
} else {
if got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
if test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
t.Errorf("expected required during scheduling ignored during execution to be %v, got nil", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
}
} else {
if test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
t.Errorf("expected required during scheduling ignored during execution to be nil, got %v", got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
} else {
if !reflect.DeepEqual(got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) {
t.Errorf("expected required during scheduling ignored during execution to be %v, got %v", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
}
}
}
}
}
}
assert.Nilf(t, got, "expected affinity to be nil, got %v", got)
}
})
}
@@ -759,3 +745,11 @@ func TestGetVeleroServerLabelValue(t *testing.T) {
})
}
}
func TestBSLIsAvailable(t *testing.T) {
availableBSL := builder.ForBackupStorageLocation("velero", "available").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
unavailableBSL := builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result()
assert.True(t, BSLIsAvailable(*availableBSL))
assert.False(t, BSLIsAvailable(*unavailableBSL))
}
site/content/docs/main/customize-installation.md
+6
View File
@@ -178,6 +178,12 @@ Additionally, you may want to update the the default File System Backup operatio
- --fs-backup-timeout=240m
```
### Ephemeral-storage Requests and Limits
Velero does not set ephemeral-storage limits during installation. Limits and requests can be edited after install for clusters that monitor and restrict ephemeral-storage usage.
Plugins will use ephemeral-storage. There needs to be a sufficient requests and limit set to account for plugins and the additional ephemeral-storage used to maintain credentials and cache space for datamovers. Object storage plugins will fit comfortably into an allocation of 100MB of ephemeral-storage.
## Configure more than one storage location for backups or volume snapshots
Velero supports any number of backup storage locations and volume snapshot locations. For more details, see [about locations](locations.md).
site/content/docs/v1.14/customize-installation.md
+6
View File
@@ -178,6 +178,12 @@ Additionally, you may want to update the the default File System Backup operatio
- --fs-backup-timeout=240m
```
### Ephemeral-storage Requests and Limits
Velero does not set ephemeral-storage limits during installation. Limits and requests can be edited after install for clusters that monitor and restrict ephemeral-storage usage.
Plugins will use ephemeral-storage. There needs to be a sufficient requests and limit set to account for plugins and the additional ephemeral-storage used to maintain credentials and cache space for datamovers. Object storage plugins will fit comfortably into an allocation of 100MB of ephemeral-storage.
## Configure more than one storage location for backups or volume snapshots
Velero supports any number of backup storage locations and volume snapshot locations. For more details, see [about locations](locations.md).