mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-08-27 19:36:19 +00:00
Cherry pick the in-place restore implementation PRs from feature branch to main (#10415)
* Update CRDs and CLI to support in-place restore (#10038) Update CRDs(Restore, DataDownload, PodVolumeRestore) and restore create CLI to support in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore (#10066) Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Restore Exposer and PVC CSI to support in-place restore (#10104) 1. Update Restore Exposer to support exposing with existing PV for in-place restore 2. Update PVC CSI RIA to continue the restore process for in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Block uploader to support increase restore (#10244) Update Block uploader to support increase restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Exposer to recreate the target PV if the volume mode is different with the restore PVC (#10257) Update Exposer to recreate the target PV if the volume mode is different with t he restore PVC Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Preserve PVC selected-node annotation via carrier annotation for in-place restore For in-place volume data restore, the existing PVC is deleted and recreated. For StorageClasses with the WaitForFirstConsumer volume binding mode, losing the volume.kubernetes.io/selected-node annotation could let the scheduler place the recreated workload Pod in a different zone than the original PV, leaving it stuck in ContainerCreating. Instead of relying on RestoreItemAction execution order (the generic PVC RIA unconditionally strips the selected-node annotation), the PVC CSI RIA now captures the annotation from the existing PVC right before deleting it and carries it on the target PVC via the Velero-internal restore.velero.io/inplace-restore-selected-node annotation. The restore engine translates the carrier back to the Kubernetes annotation after all RestoreItemActions have run and always strips the carrier so it never lands on the cluster. This makes the behavior independent of RIA ordering: the Kubernetes annotation is stripped by default on every path (including when the target PVC does not exist and Velero falls back to provisioning a new PVC), and preservation only happens when the CSI RIA explicitly captured a value from the existing PVC. Signed-off-by: chlins <chlins.zhang@gmail.com> * Update the control path to make the in-place incremental restore with block data mover work E2E (#10410) Update the control path to make the in-place incremental restore with block data mover work E2E Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> --------- Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> Signed-off-by: chlins <chlins.zhang@gmail.com> Co-authored-by: chlins <chlins.zhang@gmail.com>
This commit is contained in:
co-authored by
chlins
parent
ac5744c7b4
commit
b7d83a6f2b
@@ -132,6 +132,9 @@ spec:
|
||||
repoIdentifier:
|
||||
description: RepoIdentifier is the backup repository identifier.
|
||||
type: string
|
||||
restoreType:
|
||||
description: RestoreType indicates the type of the restore.
|
||||
type: string
|
||||
snapshotID:
|
||||
description: SnapshotID is the ID of the volume snapshot to be restored.
|
||||
type: string
|
||||
@@ -167,6 +170,7 @@ spec:
|
||||
- backupStorageLocation
|
||||
- pod
|
||||
- repoIdentifier
|
||||
- restoreType
|
||||
- snapshotID
|
||||
- sourceNamespace
|
||||
- volume
|
||||
|
||||
@@ -89,6 +89,11 @@ spec:
|
||||
for the Kubernetes resource to be restored
|
||||
nullable: true
|
||||
type: string
|
||||
existingVolumeDataPolicy:
|
||||
description: ExistingVolumeDataPolicy specifies the restore behavior
|
||||
for the volume data to be restored
|
||||
nullable: true
|
||||
type: string
|
||||
hooks:
|
||||
description: Hooks represent custom behaviors that should be executed
|
||||
during or post restore.
|
||||
@@ -499,6 +504,12 @@ spec:
|
||||
description: UploaderConfig specifies the configuration for the restore.
|
||||
nullable: true
|
||||
properties:
|
||||
deleteExtraFiles:
|
||||
description: |-
|
||||
DeleteExtraFiles specifies whether to delete the extra files in the target volume that do not exist in the backup.
|
||||
This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores.
|
||||
nullable: true
|
||||
type: boolean
|
||||
parallelFilesDownload:
|
||||
description: ParallelFilesDownload is the concurrency number setting
|
||||
for restore.
|
||||
|
||||
@@ -83,6 +83,34 @@ spec:
|
||||
Cancel indicates request to cancel the ongoing DataDownload. It can be set
|
||||
when the DataDownload is in InProgress phase
|
||||
type: boolean
|
||||
csiSnapshot:
|
||||
description: CSISnapshot provides the information of the CSI snapshot
|
||||
used to do the incremental restore.
|
||||
nullable: true
|
||||
properties:
|
||||
driver:
|
||||
description: Driver is the driver used by the VolumeSnapshotContent
|
||||
type: string
|
||||
snapshotClass:
|
||||
description: SnapshotClass is the name of the snapshot class that
|
||||
the volume snapshot is created with
|
||||
type: string
|
||||
storageClass:
|
||||
description: StorageClass is the name of the storage class of
|
||||
the PVC that the volume snapshot is created from
|
||||
type: string
|
||||
volumeSnapshot:
|
||||
description: VolumeSnapshot is the name of the volume snapshot
|
||||
to be backed up
|
||||
type: string
|
||||
volumeSnapshotNamespace:
|
||||
description: VolumeSnapshotNamespace is the namespece of the volume
|
||||
snapshot to be backed up
|
||||
type: string
|
||||
required:
|
||||
- storageClass
|
||||
- volumeSnapshot
|
||||
type: object
|
||||
dataMoverConfig:
|
||||
additionalProperties:
|
||||
type: string
|
||||
@@ -106,6 +134,9 @@ spec:
|
||||
OperationTimeout specifies the time used to wait internal operations,
|
||||
before returning error as timeout.
|
||||
type: string
|
||||
restoreType:
|
||||
description: RestoreType indicates the type of the restore.
|
||||
type: string
|
||||
snapshotID:
|
||||
description: SnapshotID is the ID of the Velero backup snapshot to
|
||||
be restored from.
|
||||
@@ -145,6 +176,7 @@ spec:
|
||||
required:
|
||||
- backupStorageLocation
|
||||
- operationTimeout
|
||||
- restoreType
|
||||
- snapshotID
|
||||
- sourceNamespace
|
||||
- targetVolume
|
||||
|
||||
@@ -110,6 +110,10 @@ spec:
|
||||
description: VolumeSnapshot is the name of the volume snapshot
|
||||
to be backed up
|
||||
type: string
|
||||
volumeSnapshotNamespace:
|
||||
description: VolumeSnapshotNamespace is the namespece of the volume
|
||||
snapshot to be backed up
|
||||
type: string
|
||||
required:
|
||||
- storageClass
|
||||
- volumeSnapshot
|
||||
|
||||
@@ -212,7 +212,9 @@ Users must manage the lifecycle of their workloads before starting the restore.
|
||||
When performing an in-place restore, Velero deletes the existing target PVC and recreates it. For StorageClasses using the `WaitForFirstConsumer` volume binding mode, this recreation resets the scheduling lifecycle. Even though Velero adds a selector to the PVC spec to ensure it binds exclusively to the original PV, a scheduling issue can still occur. If the target PVC loses its node affinity, the Kubernetes Scheduler might schedule the recreated business Pod to a different availability zone. Because the original PV is physically constrained to its original zone, the Pod will fail to mount the volume and remain stuck in the `ContainerCreating` state with an attachment error.
|
||||
|
||||
**Solution**:
|
||||
During the PVC Restore Item Action (RIA), Velero must extract the `volume.kubernetes.io/selected-node` annotation from the original PVC. When Velero recreates the target PVC, it must inject this annotation back into the PVC spec.
|
||||
During the PVC CSI Restore Item Action (RIA), right before deleting the existing PVC, Velero extracts the `volume.kubernetes.io/selected-node` annotation from that PVC and carries it on the PVC to be restored via a Velero-internal carrier annotation (`restore.velero.io/inplace-restore-selected-node`). After all Restore Item Actions have run, the restore engine translates the carrier back to the `volume.kubernetes.io/selected-node` annotation and strips the carrier so it never lands on the cluster.
|
||||
|
||||
A carrier annotation is used instead of the Kubernetes annotation directly because the generic PVC RIA unconditionally strips the `selected-node` annotation during restore, and the execution order of Restore Item Actions is not a documented contract. With the carrier, the behavior is independent of the RIA execution order: the Kubernetes annotation is stripped by default on every path (including when the target PVC does not exist and Velero falls back to provisioning a new PVC), and preservation only happens when the PVC CSI RIA explicitly captured a value from the existing PVC.
|
||||
By preserving the `selected-node` annotation, the Kubernetes Scheduler is forced to schedule the recreated business Pod to the original node/zone, ensuring it successfully mounts the restored PV.
|
||||
|
||||
### Namespace Mapping
|
||||
@@ -260,10 +262,8 @@ This section outlines the step-by-step control path and data path workflows for
|
||||
|
||||
**Control Path**
|
||||
|
||||
PVC RIA:
|
||||
- Preserve the `volume.kubernetes.io/selected-node` annotation to ensure correct scheduling during target PVC recreation.
|
||||
|
||||
PVC CSI RIA:
|
||||
- Capture the `volume.kubernetes.io/selected-node` annotation from the existing PVC into the Velero-internal carrier annotation before deleting the PVC, so the restore engine can re-apply it to the recreated target PVC (see [Handling Cross-Zone Scheduling](#handling-cross-zone-scheduling-waitforfirstconsumer)).
|
||||
- Create a snapshot of the existing `PVC` to serve as the baseline for CBT delta calculations.
|
||||
- Patch the existing PV's reclaim policy to `Retain`.
|
||||
- Delete the existing PVC.
|
||||
@@ -308,10 +308,8 @@ The workflow is identical to the **In-place Incremental Restore for CSI Snapshot
|
||||
|
||||
**Control Path**
|
||||
|
||||
PVC RIA:
|
||||
- Preserve the `volume.kubernetes.io/selected-node` annotation to ensure correct scheduling during target PVC recreation.
|
||||
|
||||
PVC CSI RIA:
|
||||
- Capture the `volume.kubernetes.io/selected-node` annotation from the existing PVC into the Velero-internal carrier annotation before deleting the PVC, so the restore engine can re-apply it to the recreated target PVC (see [Handling Cross-Zone Scheduling](#handling-cross-zone-scheduling-waitforfirstconsumer)).
|
||||
- Create a snapshot of the existing `PVC` to serve as the baseline for CBT delta calculations.
|
||||
- Patch the existing `PV` to set its `persistentVolumeReclaimPolicy` to `Retain`.
|
||||
- Delete the existing `PVC`.
|
||||
|
||||
@@ -174,6 +174,17 @@ const (
|
||||
// Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the
|
||||
// annotation is never inspected and AdditionalItems are not processed.
|
||||
MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items"
|
||||
|
||||
// InplaceRestoreSelectedNodeAnnotation is a Velero-internal carrier annotation set by the
|
||||
// PVC CSI RestoreItemAction during an in-place volume data restore. It carries the
|
||||
// "volume.kubernetes.io/selected-node" value captured from the existing PVC right before
|
||||
// that PVC is deleted, so the restore engine can re-apply it to the recreated target PVC
|
||||
// after all RestoreItemActions have run. This keeps the recreated PVC (and the workload
|
||||
// Pod, for WaitForFirstConsumer StorageClasses) scheduled to the original node/zone.
|
||||
// The annotation is always translated and stripped by the restore engine; it never lands
|
||||
// on the cluster. Using a carrier annotation avoids any dependency on the execution order
|
||||
// of RestoreItemActions.
|
||||
InplaceRestoreSelectedNodeAnnotation = "restore.velero.io/inplace-restore-selected-node"
|
||||
// SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to
|
||||
// find out if the snapshot was skipped b/c the PV is not provisioned via CSI
|
||||
SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv"
|
||||
|
||||
@@ -46,6 +46,9 @@ type PodVolumeRestoreSpec struct {
|
||||
// SnapshotID is the ID of the volume snapshot to be restored.
|
||||
SnapshotID string `json:"snapshotID"`
|
||||
|
||||
// RestoreType indicates the type of the restore.
|
||||
RestoreType string `json:"restoreType"`
|
||||
|
||||
// SourceNamespace is the original namespace for namaspace mapping.
|
||||
SourceNamespace string `json:"sourceNamespace"`
|
||||
|
||||
|
||||
@@ -113,7 +113,12 @@ type RestoreSpec struct {
|
||||
// ExistingResourcePolicy specifies the restore behavior for the Kubernetes resource to be restored
|
||||
// +optional
|
||||
// +nullable
|
||||
ExistingResourcePolicy PolicyType `json:"existingResourcePolicy,omitempty"`
|
||||
ExistingResourcePolicy ResourcePolicyType `json:"existingResourcePolicy,omitempty"`
|
||||
|
||||
// ExistingVolumeDataPolicy specifies the restore behavior for the volume data to be restored
|
||||
// +optional
|
||||
// +nullable
|
||||
ExistingVolumeDataPolicy VolumeDataPolicyType `json:"existingVolumeDataPolicy,omitempty"`
|
||||
|
||||
// ItemOperationTimeout specifies the time used to wait for RestoreItemAction operations
|
||||
// The default value is 4 hour.
|
||||
@@ -158,6 +163,11 @@ type UploaderConfigForRestore struct {
|
||||
// ParallelFilesDownload is the concurrency number setting for restore.
|
||||
// +optional
|
||||
ParallelFilesDownload int `json:"parallelFilesDownload,omitempty"`
|
||||
// DeleteExtraFiles specifies whether to delete the extra files in the target volume that do not exist in the backup.
|
||||
// This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores.
|
||||
// +optional
|
||||
// +nullable
|
||||
DeleteExtraFiles *bool `json:"deleteExtraFiles,omitempty"`
|
||||
}
|
||||
|
||||
// RestoreHooks contains custom behaviors that should be executed during or post restore.
|
||||
@@ -324,13 +334,22 @@ const (
|
||||
// The failing error is recorded in status.FailureReason.
|
||||
RestorePhaseFailed RestorePhase = "Failed"
|
||||
|
||||
// PolicyTypeNone means velero will not overwrite the resource
|
||||
// ResourcePolicyTypeNone means velero will not overwrite the resource
|
||||
// in cluster with the one in backup whether changed/unchanged.
|
||||
PolicyTypeNone PolicyType = "none"
|
||||
ResourcePolicyTypeNone ResourcePolicyType = "none"
|
||||
|
||||
// PolicyTypeUpdate means velero will try to attempt a patch on
|
||||
// ResourcePolicyTypeUpdate means velero will try to attempt a patch on
|
||||
// the changed resources.
|
||||
PolicyTypeUpdate PolicyType = "update"
|
||||
ResourcePolicyTypeUpdate ResourcePolicyType = "update"
|
||||
|
||||
// VolumeDataPolicyTypeNone means velero will skip and not overwrite the volume data if the volume already exists
|
||||
VolumeDataPolicyTypeNone VolumeDataPolicyType = "none"
|
||||
|
||||
// VolumeDataPolicyTypeFull means velero will try to restore the volume data fully if the volume already exists.
|
||||
VolumeDataPolicyTypeFull VolumeDataPolicyType = "full"
|
||||
|
||||
// VolumeDataPolicyTypeIncremental means velero will try to restore the volume data incrementally if the volume already exists.
|
||||
VolumeDataPolicyTypeIncremental VolumeDataPolicyType = "incremental"
|
||||
)
|
||||
|
||||
// RestoreStatus captures the current status of a Velero restore
|
||||
@@ -441,6 +460,18 @@ type Restore struct {
|
||||
Status RestoreStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Restore) IsVolumeDataInplaceRestore() bool {
|
||||
return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeFull || r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeIncremental
|
||||
}
|
||||
|
||||
func (r *Restore) IsVolumeDataInplaceFullRestore() bool {
|
||||
return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeFull
|
||||
}
|
||||
|
||||
func (r *Restore) IsVolumeDataInplaceIncrementalRestore() bool {
|
||||
return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeIncremental
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// RestoreList is a list of Restores.
|
||||
@@ -453,5 +484,8 @@ type RestoreList struct {
|
||||
Items []Restore `json:"items"`
|
||||
}
|
||||
|
||||
// PolicyType helps specify the ExistingResourcePolicy
|
||||
type PolicyType string
|
||||
// ResourcePolicyType helps specify the ExistingResourcePolicy
|
||||
type ResourcePolicyType string
|
||||
|
||||
// VolumeDataPolicyType helps specify the ExistingVolumeDataPolicy
|
||||
type VolumeDataPolicyType string
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsVolumeDataInplaceRestore(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
existingVolumeDataPolicy VolumeDataPolicyType
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "empty policy",
|
||||
existingVolumeDataPolicy: "",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "none policy",
|
||||
existingVolumeDataPolicy: VolumeDataPolicyTypeNone,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "full policy",
|
||||
existingVolumeDataPolicy: VolumeDataPolicyTypeFull,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "incremental policy",
|
||||
existingVolumeDataPolicy: VolumeDataPolicyTypeIncremental,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "unknown policy",
|
||||
existingVolumeDataPolicy: VolumeDataPolicyType("unknown"),
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
restore := &Restore{
|
||||
Spec: RestoreSpec{
|
||||
ExistingVolumeDataPolicy: tc.existingVolumeDataPolicy,
|
||||
},
|
||||
}
|
||||
actual := restore.IsVolumeDataInplaceRestore()
|
||||
if actual != tc.expected {
|
||||
t.Errorf("expected %v, got %v", tc.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1771,6 +1771,11 @@ func (in *UploaderConfigForRestore) DeepCopyInto(out *UploaderConfigForRestore)
|
||||
*out = new(bool)
|
||||
**out = **in
|
||||
}
|
||||
if in.DeleteExtraFiles != nil {
|
||||
in, out := &in.DeleteExtraFiles, &out.DeleteExtraFiles
|
||||
*out = new(bool)
|
||||
**out = **in
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForRestore.
|
||||
|
||||
@@ -39,6 +39,14 @@ type DataDownloadSpec struct {
|
||||
// SnapshotID is the ID of the Velero backup snapshot to be restored from.
|
||||
SnapshotID string `json:"snapshotID"`
|
||||
|
||||
// RestoreType indicates the type of the restore.
|
||||
RestoreType string `json:"restoreType"`
|
||||
|
||||
// CSISnapshot provides the information of the CSI snapshot used to do the incremental restore.
|
||||
// +optional
|
||||
// +nullable
|
||||
CSISnapshot *CSISnapshotSpec `json:"csiSnapshot"`
|
||||
|
||||
// SourceNamespace is the original namespace where the volume is backed up from.
|
||||
// It may be different from SourcePVC's namespace if namespace is remapped during restore.
|
||||
SourceNamespace string `json:"sourceNamespace"`
|
||||
|
||||
@@ -80,6 +80,10 @@ const (
|
||||
|
||||
// CSISnapshotSpec is the specification for a CSI snapshot.
|
||||
type CSISnapshotSpec struct {
|
||||
// VolumeSnapshotNamespace is the namespece of the volume snapshot to be backed up
|
||||
// +optional
|
||||
VolumeSnapshotNamespace string `json:"volumeSnapshotNamespace"`
|
||||
|
||||
// VolumeSnapshot is the name of the volume snapshot to be backed up
|
||||
VolumeSnapshot string `json:"volumeSnapshot"`
|
||||
|
||||
|
||||
@@ -86,6 +86,11 @@ func (in *DataDownloadList) DeepCopyObject() runtime.Object {
|
||||
func (in *DataDownloadSpec) DeepCopyInto(out *DataDownloadSpec) {
|
||||
*out = *in
|
||||
out.TargetVolume = in.TargetVolume
|
||||
if in.CSISnapshot != nil {
|
||||
in, out := &in.CSISnapshot, &out.CSISnapshot
|
||||
*out = new(CSISnapshotSpec)
|
||||
**out = **in
|
||||
}
|
||||
if in.DataMoverConfig != nil {
|
||||
in, out := &in.DataMoverConfig, &out.DataMoverConfig
|
||||
*out = make(map[string]string, len(*in))
|
||||
|
||||
@@ -98,7 +98,13 @@ func (b *RestoreBuilder) ExcludedResources(resources ...string) *RestoreBuilder
|
||||
|
||||
// ExistingResourcePolicy sets the Restore's resource policy.
|
||||
func (b *RestoreBuilder) ExistingResourcePolicy(policy string) *RestoreBuilder {
|
||||
b.object.Spec.ExistingResourcePolicy = velerov1api.PolicyType(policy)
|
||||
b.object.Spec.ExistingResourcePolicy = velerov1api.ResourcePolicyType(policy)
|
||||
return b
|
||||
}
|
||||
|
||||
// ExistingVolumeDataPolicy sets the Restore's volume data policy.
|
||||
func (b *RestoreBuilder) ExistingVolumeDataPolicy(policy string) *RestoreBuilder {
|
||||
b.object.Spec.ExistingVolumeDataPolicy = velerov1api.VolumeDataPolicyType(policy)
|
||||
return b
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
|
||||
"github.com/vmware-tanzu/velero/pkg/buildinfo"
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
"github.com/vmware-tanzu/velero/pkg/client"
|
||||
"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
|
||||
"github.com/vmware-tanzu/velero/pkg/datamover"
|
||||
@@ -56,6 +57,9 @@ type dataMoverRestoreConfig struct {
|
||||
ddName string
|
||||
cacheDir string
|
||||
resourceTimeout time.Duration
|
||||
cbtSAName string
|
||||
vsNamespace string
|
||||
volumeID string
|
||||
}
|
||||
|
||||
func NewRestoreCommand(f client.Factory) *cobra.Command {
|
||||
@@ -96,6 +100,9 @@ func NewRestoreCommand(f client.Factory) *cobra.Command {
|
||||
command.Flags().StringVar(&config.ddName, "data-download", config.ddName, "The data download name")
|
||||
command.Flags().StringVar(&config.cacheDir, "cache-volume-path", config.cacheDir, "The full path of the cache volume")
|
||||
command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.")
|
||||
command.Flags().StringVar(&config.cbtSAName, "cbt-sa-name", config.cbtSAName, "The name of the service account used by CSI's CBT service")
|
||||
command.Flags().StringVar(&config.vsNamespace, "vs-namespace", config.vsNamespace, "The namespace of the VolumeSnapshot")
|
||||
command.Flags().StringVar(&config.volumeID, "volume-id", config.volumeID, "The volume ID of the snapshot")
|
||||
|
||||
_ = command.MarkFlagRequired("volume-path")
|
||||
_ = command.MarkFlagRequired("volume-mode")
|
||||
@@ -116,6 +123,7 @@ type dataMoverRestore struct {
|
||||
config dataMoverRestoreConfig
|
||||
kubeClient kubernetes.Interface
|
||||
dataPathMgr *datapath.Manager
|
||||
cbtService cbtservice.Service
|
||||
}
|
||||
|
||||
func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, config dataMoverRestoreConfig) (*dataMoverRestore, error) {
|
||||
@@ -201,6 +209,12 @@ func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, conf
|
||||
config: config,
|
||||
namespace: factory.Namespace(),
|
||||
nodeName: nodeName,
|
||||
cbtService: cbtservice.NewService(
|
||||
logger,
|
||||
config.vsNamespace,
|
||||
config.cbtSAName,
|
||||
clientConfig,
|
||||
),
|
||||
}
|
||||
|
||||
s.kubeClient, err = factory.KubeClient()
|
||||
@@ -294,5 +308,5 @@ func (s *dataMoverRestore) createDataPathService() (dataPathService, error) {
|
||||
return datamover.NewRestoreMicroService(s.ctx, s.client, s.kubeClient, s.config.ddName, s.namespace, s.nodeName, datapath.AccessPoint{
|
||||
ByPath: s.config.volumePath,
|
||||
VolMode: uploader.PersistentVolumeMode(s.config.volumeMode),
|
||||
}, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.config.cacheDir, s.logger), nil
|
||||
}, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.config.cacheDir, s.config.volumeID, s.cbtService, s.logger), nil
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
"github.com/bombsimon/logrusr/v3"
|
||||
"github.com/cockroachdb/errors"
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -175,6 +176,10 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
|
||||
cancelFunc()
|
||||
return nil, err
|
||||
}
|
||||
if err := snapshotv1api.AddToScheme(scheme); err != nil {
|
||||
cancelFunc()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
|
||||
@@ -484,6 +489,7 @@ func (s *nodeAgentServer) run() {
|
||||
s.repoConfigMgr,
|
||||
podLabels,
|
||||
podAnnotations,
|
||||
csiSnapshotMetadataServiceConfigs,
|
||||
)
|
||||
|
||||
if err := dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
|
||||
|
||||
@@ -99,6 +99,7 @@ type CreateOptions struct {
|
||||
IncludeNamespaces flag.StringArray
|
||||
ExcludeNamespaces flag.StringArray
|
||||
ExistingResourcePolicy string
|
||||
ExistingVolumeDataPolicy string
|
||||
IncludeResources flag.StringArray
|
||||
ExcludeResources flag.StringArray
|
||||
StatusIncludeResources flag.StringArray
|
||||
@@ -115,6 +116,7 @@ type CreateOptions struct {
|
||||
SkipDefaultResourceModifier bool
|
||||
WriteSparseFiles flag.OptionalBool
|
||||
ParallelFilesDownload int
|
||||
DeleteExtraFiles flag.OptionalBool
|
||||
client kbclient.WithWatch
|
||||
}
|
||||
|
||||
@@ -128,6 +130,7 @@ func NewCreateOptions() *CreateOptions {
|
||||
PreserveNodePorts: flag.NewOptionalBool(nil),
|
||||
IncludeClusterResources: flag.NewOptionalBool(nil),
|
||||
WriteSparseFiles: flag.NewOptionalBool(nil),
|
||||
DeleteExtraFiles: flag.NewOptionalBool(nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,7 +144,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
|
||||
flags.Var(&o.Annotations, "annotations", "Annotations to apply to the restore.")
|
||||
flags.Var(&o.IncludeResources, "include-resources", "Resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources).")
|
||||
flags.Var(&o.ExcludeResources, "exclude-resources", "Resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io.")
|
||||
flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow, can be - none or update")
|
||||
flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow for Kubernetes resources, can be - none or update")
|
||||
flags.StringVar(&o.ExistingVolumeDataPolicy, "existing-volume-data-policy", "", "Restore Policy to be used during the restore workflow for volume data, can be - none, full or incremental")
|
||||
flags.Var(&o.StatusIncludeResources, "status-include-resources", "Resources to include in the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.")
|
||||
flags.Var(&o.StatusExcludeResources, "status-exclude-resources", "Resources to exclude from the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.")
|
||||
flags.VarP(&o.Selector, "selector", "l", "Only restore resources matching this label selector.")
|
||||
@@ -175,6 +179,9 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
|
||||
f.NoOptDefVal = cmd.TRUE
|
||||
|
||||
flags.IntVar(&o.ParallelFilesDownload, "parallel-files-download", 0, "The number of restore operations to run in parallel. If set to 0, the default parallelism will be the number of CPUs for the node that node agent pod is running.")
|
||||
|
||||
f = flags.VarPF(&o.DeleteExtraFiles, "delete-extra-files", "", "Whether to delete extra files in the target volume that do not exist in the backup during file system restore. This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores.")
|
||||
f.NoOptDefVal = cmd.TRUE
|
||||
}
|
||||
|
||||
func (o *CreateOptions) Complete(args []string, f client.Factory) error {
|
||||
@@ -224,6 +231,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto
|
||||
return errors.New("existing-resource-policy has invalid value, it accepts only none, update as value")
|
||||
}
|
||||
|
||||
if len(o.ExistingVolumeDataPolicy) > 0 && !restore.IsVolumeDataPolicyValid(o.ExistingVolumeDataPolicy) {
|
||||
return errors.New("existing-volume-data-policy has invalid value, it accepts only none, full, incremental as value")
|
||||
}
|
||||
|
||||
if o.ParallelFilesDownload < 0 {
|
||||
return errors.New("parallel-files-download cannot be negative")
|
||||
}
|
||||
@@ -344,27 +355,29 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
|
||||
Annotations: o.Annotations.Data(),
|
||||
},
|
||||
Spec: api.RestoreSpec{
|
||||
BackupName: o.BackupName,
|
||||
ScheduleName: o.ScheduleName,
|
||||
IncludedNamespaces: o.IncludeNamespaces,
|
||||
ExcludedNamespaces: o.ExcludeNamespaces,
|
||||
IncludedResources: o.IncludeResources,
|
||||
ExcludedResources: o.ExcludeResources,
|
||||
ExistingResourcePolicy: api.PolicyType(o.ExistingResourcePolicy),
|
||||
NamespaceMapping: o.NamespaceMappings.Data(),
|
||||
LabelSelector: o.Selector.LabelSelector,
|
||||
OrLabelSelectors: o.OrSelector.OrLabelSelectors,
|
||||
RestorePVs: o.RestoreVolumes.Value,
|
||||
PreserveNodePorts: o.PreserveNodePorts.Value,
|
||||
IncludeClusterResources: o.IncludeClusterResources.Value,
|
||||
ResourceModifier: resModifiers,
|
||||
ResourcePolicy: resPolicies,
|
||||
BackupName: o.BackupName,
|
||||
ScheduleName: o.ScheduleName,
|
||||
IncludedNamespaces: o.IncludeNamespaces,
|
||||
ExcludedNamespaces: o.ExcludeNamespaces,
|
||||
IncludedResources: o.IncludeResources,
|
||||
ExcludedResources: o.ExcludeResources,
|
||||
ExistingResourcePolicy: api.ResourcePolicyType(o.ExistingResourcePolicy),
|
||||
ExistingVolumeDataPolicy: api.VolumeDataPolicyType(o.ExistingVolumeDataPolicy),
|
||||
NamespaceMapping: o.NamespaceMappings.Data(),
|
||||
LabelSelector: o.Selector.LabelSelector,
|
||||
OrLabelSelectors: o.OrSelector.OrLabelSelectors,
|
||||
RestorePVs: o.RestoreVolumes.Value,
|
||||
PreserveNodePorts: o.PreserveNodePorts.Value,
|
||||
IncludeClusterResources: o.IncludeClusterResources.Value,
|
||||
ResourceModifier: resModifiers,
|
||||
ResourcePolicy: resPolicies,
|
||||
ItemOperationTimeout: metav1.Duration{
|
||||
Duration: o.ItemOperationTimeout,
|
||||
},
|
||||
UploaderConfig: &api.UploaderConfigForRestore{
|
||||
WriteSparseFiles: o.WriteSparseFiles.Value,
|
||||
ParallelFilesDownload: o.ParallelFilesDownload,
|
||||
DeleteExtraFiles: o.DeleteExtraFiles.Value,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -68,6 +68,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
includeNamespaces := "app1,app2"
|
||||
excludeNamespaces := "pod1,pod2,pod3"
|
||||
existingResourcePolicy := "none"
|
||||
existingVolumeDataPolicy := "none"
|
||||
includeResources := "sc,sts"
|
||||
excludeResources := "job"
|
||||
statusIncludeResources := "sc,sts"
|
||||
@@ -80,6 +81,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
resourceModifierConfigMap := "modifier-cm"
|
||||
ResourcePoliciesConfigMap := "policies-cm"
|
||||
writeSparseFiles := "true"
|
||||
deleteExtraFiles := "true"
|
||||
parallel := 2
|
||||
flags := new(pflag.FlagSet)
|
||||
o := NewCreateOptions()
|
||||
@@ -92,6 +94,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
flags.Parse([]string{"--labels", labels})
|
||||
flags.Parse([]string{"--annotations", annotations})
|
||||
flags.Parse([]string{"--existing-resource-policy", existingResourcePolicy})
|
||||
flags.Parse([]string{"--existing-volume-data-policy", existingVolumeDataPolicy})
|
||||
flags.Parse([]string{"--include-namespaces", includeNamespaces})
|
||||
flags.Parse([]string{"--exclude-namespaces", excludeNamespaces})
|
||||
flags.Parse([]string{"--include-resources", includeResources})
|
||||
@@ -107,6 +110,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap})
|
||||
flags.Parse([]string{"--skip-default-resource-modifier"})
|
||||
flags.Parse([]string{"--write-sparse-files", writeSparseFiles})
|
||||
flags.Parse([]string{"--delete-extra-files", deleteExtraFiles})
|
||||
flags.Parse([]string{"--parallel-files-download", "2"})
|
||||
client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch)
|
||||
|
||||
@@ -134,6 +138,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
require.Equal(t, includeNamespaces, o.IncludeNamespaces.String())
|
||||
require.Equal(t, excludeNamespaces, o.ExcludeNamespaces.String())
|
||||
require.Equal(t, existingResourcePolicy, o.ExistingResourcePolicy)
|
||||
require.Equal(t, existingVolumeDataPolicy, o.ExistingVolumeDataPolicy)
|
||||
require.Equal(t, includeResources, o.IncludeResources.String())
|
||||
require.Equal(t, excludeResources, o.ExcludeResources.String())
|
||||
|
||||
@@ -149,6 +154,7 @@ func TestCreateCommand(t *testing.T) {
|
||||
require.True(t, o.SkipDefaultResourceModifier)
|
||||
require.Equal(t, writeSparseFiles, o.WriteSparseFiles.String())
|
||||
require.Equal(t, parallel, o.ParallelFilesDownload)
|
||||
require.Equal(t, deleteExtraFiles, o.DeleteExtraFiles.String())
|
||||
})
|
||||
|
||||
t.Run("create a restore from schedule", func(t *testing.T) {
|
||||
|
||||
@@ -59,27 +59,28 @@ import (
|
||||
|
||||
// DataDownloadReconciler reconciles a DataDownload object
|
||||
type DataDownloadReconciler struct {
|
||||
client client.Client
|
||||
kubeClient kubernetes.Interface
|
||||
mgr manager.Manager
|
||||
logger logrus.FieldLogger
|
||||
Clock clock.WithTickerAndDelayedExecution
|
||||
restoreExposer exposer.GenericRestoreExposer
|
||||
nodeName string
|
||||
dataPathMgr *datapath.Manager
|
||||
vgdpCounter *exposer.VgdpCounter
|
||||
loadAffinity []*kube.LoadAffinity
|
||||
restorePVCConfig velerotypes.RestorePVC
|
||||
backupRepoConfigs map[string]string
|
||||
cacheVolumeConfigs *velerotypes.CachePVC
|
||||
podResources corev1api.ResourceRequirements
|
||||
preparingTimeout time.Duration
|
||||
metrics *metrics.ServerMetrics
|
||||
cancelledDataDownload sync.Map
|
||||
dataMovePriorityClass string
|
||||
repoConfigMgr repository.ConfigManager
|
||||
podLabels map[string]string
|
||||
podAnnotations map[string]string
|
||||
client client.Client
|
||||
kubeClient kubernetes.Interface
|
||||
mgr manager.Manager
|
||||
logger logrus.FieldLogger
|
||||
Clock clock.WithTickerAndDelayedExecution
|
||||
restoreExposer exposer.GenericRestoreExposer
|
||||
nodeName string
|
||||
dataPathMgr *datapath.Manager
|
||||
vgdpCounter *exposer.VgdpCounter
|
||||
loadAffinity []*kube.LoadAffinity
|
||||
restorePVCConfig velerotypes.RestorePVC
|
||||
backupRepoConfigs map[string]string
|
||||
cacheVolumeConfigs *velerotypes.CachePVC
|
||||
podResources corev1api.ResourceRequirements
|
||||
preparingTimeout time.Duration
|
||||
metrics *metrics.ServerMetrics
|
||||
cancelledDataDownload sync.Map
|
||||
dataMovePriorityClass string
|
||||
repoConfigMgr repository.ConfigManager
|
||||
podLabels map[string]string
|
||||
podAnnotations map[string]string
|
||||
snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
}
|
||||
|
||||
func NewDataDownloadReconciler(
|
||||
@@ -101,28 +102,30 @@ func NewDataDownloadReconciler(
|
||||
repoConfigMgr repository.ConfigManager,
|
||||
podLabels map[string]string,
|
||||
podAnnotations map[string]string,
|
||||
snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService,
|
||||
) *DataDownloadReconciler {
|
||||
return &DataDownloadReconciler{
|
||||
client: client,
|
||||
kubeClient: kubeClient,
|
||||
mgr: mgr,
|
||||
logger: logger.WithField("controller", "DataDownload"),
|
||||
Clock: &clock.RealClock{},
|
||||
nodeName: nodeName,
|
||||
restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, logger),
|
||||
restorePVCConfig: restorePVCConfig,
|
||||
backupRepoConfigs: backupRepoConfigs,
|
||||
cacheVolumeConfigs: cacheVolumeConfigs,
|
||||
dataPathMgr: dataPathMgr,
|
||||
vgdpCounter: counter,
|
||||
loadAffinity: loadAffinity,
|
||||
podResources: podResources,
|
||||
preparingTimeout: preparingTimeout,
|
||||
metrics: metrics,
|
||||
dataMovePriorityClass: dataMovePriorityClass,
|
||||
repoConfigMgr: repoConfigMgr,
|
||||
podLabels: podLabels,
|
||||
podAnnotations: podAnnotations,
|
||||
client: client,
|
||||
kubeClient: kubeClient,
|
||||
mgr: mgr,
|
||||
logger: logger.WithField("controller", "DataDownload"),
|
||||
Clock: &clock.RealClock{},
|
||||
nodeName: nodeName,
|
||||
restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, client, logger),
|
||||
restorePVCConfig: restorePVCConfig,
|
||||
backupRepoConfigs: backupRepoConfigs,
|
||||
cacheVolumeConfigs: cacheVolumeConfigs,
|
||||
dataPathMgr: dataPathMgr,
|
||||
vgdpCounter: counter,
|
||||
loadAffinity: loadAffinity,
|
||||
podResources: podResources,
|
||||
preparingTimeout: preparingTimeout,
|
||||
metrics: metrics,
|
||||
dataMovePriorityClass: dataMovePriorityClass,
|
||||
repoConfigMgr: repoConfigMgr,
|
||||
podLabels: podLabels,
|
||||
podAnnotations: podAnnotations,
|
||||
snapshotMetadataServiceConfigs: snapshotMetadataServiceConfigs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -488,7 +491,9 @@ func (r *DataDownloadReconciler) OnDataDownloadCompleted(ctx context.Context, na
|
||||
}
|
||||
|
||||
log.Info("Cleaning up exposed environment")
|
||||
r.restoreExposer.CleanUp(ctx, objRef)
|
||||
r.restoreExposer.CleanUp(ctx, objRef, &exposer.GenericRestoreCleanUpParam{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
})
|
||||
|
||||
if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool {
|
||||
if isDataDownloadInFinalState(dd) {
|
||||
@@ -537,7 +542,9 @@ func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, na
|
||||
return
|
||||
}
|
||||
// cleans up any objects generated during the snapshot expose
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(&dd))
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(&dd), &exposer.GenericRestoreCleanUpParam{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
})
|
||||
|
||||
if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool {
|
||||
if isDataDownloadInFinalState(dd) {
|
||||
@@ -587,7 +594,9 @@ func (r *DataDownloadReconciler) tryCancelDataDownload(ctx context.Context, dd *
|
||||
|
||||
// success update
|
||||
r.metrics.RegisterDataDownloadCancel(r.nodeName)
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd))
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
})
|
||||
|
||||
log.Warn("data download is canceled")
|
||||
|
||||
@@ -735,7 +744,9 @@ func (r *DataDownloadReconciler) prepareDataDownload(ssb *velerov2alpha1api.Data
|
||||
|
||||
func (r *DataDownloadReconciler) errorOut(ctx context.Context, dd *velerov2alpha1api.DataDownload, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) {
|
||||
if r.restoreExposer != nil {
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd))
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
})
|
||||
}
|
||||
return ctrl.Result{}, r.updateStatusToFailed(ctx, dd, err, msg, log)
|
||||
}
|
||||
@@ -825,7 +836,9 @@ func (r *DataDownloadReconciler) onPrepareTimeout(ctx context.Context, dd *veler
|
||||
log.Warnf("[Diagnose DD expose]%s", diag)
|
||||
}
|
||||
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd))
|
||||
r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
})
|
||||
|
||||
log.Info("Datadownload has been cleaned up")
|
||||
|
||||
@@ -937,6 +950,7 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown
|
||||
|
||||
return exposer.GenericRestoreExposeParam{
|
||||
TargetPVCName: dd.Spec.TargetVolume.PVC,
|
||||
TargetPVName: dd.Spec.TargetVolume.PV,
|
||||
TargetNamespace: dd.Spec.TargetVolume.Namespace,
|
||||
HostingPodLabels: hostingPodLabels,
|
||||
HostingPodAnnotations: hostingPodAnnotation,
|
||||
@@ -951,6 +965,10 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown
|
||||
RestoreSize: dd.Spec.SnapshotSize,
|
||||
CacheVolume: cacheVolume,
|
||||
DataMover: dd.Spec.DataMover,
|
||||
CSI: &exposer.GenericRestoreExposeCSI{
|
||||
Snapshot: dd.Spec.CSISnapshot,
|
||||
SnapshotMetadataServiceConfigs: r.snapshotMetadataServiceConfigs,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,6 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder {
|
||||
BackupStorageLocation("bsl-loc").
|
||||
DataMover("velero").
|
||||
SnapshotID("test-snapshot-id").TargetVolume(velerov2alpha1api.TargetVolumeSpec{
|
||||
PV: "test-pv",
|
||||
PVC: "test-pvc",
|
||||
Namespace: "test-ns",
|
||||
})
|
||||
@@ -151,6 +150,7 @@ func initDataDownloadReconcilerWithError(t *testing.T, objects []any, needError
|
||||
nil,
|
||||
nil, // podLabels
|
||||
nil, // podAnnotations
|
||||
nil, // snapshotMetadataServiceConfigs
|
||||
), nil
|
||||
}
|
||||
|
||||
@@ -186,6 +186,7 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
dd *velerov2alpha1api.DataDownload
|
||||
notCreateDD bool
|
||||
targetPVC *corev1api.PersistentVolumeClaim
|
||||
targetPV *corev1api.PersistentVolume
|
||||
dataMgr *datapath.Manager
|
||||
needErrs []bool
|
||||
needCreateFSBR bool
|
||||
@@ -197,6 +198,7 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
isPeekExposeErr bool
|
||||
isNilExposer bool
|
||||
notNilExpose bool
|
||||
mockExpose bool
|
||||
notMockCleanUp bool
|
||||
mockInit bool
|
||||
mockInitErr error
|
||||
@@ -354,6 +356,16 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("sc").Result(),
|
||||
expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(),
|
||||
},
|
||||
{
|
||||
name: "dd succeeds for accepted with target PV set",
|
||||
dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).TargetVolume(velerov2alpha1api.TargetVolumeSpec{PVC: "test-pvc", Namespace: "test-ns", PV: "test-pv"}).Result(),
|
||||
targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("sc").Result(),
|
||||
targetPV: builder.ForPersistentVolume("test-pv").Result(),
|
||||
expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).TargetVolume(velerov2alpha1api.TargetVolumeSpec{PVC: "test-pvc", Namespace: "test-ns", PV: "test-pv"}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(),
|
||||
mockExpose: true,
|
||||
notMockCleanUp: true,
|
||||
notNilExpose: true,
|
||||
},
|
||||
{
|
||||
name: "prepare timeout on accepted",
|
||||
dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Finalizers([]string{DataUploadDownloadFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(),
|
||||
@@ -490,6 +502,10 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
objects = append(objects, test.targetPVC)
|
||||
}
|
||||
|
||||
if test.targetPV != nil {
|
||||
objects = append(objects, test.targetPV)
|
||||
}
|
||||
|
||||
r, err := initDataDownloadReconciler(t, objects, test.needErrs...)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -546,7 +562,7 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
return asyncBR
|
||||
}
|
||||
|
||||
if test.isExposeErr || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose {
|
||||
if test.isExposeErr || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose || test.mockExpose {
|
||||
if test.isNilExposer {
|
||||
r.restoreExposer = nil
|
||||
} else {
|
||||
@@ -554,6 +570,8 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
ep := exposermockes.NewGenericRestoreExposer(t)
|
||||
if test.isExposeErr {
|
||||
ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer"))
|
||||
} else if test.mockExpose {
|
||||
ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
} else if test.notNilExpose {
|
||||
hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result()
|
||||
hostingPod.ObjectMeta.SetUID("test-uid")
|
||||
@@ -568,7 +586,7 @@ func TestDataDownloadReconcile(t *testing.T) {
|
||||
}
|
||||
|
||||
if !test.notMockCleanUp {
|
||||
ep.On("CleanUp", mock.Anything, mock.Anything).Return()
|
||||
ep.On("CleanUp", mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
}
|
||||
return ep
|
||||
}()
|
||||
@@ -727,7 +745,7 @@ func TestOnDataDownloadCompleted(t *testing.T) {
|
||||
} else {
|
||||
ep.On("RebindVolume", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
}
|
||||
ep.On("CleanUp", mock.Anything, mock.Anything).Return()
|
||||
ep.On("CleanUp", mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
return ep
|
||||
}()
|
||||
|
||||
@@ -1105,7 +1123,8 @@ func (dt *ddResumeTestHelper) RebindVolume(context.Context, corev1api.ObjectRefe
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dt *ddResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference) {}
|
||||
func (dt *ddResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference, *exposer.GenericRestoreCleanUpParam) {
|
||||
}
|
||||
|
||||
func (dt *ddResumeTestHelper) newMicroServiceBRWatcher(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string,
|
||||
datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
@@ -1328,6 +1347,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) {
|
||||
|
||||
baseDataDownload := dataDownloadBuilder().Result()
|
||||
baseDataDownload.Namespace = velerov1api.DefaultNamespace
|
||||
baseDataDownload.Spec.TargetVolume.PV = "pv-1"
|
||||
baseDataDownload.Spec.OperationTimeout = metav1.Duration{Duration: time.Minute * 10}
|
||||
baseDataDownload.Spec.SnapshotSize = 5368709120 // 5Gi
|
||||
|
||||
@@ -1427,6 +1447,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) {
|
||||
nil, // repoConfigMgr (unused when cacheVolumeConfigs is nil)
|
||||
tt.args.customLabels,
|
||||
tt.args.customAnnotations,
|
||||
nil,
|
||||
)
|
||||
|
||||
// Act
|
||||
@@ -1437,6 +1458,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) {
|
||||
|
||||
// Core fields
|
||||
assert.Equal(t, baseDataDownload.Spec.TargetVolume.PVC, got.TargetPVCName)
|
||||
assert.Equal(t, baseDataDownload.Spec.TargetVolume.PV, got.TargetPVName)
|
||||
assert.Equal(t, baseDataDownload.Spec.TargetVolume.Namespace, got.TargetNamespace)
|
||||
assert.Equal(t, baseDataDownload.Spec.DataMover, got.DataMover)
|
||||
|
||||
|
||||
@@ -365,10 +365,15 @@ func (r *restoreReconciler) validateAndComplete(ctx context.Context, restore *ap
|
||||
}
|
||||
|
||||
// validate ExistingResourcePolicy
|
||||
if restore.Spec.ExistingResourcePolicy != "" && !pkgrestoreUtil.IsResourcePolicyValid(string(restore.Spec.ExistingResourcePolicy)) {
|
||||
if !pkgrestoreUtil.IsResourcePolicyValid(string(restore.Spec.ExistingResourcePolicy)) {
|
||||
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid ExistingResourcePolicy: %s", restore.Spec.ExistingResourcePolicy))
|
||||
}
|
||||
|
||||
// validate ExistingVolumeDataPolicy
|
||||
if !pkgrestoreUtil.IsVolumeDataPolicyValid(string(restore.Spec.ExistingVolumeDataPolicy)) {
|
||||
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid ExistingVolumeDataPolicy: %s", restore.Spec.ExistingVolumeDataPolicy))
|
||||
}
|
||||
|
||||
// if ScheduleName is specified, fill in BackupName with the most recent successful backup from
|
||||
// the schedule
|
||||
if restore.Spec.ScheduleName != "" {
|
||||
|
||||
@@ -350,6 +350,39 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
expectedCompletedTime: ×tamp,
|
||||
expectedRestorerCall: nil, // this restore should fail validation and not be passed to the restorer
|
||||
},
|
||||
{
|
||||
name: "valid restore with update existingvolumedatapolicy(full) gets executed",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("full").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
expectedCompletedTime: ×tamp,
|
||||
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingVolumeDataPolicy("full").Result(),
|
||||
},
|
||||
{
|
||||
name: "valid restore with update existingvolumedatapolicy(incremental) gets executed",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("incremental").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
expectedCompletedTime: ×tamp,
|
||||
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingVolumeDataPolicy("incremental").Result(),
|
||||
},
|
||||
{
|
||||
name: "invalid restore with invalid existingvolumedatapolicy errors",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "invalidexistingvolumedatapolicy", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("invalid").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseFailedValidation),
|
||||
expectedStartTime: ×tamp,
|
||||
expectedCompletedTime: ×tamp,
|
||||
expectedRestorerCall: nil, // this restore should fail validation and not be passed to the restorer
|
||||
},
|
||||
{
|
||||
name: "valid restore gets executed",
|
||||
location: defaultStorageLocation,
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
"github.com/vmware-tanzu/velero/pkg/datapath"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository"
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader"
|
||||
@@ -62,11 +63,14 @@ type RestoreMicroService struct {
|
||||
ddHandler cachetool.ResourceEventHandlerRegistration
|
||||
nodeName string
|
||||
cacheDir string
|
||||
|
||||
volumeID string
|
||||
cbtService cbtservice.Service
|
||||
}
|
||||
|
||||
func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataDownloadName string, namespace string, nodeName string,
|
||||
sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter,
|
||||
ddInformer cache.Informer, cacheDir string, log logrus.FieldLogger) *RestoreMicroService {
|
||||
ddInformer cache.Informer, cacheDir string, volumeID string, cbtService cbtservice.Service, log logrus.FieldLogger) *RestoreMicroService {
|
||||
return &RestoreMicroService{
|
||||
ctx: ctx,
|
||||
client: client,
|
||||
@@ -82,6 +86,8 @@ func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClien
|
||||
resultSignal: make(chan dataPathResult),
|
||||
ddInformer: ddInformer,
|
||||
cacheDir: cacheDir,
|
||||
volumeID: volumeID,
|
||||
cbtService: cbtService,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +186,16 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
}
|
||||
log.Info("Async br init")
|
||||
|
||||
if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig, &datapath.RestoreStartParam{}); err != nil {
|
||||
param := &datapath.RestoreStartParam{
|
||||
Incremental: dd.Spec.RestoreType == string(velerov1api.VolumeDataPolicyTypeIncremental),
|
||||
CBTService: r.cbtService,
|
||||
}
|
||||
if dd.Spec.CSISnapshot != nil {
|
||||
param.VolumeSnapshotNamespace = dd.Spec.CSISnapshot.VolumeSnapshotNamespace
|
||||
param.VolumeSnapshotName = dd.Spec.CSISnapshot.VolumeSnapshot
|
||||
param.VolumeID = r.volumeID
|
||||
}
|
||||
if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig, param); err != nil {
|
||||
return "", errors.Wrap(err, "error starting data path restore")
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,11 @@ type BackupStartParam struct {
|
||||
|
||||
// RestoreStartParam define the input param for restore start
|
||||
type RestoreStartParam struct {
|
||||
Incremental bool
|
||||
VolumeSnapshotNamespace string
|
||||
VolumeSnapshotName string
|
||||
VolumeID string
|
||||
CBTService cbtservice.Service
|
||||
}
|
||||
|
||||
type generalDataPath struct {
|
||||
@@ -235,6 +240,8 @@ func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, u
|
||||
|
||||
dp.wgDataPath.Add(1)
|
||||
|
||||
restoreParam := param.(*RestoreStartParam)
|
||||
|
||||
go func() {
|
||||
dp.log.Info("Start data path restore")
|
||||
|
||||
@@ -243,7 +250,14 @@ func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, u
|
||||
dp.wgDataPath.Done()
|
||||
}()
|
||||
|
||||
totalBytes, err := dp.uploaderProv.RunRestore(dp.ctx, snapshotID, target.ByPath, target.VolMode, uploaderConfigs, dp)
|
||||
totalBytes, err := dp.uploaderProv.RunRestore(dp.ctx, snapshotID, target.ByPath, restoreParam.Incremental,
|
||||
provider.CBTParam{
|
||||
Source: cbtservice.SourceInfo{
|
||||
Snapshot: restoreParam.VolumeSnapshotName,
|
||||
VolumeID: restoreParam.VolumeID,
|
||||
},
|
||||
Service: restoreParam.CBTService,
|
||||
}, target.VolMode, uploaderConfigs, dp)
|
||||
|
||||
if err == provider.ErrorCanceled {
|
||||
dp.callbacks.OnCancelled(context.Background(), dp.namespace, dp.jobName)
|
||||
|
||||
@@ -190,7 +190,7 @@ func TestAsyncRestore(t *testing.T) {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dp := newGeneralDataPath("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*generalDataPath)
|
||||
mockProvider := providerMock.NewProvider(t)
|
||||
mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err)
|
||||
mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err)
|
||||
mockProvider.On("Close", mock.Anything).Return(nil)
|
||||
dp.uploaderProv = mockProvider
|
||||
dp.initialized = true
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
@@ -116,12 +115,6 @@ type CSISnapshotExposeWaitParam struct {
|
||||
NodeName string
|
||||
}
|
||||
|
||||
type cbtInfo struct {
|
||||
changeID string
|
||||
volumeID string
|
||||
snapshotID string
|
||||
}
|
||||
|
||||
// NewCSISnapshotExposer create a new instance of CSI snapshot exposer
|
||||
func NewCSISnapshotExposer(kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) SnapshotExposer {
|
||||
return &csiSnapshotExposer{
|
||||
@@ -299,9 +292,9 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
|
||||
|
||||
affinity := kube.GetLoadAffinityByStorageClass(csiExposeParam.Affinity, backupPVCStorageClass, curLog)
|
||||
|
||||
var cbtInfo cbtInfo
|
||||
var cbtInfo csi.CBTInfo
|
||||
if csiExposeParam.DataMover == datamover.DataMoverTypeVeleroBlock {
|
||||
cbtInfo, err = e.getCBTInfo(ctx, backupVS, backupVSC, csiExposeParam.SourcePVName)
|
||||
cbtInfo, err = csi.GetCBTInfo(ctx, e.kubeClient, e.log, backupVS, backupVSC, csiExposeParam.SourcePVName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error to get CBT info")
|
||||
}
|
||||
@@ -341,49 +334,6 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *csiSnapshotExposer) getCBTInfo(ctx context.Context, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (cbtInfo, error) {
|
||||
cbtInfo := cbtInfo{}
|
||||
if vs == nil || vsc == nil {
|
||||
return cbtInfo, errors.New("vs or vsc is nil")
|
||||
}
|
||||
|
||||
cbtInfo.snapshotID = vs.Name
|
||||
|
||||
if vs.Annotations != nil &&
|
||||
(vs.Annotations[util.VSphereCNSChangeIDAnno] != "" ||
|
||||
vs.Annotations[util.VSphereCNSSnapshotAnno] != "") {
|
||||
cbtInfo.changeID = vs.Annotations[util.VSphereCNSChangeIDAnno]
|
||||
|
||||
splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+")
|
||||
if len(splitSnapshotAnno) >= 2 {
|
||||
cbtInfo.volumeID = splitSnapshotAnno[0]
|
||||
}
|
||||
|
||||
e.log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.volumeID, cbtInfo.changeID)
|
||||
} else {
|
||||
pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err)
|
||||
}
|
||||
|
||||
if vsc.Status != nil && vsc.Status.SnapshotHandle != nil {
|
||||
cbtInfo.changeID = *vsc.Status.SnapshotHandle
|
||||
}
|
||||
|
||||
if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" {
|
||||
cbtInfo.volumeID = pv.Spec.CSI.VolumeHandle
|
||||
}
|
||||
|
||||
e.log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.volumeID, cbtInfo.changeID)
|
||||
}
|
||||
|
||||
if cbtInfo.volumeID == "" {
|
||||
return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT")
|
||||
}
|
||||
|
||||
return cbtInfo, nil
|
||||
}
|
||||
|
||||
func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, timeout time.Duration, param any) (*ExposeResult, error) {
|
||||
exposeWaitParam := param.(*CSISnapshotExposeWaitParam)
|
||||
|
||||
@@ -720,7 +670,7 @@ func (e *csiSnapshotExposer) createBackupPod(
|
||||
intoleratableNodes []string,
|
||||
volumeTopology *corev1api.NodeSelector,
|
||||
csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService,
|
||||
cbtInfo *cbtInfo,
|
||||
cbtInfo *csi.CBTInfo,
|
||||
) (*corev1api.Pod, error) {
|
||||
podName := ownerObject.Name
|
||||
|
||||
@@ -776,9 +726,9 @@ func (e *csiSnapshotExposer) createBackupPod(
|
||||
}
|
||||
|
||||
if cbtInfo != nil {
|
||||
args = append(args, fmt.Sprintf("--change-id=%s", cbtInfo.changeID))
|
||||
args = append(args, fmt.Sprintf("--volume-id=%s", cbtInfo.volumeID))
|
||||
args = append(args, fmt.Sprintf("--snapshot-id=%s", cbtInfo.snapshotID))
|
||||
args = append(args, fmt.Sprintf("--change-id=%s", cbtInfo.ChangeID))
|
||||
args = append(args, fmt.Sprintf("--volume-id=%s", cbtInfo.VolumeID))
|
||||
args = append(args, fmt.Sprintf("--snapshot-id=%s", cbtInfo.SnapshotID))
|
||||
}
|
||||
|
||||
args = append(args, podInfo.logFormatArgs...)
|
||||
|
||||
@@ -47,6 +47,7 @@ import (
|
||||
velerotypes "github.com/vmware-tanzu/velero/pkg/types"
|
||||
"github.com/vmware-tanzu/velero/pkg/util"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/csi"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/datamover"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/kube"
|
||||
)
|
||||
@@ -1326,6 +1327,9 @@ func TestGetExpose(t *testing.T) {
|
||||
Spec: corev1api.PersistentVolumeClaimSpec{
|
||||
VolumeName: "fake-pv-name",
|
||||
},
|
||||
Status: corev1api.PersistentVolumeClaimStatus{
|
||||
Phase: corev1api.ClaimBound,
|
||||
},
|
||||
}
|
||||
|
||||
backupPV := &corev1api.PersistentVolume{
|
||||
@@ -2213,7 +2217,7 @@ func TestGetCBTInfo(t *testing.T) {
|
||||
vsc *snapshotv1api.VolumeSnapshotContent
|
||||
pv *corev1api.PersistentVolume
|
||||
sourcePVName string
|
||||
want cbtInfo
|
||||
want csi.CBTInfo
|
||||
wantErrSubstr string
|
||||
}{
|
||||
{
|
||||
@@ -2236,10 +2240,10 @@ func TestGetCBTInfo(t *testing.T) {
|
||||
},
|
||||
vsc: &snapshotv1api.VolumeSnapshotContent{},
|
||||
sourcePVName: "pv-ignored",
|
||||
want: cbtInfo{
|
||||
changeID: "change-id-1",
|
||||
volumeID: "volume-id-1",
|
||||
snapshotID: "vs-anno",
|
||||
want: csi.CBTInfo{
|
||||
ChangeID: "change-id-1",
|
||||
VolumeID: "volume-id-1",
|
||||
SnapshotID: "vs-anno",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -2263,10 +2267,10 @@ func TestGetCBTInfo(t *testing.T) {
|
||||
},
|
||||
},
|
||||
sourcePVName: "pv-1",
|
||||
want: cbtInfo{
|
||||
changeID: "snapshot-handle-1",
|
||||
volumeID: "csi-volume-handle-1",
|
||||
snapshotID: "vs-fallback",
|
||||
want: csi.CBTInfo{
|
||||
ChangeID: "snapshot-handle-1",
|
||||
VolumeID: "csi-volume-handle-1",
|
||||
SnapshotID: "vs-fallback",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -2329,7 +2333,7 @@ func TestGetCBTInfo(t *testing.T) {
|
||||
log: logrus.StandardLogger(),
|
||||
}
|
||||
|
||||
got, err := exposer.getCBTInfo(context.Background(), tc.vs, tc.vsc, tc.sourcePVName)
|
||||
got, err := csi.GetCBTInfo(context.Background(), exposer.kubeClient, exposer.log, tc.vs, tc.vsc, tc.sourcePVName)
|
||||
|
||||
if tc.wantErrSubstr != "" {
|
||||
if err == nil {
|
||||
@@ -2344,8 +2348,8 @@ func TestGetCBTInfo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got.changeID != tc.want.changeID || got.volumeID != tc.want.volumeID || got.snapshotID != tc.want.snapshotID {
|
||||
t.Fatalf("unexpected cbtInfo, want %+v, got %+v", tc.want, got)
|
||||
if got.ChangeID != tc.want.ChangeID || got.VolumeID != tc.want.VolumeID || got.SnapshotID != tc.want.SnapshotID {
|
||||
t.Fatalf("unexpected CBTInfo, want %+v, got %+v", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/google/uuid"
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -31,18 +32,31 @@ import (
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
|
||||
"github.com/vmware-tanzu/velero/pkg/nodeagent"
|
||||
velerotypes "github.com/vmware-tanzu/velero/pkg/types"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/csi"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/datamover"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/kube"
|
||||
)
|
||||
|
||||
// GenericRestoreExposeCSI define the CSI specific input param for Generic Restore Expose
|
||||
type GenericRestoreExposeCSI struct {
|
||||
// Snapshot is the CSI snapshot spec
|
||||
Snapshot *velerov2alpha1api.CSISnapshotSpec
|
||||
// SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service
|
||||
SnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
}
|
||||
|
||||
// GenericRestoreExposeParam define the input param for Generic Restore Expose
|
||||
type GenericRestoreExposeParam struct {
|
||||
// TargetPVCName is the target volume name to be restored
|
||||
TargetPVCName string
|
||||
|
||||
// TargetPVName is the target persistent volume name to be restored
|
||||
TargetPVName string
|
||||
|
||||
// TargetNamespace is the namespace of the volume to be restored
|
||||
TargetNamespace string
|
||||
|
||||
@@ -84,6 +98,9 @@ type GenericRestoreExposeParam struct {
|
||||
|
||||
// DataMover is the data mover type, e.g., velero-fs, velero-block
|
||||
DataMover string
|
||||
|
||||
// SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service
|
||||
CSI *GenericRestoreExposeCSI
|
||||
}
|
||||
|
||||
// GenericRestoreRebindVolumeParam define the input param for Generic Restore Rebind Volume
|
||||
@@ -101,6 +118,11 @@ type GenericRestoreRebindVolumeParam struct {
|
||||
TargetFSType string
|
||||
}
|
||||
|
||||
// GenericRestoreCleanUpParam define the input param for Generic Restore CleanUp
|
||||
type GenericRestoreCleanUpParam struct {
|
||||
Snapshot *velerov2alpha1api.CSISnapshotSpec
|
||||
}
|
||||
|
||||
// GenericRestoreExposer is the interfaces for a generic restore exposer
|
||||
type GenericRestoreExposer interface {
|
||||
// Expose starts the process to a restore expose, the expose process may take long time
|
||||
@@ -124,19 +146,21 @@ type GenericRestoreExposer interface {
|
||||
RebindVolume(context.Context, corev1api.ObjectReference, GenericRestoreRebindVolumeParam) error
|
||||
|
||||
// CleanUp cleans up any objects generated during the restore expose
|
||||
CleanUp(context.Context, corev1api.ObjectReference)
|
||||
CleanUp(context.Context, corev1api.ObjectReference, *GenericRestoreCleanUpParam)
|
||||
}
|
||||
|
||||
// NewGenericRestoreExposer creates a new instance of generic restore exposer
|
||||
func NewGenericRestoreExposer(kubeClient kubernetes.Interface, log logrus.FieldLogger) GenericRestoreExposer {
|
||||
func NewGenericRestoreExposer(kubeClient kubernetes.Interface, ctrlClient client.Client, log logrus.FieldLogger) GenericRestoreExposer {
|
||||
return &genericRestoreExposer{
|
||||
kubeClient: kubeClient,
|
||||
ctrlClient: ctrlClient,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
type genericRestoreExposer struct {
|
||||
kubeClient kubernetes.Interface
|
||||
ctrlClient client.Client
|
||||
log logrus.FieldLogger
|
||||
}
|
||||
|
||||
@@ -144,9 +168,11 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap
|
||||
curLog := e.log.WithFields(logrus.Fields{
|
||||
"owner": ownerObject.Name,
|
||||
"target PVC": param.TargetPVCName,
|
||||
"target PV": param.TargetPVName,
|
||||
"target namespace": param.TargetNamespace,
|
||||
})
|
||||
|
||||
curLog.Info("Waiting for target PVC to be consumed")
|
||||
selectedNode, targetPVC, err := kube.WaitPVCConsumed(
|
||||
ctx,
|
||||
e.kubeClient.CoreV1(),
|
||||
@@ -226,7 +252,16 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap
|
||||
}
|
||||
}()
|
||||
|
||||
restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode, param.DataMover)
|
||||
curLog.Info("Creating restore PVC")
|
||||
|
||||
var targetPV *corev1api.PersistentVolume
|
||||
if len(param.TargetPVName) > 0 {
|
||||
targetPV, err = e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, param.TargetPVName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "fail to get the target PV %s", param.TargetPVName)
|
||||
}
|
||||
}
|
||||
restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, targetPV, selectedNode, param.DataMover, param.ExposeTimeout)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error to create restore pvc")
|
||||
}
|
||||
@@ -235,10 +270,44 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog)
|
||||
if len(param.TargetPVName) == 0 {
|
||||
kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog)
|
||||
} else {
|
||||
// cannot delete PV if param.TargetPVName is set because the PV is not created by the Expose process.
|
||||
// It's the existing PV used for in-place restore.
|
||||
kube.DeletePVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
curLog.Info("Creating restore pod")
|
||||
var volumeID string
|
||||
if param.CSI != nil && param.CSI.Snapshot != nil {
|
||||
vs := &snapshotv1api.VolumeSnapshot{}
|
||||
if err := e.ctrlClient.Get(ctx, client.ObjectKey{
|
||||
Namespace: param.CSI.Snapshot.VolumeSnapshotNamespace,
|
||||
Name: param.CSI.Snapshot.VolumeSnapshot,
|
||||
}, vs); err != nil {
|
||||
return errors.Wrapf(err, "error to get volume snapshot %s/%s", param.CSI.Snapshot.VolumeSnapshotNamespace, param.CSI.Snapshot.VolumeSnapshot)
|
||||
}
|
||||
|
||||
vsc, err := csi.GetVSCForVS(ctx, vs, e.ctrlClient)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error to get volume snapshot content for volume snapshot %s/%s", vs.Namespace, vs.Name)
|
||||
}
|
||||
|
||||
var cbtInfo csi.CBTInfo
|
||||
cbtInfo, err = csi.GetCBTInfo(ctx, e.kubeClient, e.log, vs, vsc, param.TargetPVName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error to get CBT info")
|
||||
}
|
||||
curLog.Debugf("CBT info: %+v", cbtInfo)
|
||||
volumeID = cbtInfo.VolumeID
|
||||
}
|
||||
var csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
if param.CSI != nil {
|
||||
csiSnapshotMetadataServiceConfigs = param.CSI.SnapshotMetadataServiceConfigs
|
||||
}
|
||||
restorePod, err := e.createRestorePod(
|
||||
ctx,
|
||||
ownerObject,
|
||||
@@ -253,6 +322,9 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap
|
||||
affinity,
|
||||
param.PriorityClassName,
|
||||
cachePVC,
|
||||
param.TargetNamespace,
|
||||
volumeID,
|
||||
csiSnapshotMetadataServiceConfigs,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error to create restore pod")
|
||||
@@ -419,7 +491,7 @@ func (e *genericRestoreExposer) DiagnoseExpose(ctx context.Context, ownerObject
|
||||
return diag
|
||||
}
|
||||
|
||||
func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference) {
|
||||
func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference, param *GenericRestoreCleanUpParam) {
|
||||
restorePodName := ownerObject.Name
|
||||
restorePVCName := ownerObject.Name
|
||||
cachePVCName := getCachePVCName(ownerObject)
|
||||
@@ -432,6 +504,11 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1a
|
||||
BackupPVCSecretLabel, string(ownerObject.UID), e.log)
|
||||
kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace,
|
||||
BackupPVCSecretLabel, string(ownerObject.UID), e.log)
|
||||
|
||||
if param.Snapshot != nil {
|
||||
kube.EnsureDeleteVolumeSnapshotIfAny(ctx, e.ctrlClient, param.Snapshot.VolumeSnapshotNamespace,
|
||||
param.Snapshot.VolumeSnapshot, 0, e.log)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam) error {
|
||||
@@ -645,6 +722,9 @@ func (e *genericRestoreExposer) createRestorePod(
|
||||
affinity *kube.LoadAffinity,
|
||||
priorityClassName string,
|
||||
cachePVC *corev1api.PersistentVolumeClaim,
|
||||
volumeSnapshotNamespace string,
|
||||
volumeID string,
|
||||
csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService,
|
||||
) (*corev1api.Pod, error) {
|
||||
restorePodName := ownerObject.Name
|
||||
restorePVCName := ownerObject.Name
|
||||
@@ -725,6 +805,14 @@ func (e *genericRestoreExposer) createRestorePod(
|
||||
fmt.Sprintf("--cache-volume-path=%s", cacheVolumePath),
|
||||
}
|
||||
|
||||
if len(volumeID) > 0 {
|
||||
args = append(args, fmt.Sprintf("--vs-namespace=%s", volumeSnapshotNamespace))
|
||||
args = append(args, fmt.Sprintf("--volume-id=%s", volumeID))
|
||||
}
|
||||
if csiSnapshotMetadataServiceConfigs != nil && csiSnapshotMetadataServiceConfigs.SAName != "" {
|
||||
args = append(args, fmt.Sprintf("--cbt-sa-name=%s", csiSnapshotMetadataServiceConfigs.SAName))
|
||||
}
|
||||
|
||||
args = append(args, podInfo.logFormatArgs...)
|
||||
args = append(args, podInfo.logLevelArgs...)
|
||||
|
||||
@@ -843,7 +931,7 @@ func (e *genericRestoreExposer) createRestorePod(
|
||||
return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{})
|
||||
}
|
||||
|
||||
func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, selectedNode string, dataMover string) (*corev1api.PersistentVolumeClaim, error) {
|
||||
func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, targetPV *corev1api.PersistentVolume, selectedNode string, dataMover string, operationTimeout time.Duration) (*corev1api.PersistentVolumeClaim, error) {
|
||||
restorePVCName := ownerObject.Name
|
||||
|
||||
pvcObj := &corev1api.PersistentVolumeClaim{
|
||||
@@ -871,9 +959,10 @@ func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObjec
|
||||
}
|
||||
|
||||
if selectedNode != "" {
|
||||
pvcObj.Annotations = map[string]string{
|
||||
kube.KubeAnnSelectedNode: selectedNode,
|
||||
if pvcObj.Annotations == nil {
|
||||
pvcObj.Annotations = make(map[string]string)
|
||||
}
|
||||
pvcObj.Annotations[kube.KubeAnnSelectedNode] = selectedNode
|
||||
}
|
||||
|
||||
if dataMover == datamover.DataMoverTypeVeleroBlock {
|
||||
@@ -884,5 +973,64 @@ func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObjec
|
||||
*pvcObj.Spec.VolumeMode = corev1api.PersistentVolumeBlock
|
||||
}
|
||||
|
||||
return e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{})
|
||||
volumeName := ""
|
||||
sameVolumeMode := true
|
||||
if targetPV != nil {
|
||||
volumeName = targetPV.Name
|
||||
sameVolumeMode = kube.GetVolumeModeByPVC(pvcObj) == kube.GetVolumeModeByPV(targetPV)
|
||||
if !sameVolumeMode {
|
||||
volumeName = ownerObject.Name
|
||||
}
|
||||
pvcObj.Spec.VolumeName = volumeName
|
||||
}
|
||||
|
||||
restorePVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to create the restore PVC %s in namespace %s", pvcObj.Name, pvcObj.Namespace)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
kube.DeletePVCIfAny(ctx, e.kubeClient.CoreV1(), pvcObj.Name, pvcObj.Namespace, 0, e.log)
|
||||
}
|
||||
}()
|
||||
|
||||
if targetPV != nil {
|
||||
if !sameVolumeMode {
|
||||
tmpPV := &corev1api.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: volumeName,
|
||||
},
|
||||
Spec: *targetPV.Spec.DeepCopy(),
|
||||
}
|
||||
tmpPV.Spec.VolumeMode = restorePVC.Spec.VolumeMode
|
||||
e.log.Infof("the volume mode is different, creating temporary PV %s with volume mode %s", tmpPV.Name, tmpPV.Spec.VolumeMode)
|
||||
tmpPV, err = e.kubeClient.CoreV1().PersistentVolumes().Create(ctx, tmpPV, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to create the temporary PV %s", volumeName)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
kube.DeletePVIfAny(ctx, e.kubeClient.CoreV1(), tmpPV.Name, e.log)
|
||||
}
|
||||
}()
|
||||
|
||||
e.log.Infof("deleting the target PV %s", targetPV.Name)
|
||||
if err = e.kubeClient.CoreV1().PersistentVolumes().Delete(ctx, targetPV.Name, metav1.DeleteOptions{}); err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to delete the target PV %s", targetPV.Name)
|
||||
}
|
||||
targetPV = tmpPV
|
||||
}
|
||||
|
||||
if _, err = kube.ResetPVBinding(ctx, e.kubeClient.CoreV1(), targetPV, nil, restorePVC); err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to reset PV %s binding to restore PVC %s/%s", targetPV.Name, restorePVC.Namespace, restorePVC.Name)
|
||||
}
|
||||
|
||||
if _, err = kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, operationTimeout); err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to wait restore PVC %s/%s bound", restorePVC.Namespace, restorePVC.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return restorePVC, nil
|
||||
}
|
||||
|
||||
@@ -149,6 +149,9 @@ func TestCreateRestorePodWithPriorityClass(t *testing.T) {
|
||||
nil, // affinity
|
||||
tc.expectedPriorityClass,
|
||||
nil,
|
||||
"", // volumeSnapshotNamespace
|
||||
"", // volumeID
|
||||
nil,
|
||||
)
|
||||
|
||||
require.NoError(t, err, tc.description)
|
||||
@@ -229,6 +232,9 @@ func TestCreateRestorePodWithMissingConfigMap(t *testing.T) {
|
||||
nil, // affinity
|
||||
"", // empty priority class since config map is missing
|
||||
nil,
|
||||
"", // volumeSnapshotNamespace
|
||||
"", // volumeID
|
||||
nil,
|
||||
)
|
||||
|
||||
// Should succeed even when config map is missing
|
||||
|
||||
@@ -62,6 +62,21 @@ func TestRestoreExpose(t *testing.T) {
|
||||
StorageClassName: &scName,
|
||||
},
|
||||
}
|
||||
targetPVObj := &corev1api.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "fake-target-pv",
|
||||
},
|
||||
}
|
||||
|
||||
modeBlock := corev1api.PersistentVolumeBlock
|
||||
targetPVObjWithDifferentVolumeMode := &corev1api.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "fake-target-pv",
|
||||
},
|
||||
Spec: corev1api.PersistentVolumeSpec{
|
||||
VolumeMode: &modeBlock,
|
||||
},
|
||||
}
|
||||
|
||||
modeFilesystem := corev1api.PersistentVolumeFilesystem
|
||||
targetPVCObjWithVolumeMode := &corev1api.PersistentVolumeClaim{
|
||||
@@ -119,12 +134,14 @@ func TestRestoreExpose(t *testing.T) {
|
||||
ownerRestore *velerov1.Restore
|
||||
targetPVCName string
|
||||
targetNamespace string
|
||||
targetPVName string
|
||||
kubeReactors []reactor
|
||||
cacheVolume *CacheConfigs
|
||||
dataMover string
|
||||
expectBackupPod bool
|
||||
expectBackupPVC bool
|
||||
expectCachePVC bool
|
||||
expectBackupPV bool
|
||||
err string
|
||||
}{
|
||||
{
|
||||
@@ -185,7 +202,7 @@ func TestRestoreExpose(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
err: "error to create restore pvc: fake-create-error",
|
||||
err: "error to create restore pvc: fail to create the restore PVC fake-restore in namespace velero: fake-create-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
@@ -200,6 +217,135 @@ func TestRestoreExpose(t *testing.T) {
|
||||
expectBackupPod: true,
|
||||
expectBackupPVC: true,
|
||||
},
|
||||
{
|
||||
name: "succeed with target PV set",
|
||||
targetPVCName: "fake-target-pvc",
|
||||
targetNamespace: "fake-ns",
|
||||
targetPVName: "fake-target-pv",
|
||||
ownerRestore: restore,
|
||||
kubeClientObj: []runtime.Object{
|
||||
targetPVCObj,
|
||||
targetPVObj,
|
||||
daemonSet,
|
||||
storageClass,
|
||||
},
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "get",
|
||||
resource: "persistentvolumeclaims",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
getAction := action.(clientTesting.GetAction)
|
||||
if getAction.GetName() == "fake-restore" {
|
||||
return true, &corev1api.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "fake-restore",
|
||||
Namespace: velerov1.DefaultNamespace,
|
||||
},
|
||||
Spec: corev1api.PersistentVolumeClaimSpec{
|
||||
VolumeName: "fake-target-pv",
|
||||
},
|
||||
Status: corev1api.PersistentVolumeClaimStatus{
|
||||
Phase: corev1api.ClaimBound,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return false, nil, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
expectBackupPod: true,
|
||||
expectBackupPVC: true,
|
||||
},
|
||||
{
|
||||
name: "create temporary PV fail",
|
||||
targetPVCName: "fake-target-pvc",
|
||||
targetNamespace: "fake-ns",
|
||||
targetPVName: "fake-target-pv",
|
||||
ownerRestore: restore,
|
||||
kubeClientObj: []runtime.Object{
|
||||
targetPVCObj,
|
||||
targetPVObjWithDifferentVolumeMode,
|
||||
daemonSet,
|
||||
storageClass,
|
||||
},
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "create",
|
||||
resource: "persistentvolumes",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
return true, nil, errors.New("fake-create-pv-error")
|
||||
},
|
||||
},
|
||||
},
|
||||
err: "error to create restore pvc: fail to create the temporary PV fake-restore: fake-create-pv-error",
|
||||
},
|
||||
{
|
||||
name: "delete original PV fail",
|
||||
targetPVCName: "fake-target-pvc",
|
||||
targetNamespace: "fake-ns",
|
||||
targetPVName: "fake-target-pv",
|
||||
ownerRestore: restore,
|
||||
kubeClientObj: []runtime.Object{
|
||||
targetPVCObj,
|
||||
targetPVObjWithDifferentVolumeMode,
|
||||
daemonSet,
|
||||
storageClass,
|
||||
},
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "delete",
|
||||
resource: "persistentvolumes",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
deleteAction := action.(clientTesting.DeleteAction)
|
||||
if deleteAction.GetName() == "fake-target-pv" {
|
||||
return true, nil, errors.New("fake-delete-pv-error")
|
||||
}
|
||||
return false, nil, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
err: "error to create restore pvc: fail to delete the target PV fake-target-pv: fake-delete-pv-error",
|
||||
},
|
||||
{
|
||||
name: "succeed with target PV set and different volume mode",
|
||||
targetPVCName: "fake-target-pvc",
|
||||
targetNamespace: "fake-ns",
|
||||
targetPVName: "fake-target-pv",
|
||||
ownerRestore: restore,
|
||||
kubeClientObj: []runtime.Object{
|
||||
targetPVCObj,
|
||||
targetPVObjWithDifferentVolumeMode,
|
||||
daemonSet,
|
||||
storageClass,
|
||||
},
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "get",
|
||||
resource: "persistentvolumeclaims",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
getAction := action.(clientTesting.GetAction)
|
||||
if getAction.GetName() == "fake-restore" {
|
||||
return true, &corev1api.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "fake-restore",
|
||||
Namespace: velerov1.DefaultNamespace,
|
||||
},
|
||||
Spec: corev1api.PersistentVolumeClaimSpec{
|
||||
VolumeName: "fake-restore",
|
||||
},
|
||||
Status: corev1api.PersistentVolumeClaimStatus{
|
||||
Phase: corev1api.ClaimBound,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return false, nil, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
expectBackupPod: true,
|
||||
expectBackupPVC: true,
|
||||
expectBackupPV: true,
|
||||
},
|
||||
{
|
||||
name: "succeed, cache config, no cache volume",
|
||||
targetPVCName: "fake-target-pvc",
|
||||
@@ -311,6 +457,7 @@ func TestRestoreExpose(t *testing.T) {
|
||||
GenericRestoreExposeParam{
|
||||
TargetPVCName: test.targetPVCName,
|
||||
TargetNamespace: test.targetNamespace,
|
||||
TargetPVName: test.targetPVName,
|
||||
HostingPodLabels: map[string]string{},
|
||||
Resources: corev1api.ResourceRequirements{},
|
||||
ExposeTimeout: time.Millisecond,
|
||||
@@ -330,7 +477,7 @@ func TestRestoreExpose(t *testing.T) {
|
||||
if test.expectBackupPod {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err)
|
||||
}
|
||||
|
||||
pvc, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
@@ -341,14 +488,31 @@ func TestRestoreExpose(t *testing.T) {
|
||||
require.Equal(t, corev1api.PersistentVolumeBlock, *pvc.Spec.VolumeMode)
|
||||
}
|
||||
} else {
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err)
|
||||
}
|
||||
|
||||
_, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), getCachePVCName(ownerObject), metav1.GetOptions{})
|
||||
if test.expectCachePVC {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err)
|
||||
}
|
||||
|
||||
_, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
if test.expectBackupPV {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err)
|
||||
}
|
||||
|
||||
if test.targetPVName != "" && !test.expectBackupPV && test.err == "" {
|
||||
// if targetPVName was provided, and sameVolumeMode was true, the original PV should still exist
|
||||
_, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), test.targetPVName, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
} else if test.targetPVName != "" && test.expectBackupPV {
|
||||
// if targetPVName was provided, and sameVolumeMode was false (expectBackupPV is true), the original PV should be deleted
|
||||
_, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), test.targetPVName, metav1.GetOptions{})
|
||||
require.True(t, apierrors.IsNotFound(err), "expected original PV %s to be deleted, but it still exists", test.targetPVName)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -480,6 +644,9 @@ func TestRebindVolume(t *testing.T) {
|
||||
Spec: corev1api.PersistentVolumeClaimSpec{
|
||||
VolumeName: "fake-restore-pv",
|
||||
},
|
||||
Status: corev1api.PersistentVolumeClaimStatus{
|
||||
Phase: corev1api.ClaimBound,
|
||||
},
|
||||
}
|
||||
|
||||
restorePVObj := &corev1api.PersistentVolume{
|
||||
@@ -1508,6 +1675,9 @@ func TestCreateRestorePod(t *testing.T) {
|
||||
test.affinity,
|
||||
"", // priority class name
|
||||
nil,
|
||||
"", // volumeSnapshotNamespace
|
||||
"", // volumeID
|
||||
nil,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -42,8 +42,8 @@ func (_m *GenericRestoreExposer) EXPECT() *GenericRestoreExposer_Expecter {
|
||||
}
|
||||
|
||||
// CleanUp provides a mock function for the type GenericRestoreExposer
|
||||
func (_mock *GenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) {
|
||||
_mock.Called(context1, objectReference)
|
||||
func (_mock *GenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam) {
|
||||
_mock.Called(context1, objectReference, param)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -55,11 +55,12 @@ type GenericRestoreExposer_CleanUp_Call struct {
|
||||
// CleanUp is a helper method to define mock.On call
|
||||
// - context1 context.Context
|
||||
// - objectReference v1.ObjectReference
|
||||
func (_e *GenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *GenericRestoreExposer_CleanUp_Call {
|
||||
return &GenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)}
|
||||
// - param *exposer.GenericRestoreCleanUpParam
|
||||
func (_e *GenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}, param interface{}) *GenericRestoreExposer_CleanUp_Call {
|
||||
return &GenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference, param)}
|
||||
}
|
||||
|
||||
func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call {
|
||||
func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam)) *GenericRestoreExposer_CleanUp_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
@@ -69,9 +70,14 @@ func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Cont
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(v1.ObjectReference)
|
||||
}
|
||||
var arg2 *exposer.GenericRestoreCleanUpParam
|
||||
if args[2] != nil {
|
||||
arg2 = args[2].(*exposer.GenericRestoreCleanUpParam)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
arg2,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
@@ -82,7 +88,7 @@ func (_c *GenericRestoreExposer_CleanUp_Call) Return() *GenericRestoreExposer_Cl
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *GenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call {
|
||||
func (_c *GenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam)) *GenericRestoreExposer_CleanUp_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -184,7 +184,9 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
|
||||
log.Info("Async fs br init")
|
||||
|
||||
if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings, &datapath.RestoreStartParam{}); err != nil {
|
||||
if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings, &datapath.RestoreStartParam{
|
||||
Incremental: pvr.Spec.RestoreType == string(velerov1api.VolumeDataPolicyTypeIncremental),
|
||||
}); err != nil {
|
||||
return "", errors.Wrap(err, "error starting data path restore")
|
||||
}
|
||||
|
||||
|
||||
@@ -297,6 +297,10 @@ func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backu
|
||||
pvr.Spec.UploaderSettings = uploaderutil.StoreRestoreConfig(restore.Spec.UploaderConfig)
|
||||
}
|
||||
|
||||
if restore.IsVolumeDataInplaceRestore() {
|
||||
pvr.Spec.RestoreType = string(restore.Spec.ExistingVolumeDataPolicy)
|
||||
}
|
||||
|
||||
return pvr
|
||||
}
|
||||
|
||||
|
||||
@@ -20,17 +20,20 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilrand "k8s.io/apimachinery/pkg/util/rand"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
crclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
@@ -44,6 +47,9 @@ import (
|
||||
uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util"
|
||||
"github.com/vmware-tanzu/velero/pkg/util"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/csi"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/datamover"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/kube"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -53,12 +59,14 @@ const (
|
||||
|
||||
// pvcRestoreItemAction is a restore item action plugin for Velero
|
||||
type pvcRestoreItemAction struct {
|
||||
log logrus.FieldLogger
|
||||
crClient crclient.Client
|
||||
log logrus.FieldLogger
|
||||
crClient crclient.Client
|
||||
kubeClient kubernetes.Interface
|
||||
csiSnapshotClient snapshotter.SnapshotV1Interface
|
||||
}
|
||||
|
||||
// AppliesTo returns information indicating that the
|
||||
// PVCRestoreItemAction should be run while restoring PVCs.
|
||||
// PVCCSIRestoreItemAction should be run while restoring PVCs.
|
||||
func (p *pvcRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) {
|
||||
return velero.ResourceSelector{
|
||||
IncludedResources: []string{"persistentvolumeclaims"},
|
||||
@@ -83,28 +91,178 @@ func (p *pvcRestoreItemAction) Execute(
|
||||
}
|
||||
|
||||
logger := p.log.WithFields(logrus.Fields{
|
||||
"Action": "PVCRestoreItemAction",
|
||||
"Action": "PVCCSIRestoreItemAction",
|
||||
"PVC": pvc.Namespace + "/" + pvc.Name,
|
||||
"Restore": input.Restore.Namespace + "/" + input.Restore.Name,
|
||||
})
|
||||
logger.Info("Starting PVCRestoreItemAction for PVC")
|
||||
logger.Info("Starting PVCCSIRestoreItemAction for PVC")
|
||||
|
||||
// make sure this RIA only runs for CSI snapshot
|
||||
vsName, nameOK := pvcFromBackup.Annotations[velerov1api.VolumeSnapshotLabel]
|
||||
if !nameOK {
|
||||
logger.Info("Skipping PVCRestoreItemAction for PVC, PVC does not have a CSI VolumeSnapshot.")
|
||||
logger.Info("Skipping PVCCSIRestoreItemAction for PVC, PVC does not have a CSI VolumeSnapshot.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// If PVC already exists, returns early.
|
||||
if p.isResourceExist(pvc, *input.Restore) {
|
||||
pvcExists, existingPVC, err := p.isResourceExist(&pvc, *input.Restore)
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var output *velero.RestoreItemActionExecuteOutput
|
||||
if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) {
|
||||
output, err = p.executeWithoutPVRestore(logger, input, pvcExists, &pvc)
|
||||
} else {
|
||||
backup := new(velerov1api.Backup)
|
||||
if err := p.crClient.Get(context.TODO(), crclient.ObjectKey{Namespace: input.Restore.Namespace, Name: input.Restore.Spec.BackupName}, backup); err != nil {
|
||||
return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error())
|
||||
}
|
||||
if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
|
||||
output, err = p.executeWithDataMove(logger, input, backup, pvcExists, existingPVC, &pvc, &pvcFromBackup)
|
||||
} else {
|
||||
output, err = p.executeWithoutDataMove(logger, input, pvcExists, &pvc, vsName)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
logger.Info("Returning from PVCCSIRestoreItemAction for PVC")
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) executeWithoutPVRestore(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, pvcExists bool, pvc *corev1api.PersistentVolumeClaim) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
if pvcExists {
|
||||
logger.Warnf("PVC already exists. Skip restore this PVC.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
logger.Info("Restore did not request for PVs to be restored from snapshot")
|
||||
pvc.Spec.VolumeName = ""
|
||||
pvc.Spec.DataSource = nil
|
||||
pvc.Spec.DataSourceRef = nil
|
||||
|
||||
unstructuredPVC, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) executeWithoutDataMove(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, pvcExists bool, pvc *corev1api.PersistentVolumeClaim, vsName string) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
if pvcExists {
|
||||
logger.Warnf("PVC already exists. Skip restore this PVC.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
//To avoid confilcs, vs and vsc get a new uniq name based in restore UID
|
||||
// and vs name old name
|
||||
newVSName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsName)
|
||||
|
||||
logger.Debugf("Setting PVC source to VolumeSnapshot new name: %s", newVSName)
|
||||
resetPVCSourceToVolumeSnapshot(pvc, newVSName)
|
||||
|
||||
// Force-restore the VolumeSnapshot even when restore resource filters
|
||||
// would otherwise exclude it (mirrors backup-side must-include).
|
||||
annotations := pvc.GetAnnotations()
|
||||
if annotations == nil {
|
||||
annotations = map[string]string{}
|
||||
}
|
||||
annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
|
||||
pvc.SetAnnotations(annotations)
|
||||
|
||||
unstructuredPVC, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC},
|
||||
AdditionalItems: []velero.ResourceIdentifier{
|
||||
{
|
||||
GroupResource: kuberesource.VolumeSnapshots,
|
||||
Name: vsName,
|
||||
Namespace: pvc.Namespace,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) executeWithDataMove(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, backup *velerov1api.Backup, pvcExists bool, existingPVC, pvc, pvcFromBackup *corev1api.PersistentVolumeClaim) (out *velero.RestoreItemActionExecuteOutput, err error) {
|
||||
ctx := context.Background()
|
||||
var existingPV *corev1api.PersistentVolume
|
||||
|
||||
// If PVC already exists and is not in-place restore, returns early.
|
||||
if pvcExists && !input.Restore.IsVolumeDataInplaceRestore() {
|
||||
logger.Warnf("PVC already exists and ExistingVolumeDataPolicy is not in-place restore. Skip restore this PVC.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
logger.Info("Start DataMover restore.")
|
||||
|
||||
// If PVC doesn't have a DataUploadNameLabel, which should be created
|
||||
// during backup, then CSI cannot handle the volume during to restore,
|
||||
// so return early to let Velero tries to fall back to Velero native snapshot.
|
||||
if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok {
|
||||
logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var dataUploadResult *velerov2alpha1.DataUploadResult
|
||||
dataUploadResult, err = getDataUploadResult(ctx, input.Restore, pvc, p.crClient)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", input.Restore.Name)
|
||||
}
|
||||
|
||||
var volumeSnapshot *snapshotv1api.VolumeSnapshot
|
||||
restoreType := input.Restore.Spec.ExistingVolumeDataPolicy
|
||||
if pvcExists {
|
||||
if existingPVC.Status.Phase != corev1api.ClaimBound {
|
||||
return nil, errors.New("ExistingVolumeDataPolicy is in-place restore, but the existing PVC is not bound.")
|
||||
}
|
||||
// take a CSI snapshot of the existing PVC as the baseline of CBT
|
||||
if input.Restore.IsVolumeDataInplaceIncrementalRestore() && datamover.IsVeleroBlockDataMover(dataUploadResult.DataMover) {
|
||||
logger.Info("ExistingVolumeDataPolicy is in-place incremental restore and data mover is velero-block. Taking a CSI snapshot of the existing PVC as the baseline of CBT...")
|
||||
volumeSnapshot, err = p.createVolumeSnapshot(ctx, logger, input.Restore, *existingPVC, dataUploadResult.SnapshotClass, backup.Spec.CSISnapshotTimeout.Duration)
|
||||
if err != nil {
|
||||
logger.Warnf("fail to create VolumeSnapshot for existing PVC %s/%s: %s, fallback to in-place full restore", existingPVC.Namespace, existingPVC.Name, err.Error())
|
||||
restoreType = velerov1api.VolumeDataPolicyTypeFull
|
||||
} else {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
csi.CleanupVolumeSnapshot(ctx, volumeSnapshot, p.crClient, logger)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// delete the existing PVC, otherwise the target PVC cannot be restored
|
||||
existingPV, err = p.deleteExistingPVC(ctx, logger, pvc, existingPVC, backup.Spec.CSISnapshotTimeout.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
operationID := label.GetValidName(
|
||||
string(velerov1api.AsyncOperationIDPrefixDataDownload) +
|
||||
string(input.Restore.UID) + "." + string(pvcFromBackup.UID))
|
||||
|
||||
// If cross-namespace restore is configured, change the namespace
|
||||
// for PVC object to be restored
|
||||
newNamespace, ok := input.Restore.Spec.NamespaceMapping[pvc.GetNamespace()]
|
||||
@@ -113,90 +271,26 @@ func (p *pvcRestoreItemAction) Execute(
|
||||
newNamespace = pvc.Namespace
|
||||
}
|
||||
|
||||
operationID := ""
|
||||
|
||||
additionalItems := []velero.ResourceIdentifier{}
|
||||
if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) {
|
||||
logger.Info("Restore did not request for PVs to be restored from snapshot")
|
||||
pvc.Spec.VolumeName = ""
|
||||
pvc.Spec.DataSource = nil
|
||||
pvc.Spec.DataSourceRef = nil
|
||||
} else {
|
||||
backup := new(velerov1api.Backup)
|
||||
err := p.crClient.Get(
|
||||
context.TODO(),
|
||||
crclient.ObjectKey{
|
||||
Namespace: input.Restore.Namespace,
|
||||
Name: input.Restore.Spec.BackupName,
|
||||
},
|
||||
backup,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
logger.Error("Fail to get backup for restore.")
|
||||
return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error())
|
||||
}
|
||||
|
||||
if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
|
||||
logger.Info("Start DataMover restore.")
|
||||
|
||||
// If PVC doesn't have a DataUploadNameLabel, which should be created
|
||||
// during backup, then CSI cannot handle the volume during to restore,
|
||||
// so return early to let Velero tries to fall back to Velero native snapshot.
|
||||
if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok {
|
||||
logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.")
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: input.Item,
|
||||
}, nil
|
||||
}
|
||||
|
||||
operationID = label.GetValidName(
|
||||
string(velerov1api.AsyncOperationIDPrefixDataDownload) +
|
||||
string(input.Restore.UID) + "." + string(pvcFromBackup.UID))
|
||||
dataDownload, err := restoreFromDataUploadResult(
|
||||
context.Background(), input.Restore, backup, &pvc, newNamespace,
|
||||
operationID, p.crClient)
|
||||
if err != nil {
|
||||
logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error())
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
logger.Infof("DataDownload %s/%s is created successfully.",
|
||||
dataDownload.Namespace, dataDownload.Name)
|
||||
} else {
|
||||
//To avoid confilcs, vs and vsc get a new uniq name based in restore UID
|
||||
// and vs name old name
|
||||
newVSName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsName)
|
||||
|
||||
p.log.Debugf("Setting PVC source to VolumeSnapshot new name: %s", newVSName)
|
||||
resetPVCSourceToVolumeSnapshot(&pvc, newVSName)
|
||||
|
||||
additionalItems = append(additionalItems, velero.ResourceIdentifier{
|
||||
GroupResource: kuberesource.VolumeSnapshots,
|
||||
Name: vsName,
|
||||
Namespace: pvc.Namespace,
|
||||
})
|
||||
|
||||
// Force-restore the VolumeSnapshot even when restore resource filters
|
||||
// would otherwise exclude it (mirrors backup-side must-include).
|
||||
annotations := pvc.GetAnnotations()
|
||||
if annotations == nil {
|
||||
annotations = map[string]string{}
|
||||
}
|
||||
annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
|
||||
pvc.SetAnnotations(annotations)
|
||||
}
|
||||
var dataDownload *velerov2alpha1.DataDownload
|
||||
dataDownload, err = restoreFromDataUploadResult(
|
||||
context.Background(), dataUploadResult, input.Restore, backup, pvc, existingPV, newNamespace,
|
||||
operationID, string(restoreType), volumeSnapshot, p.crClient)
|
||||
if err != nil {
|
||||
logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error())
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
logger.Infof("DataDownload %s/%s is created successfully.",
|
||||
dataDownload.Namespace, dataDownload.Name)
|
||||
|
||||
pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc)
|
||||
var unstructuredPVC map[string]any
|
||||
unstructuredPVC, err = runtime.DefaultUnstructuredConverter.ToUnstructured(pvc)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
logger.Info("Returning from PVCRestoreItemAction for PVC")
|
||||
|
||||
return &velero.RestoreItemActionExecuteOutput{
|
||||
UpdatedItem: &unstructured.Unstructured{Object: pvcMap},
|
||||
OperationID: operationID,
|
||||
AdditionalItems: additionalItems,
|
||||
UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC},
|
||||
OperationID: operationID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -406,8 +500,14 @@ func newDataDownload(
|
||||
backup *velerov1api.Backup,
|
||||
dataUploadResult *velerov2alpha1.DataUploadResult,
|
||||
pvc *corev1api.PersistentVolumeClaim,
|
||||
newNamespace, operationID string,
|
||||
pv *corev1api.PersistentVolume,
|
||||
newNamespace, operationID, restoreType string,
|
||||
volumeSnapshot *snapshotv1api.VolumeSnapshot,
|
||||
) *velerov2alpha1.DataDownload {
|
||||
pvName := ""
|
||||
if pv != nil {
|
||||
pvName = pv.Name
|
||||
}
|
||||
dataDownload := &velerov2alpha1.DataDownload{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: velerov2alpha1.SchemeGroupVersion.String(),
|
||||
@@ -434,6 +534,7 @@ func newDataDownload(
|
||||
Spec: velerov2alpha1.DataDownloadSpec{
|
||||
TargetVolume: velerov2alpha1.TargetVolumeSpec{
|
||||
PVC: pvc.Name,
|
||||
PV: pvName,
|
||||
Namespace: newNamespace,
|
||||
FSType: dataUploadResult.FSType,
|
||||
},
|
||||
@@ -444,8 +545,15 @@ func newDataDownload(
|
||||
SourceNamespace: dataUploadResult.SourceNamespace,
|
||||
OperationTimeout: backup.Spec.CSISnapshotTimeout,
|
||||
NodeOS: dataUploadResult.NodeOS,
|
||||
RestoreType: restoreType,
|
||||
},
|
||||
}
|
||||
if volumeSnapshot != nil {
|
||||
dataDownload.Spec.CSISnapshot = &velerov2alpha1.CSISnapshotSpec{
|
||||
VolumeSnapshot: volumeSnapshot.Name,
|
||||
VolumeSnapshotNamespace: volumeSnapshot.Namespace,
|
||||
}
|
||||
}
|
||||
if restore.Spec.UploaderConfig != nil {
|
||||
dataDownload.Spec.DataMoverConfig = uploaderUtil.StoreRestoreConfig(restore.Spec.UploaderConfig)
|
||||
}
|
||||
@@ -454,17 +562,15 @@ func newDataDownload(
|
||||
|
||||
func restoreFromDataUploadResult(
|
||||
ctx context.Context,
|
||||
dataUploadResult *velerov2alpha1.DataUploadResult,
|
||||
restore *velerov1api.Restore,
|
||||
backup *velerov1api.Backup,
|
||||
pvc *corev1api.PersistentVolumeClaim,
|
||||
newNamespace, operationID string,
|
||||
pv *corev1api.PersistentVolume,
|
||||
newNamespace, operationID, restoreType string,
|
||||
volumeSnapshot *snapshotv1api.VolumeSnapshot,
|
||||
crClient crclient.Client,
|
||||
) (*velerov2alpha1.DataDownload, error) {
|
||||
dataUploadResult, err := getDataUploadResult(ctx, restore, pvc, crClient)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s",
|
||||
restore.Name)
|
||||
}
|
||||
pvc.Spec.VolumeName = ""
|
||||
if pvc.Spec.Selector == nil {
|
||||
pvc.Spec.Selector = &metav1.LabelSelector{}
|
||||
@@ -481,10 +587,13 @@ func restoreFromDataUploadResult(
|
||||
backup,
|
||||
dataUploadResult,
|
||||
pvc,
|
||||
pv,
|
||||
newNamespace,
|
||||
operationID,
|
||||
restoreType,
|
||||
volumeSnapshot,
|
||||
)
|
||||
err = crClient.Create(ctx, dataDownload)
|
||||
err := crClient.Create(ctx, dataDownload)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to create DataDownload")
|
||||
}
|
||||
@@ -493,9 +602,9 @@ func restoreFromDataUploadResult(
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) isResourceExist(
|
||||
pvc corev1api.PersistentVolumeClaim,
|
||||
pvc *corev1api.PersistentVolumeClaim,
|
||||
restore velerov1api.Restore,
|
||||
) bool {
|
||||
) (bool, *corev1api.PersistentVolumeClaim, error) {
|
||||
// get target namespace to restore into, if different from source namespace
|
||||
targetNamespace := pvc.Namespace
|
||||
if target, ok := restore.Spec.NamespaceMapping[pvc.Namespace]; ok {
|
||||
@@ -503,17 +612,115 @@ func (p *pvcRestoreItemAction) isResourceExist(
|
||||
}
|
||||
|
||||
tmpPVC := new(corev1api.PersistentVolumeClaim)
|
||||
if err := p.crClient.Get(
|
||||
err := p.crClient.Get(
|
||||
context.Background(),
|
||||
crclient.ObjectKey{
|
||||
Name: pvc.Name,
|
||||
Namespace: targetNamespace,
|
||||
},
|
||||
tmpPVC,
|
||||
); err == nil {
|
||||
return true
|
||||
)
|
||||
if err == nil {
|
||||
return true, tmpPVC, nil
|
||||
}
|
||||
return false
|
||||
if apierrors.IsNotFound(err) {
|
||||
return false, nil, nil
|
||||
}
|
||||
return false, nil, errors.Wrapf(err, "fail to get PVC %s in namespace %s", pvc.Name, targetNamespace)
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) deleteExistingPVC(ctx context.Context, logger *logrus.Entry, targetPVC *corev1api.PersistentVolumeClaim, existingPVC *corev1api.PersistentVolumeClaim, operationTimeout time.Duration) (*corev1api.PersistentVolume, error) {
|
||||
// Capture the "selected-node" annotation from the existing PVC before it is deleted below,
|
||||
// and carry it on the target PVC via a Velero-internal carrier annotation. The restore
|
||||
// engine translates the carrier back to the Kubernetes "selected-node" annotation after
|
||||
// all RestoreItemActions have run, so the recreated target PVC keeps the same scheduling
|
||||
// constraint regardless of the order in which RestoreItemActions execute (the generic PVC
|
||||
// RIA unconditionally strips the Kubernetes annotation).
|
||||
selectedNode, exists := existingPVC.Annotations[kube.KubeAnnSelectedNode]
|
||||
if exists {
|
||||
logger.Infof("Carrying %q annotation with value %q for target PVC to keep the same selected node as the existing PVC", kube.KubeAnnSelectedNode, selectedNode)
|
||||
if targetPVC.Annotations == nil {
|
||||
targetPVC.Annotations = map[string]string{}
|
||||
}
|
||||
targetPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = selectedNode
|
||||
}
|
||||
|
||||
var err error
|
||||
logger.Info("ExistingVolumeDataPolicy is in-place restore. Deleting the existing PVC but keep the PV...")
|
||||
pv := &corev1api.PersistentVolume{}
|
||||
if err = p.crClient.Get(context.Background(), crclient.ObjectKey{Name: existingPVC.Spec.VolumeName}, pv); err != nil {
|
||||
return nil, errors.Errorf("Fail to get PV %s: %s", existingPVC.Spec.VolumeName, err.Error())
|
||||
}
|
||||
|
||||
// set reclaim policy to retain
|
||||
updatedPV, err := kube.SetPVReclaimPolicy(ctx, p.kubeClient.CoreV1(), pv, corev1api.PersistentVolumeReclaimRetain)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to set PV reclaim policy to retain for PV %s", pv.Name)
|
||||
}
|
||||
if updatedPV != nil {
|
||||
pv = updatedPV
|
||||
}
|
||||
|
||||
if err = kube.EnsureDeletePVC(ctx, p.kubeClient.CoreV1(), existingPVC.Name, existingPVC.Namespace, operationTimeout); err != nil {
|
||||
return nil, errors.Wrapf(err, "fail to delete the existing PVC %s in namespace %s", existingPVC.Name, existingPVC.Namespace)
|
||||
}
|
||||
|
||||
logger.Info("Existing PVC deleted")
|
||||
|
||||
return pv, nil
|
||||
}
|
||||
|
||||
func (p *pvcRestoreItemAction) createVolumeSnapshot(ctx context.Context, logger *logrus.Entry, restore *velerov1api.Restore, pvc corev1api.PersistentVolumeClaim, vsClass string, operationTimeout time.Duration) (vs *snapshotv1api.VolumeSnapshot, err error) {
|
||||
logger.Infof("creating VolumeSnapshot for PVC %s/%s with VolumeSnapshotClass %s", pvc.Namespace, pvc.Name, vsClass)
|
||||
|
||||
labels := map[string]string{
|
||||
velerov1api.RestoreNameLabel: label.GetValidName(restore.Name),
|
||||
}
|
||||
for k, v := range pvc.ObjectMeta.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
|
||||
vs = &snapshotv1api.VolumeSnapshot{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "velero-" + pvc.Name + "-",
|
||||
Namespace: pvc.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: snapshotv1api.VolumeSnapshotSpec{
|
||||
Source: snapshotv1api.VolumeSnapshotSource{
|
||||
PersistentVolumeClaimName: &pvc.Name,
|
||||
},
|
||||
VolumeSnapshotClassName: &vsClass,
|
||||
},
|
||||
}
|
||||
|
||||
if err := p.crClient.Create(ctx, vs); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create the VolumeSnapshot for PVC %s/%s", pvc.Namespace, pvc.Name)
|
||||
}
|
||||
|
||||
logger.Infof("VolumeSnapshot %s for PVC %s/%s created", vs.Name, pvc.Namespace, pvc.Name)
|
||||
vsName := vs.Name
|
||||
vsNamespace := vs.Namespace
|
||||
|
||||
_, err = csi.WaitUntilVSCHandleIsReady(vs, p.crClient, logger, operationTimeout)
|
||||
if err != nil {
|
||||
csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, logger)
|
||||
return nil, errors.Wrapf(err, "failed to wait for VolumeSnapshotContent of VolumeSnapshot %s/%s to be ready within timeout %v",
|
||||
vsNamespace, vsName, operationTimeout)
|
||||
}
|
||||
|
||||
var updatedVS *snapshotv1api.VolumeSnapshot
|
||||
updatedVS, err = csi.WaitVolumeSnapshotReady(ctx, p.csiSnapshotClient, vs.Name, vs.Namespace, operationTimeout, logger)
|
||||
if err != nil {
|
||||
csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, logger)
|
||||
return nil, errors.Wrapf(err, "failed to wait for VolumeSnapshot %s/%s to become Ready within timeout %v",
|
||||
vsNamespace, vsName, operationTimeout)
|
||||
}
|
||||
vs = updatedVS
|
||||
|
||||
logger.Infof("VolumeSnapshot %s for PVC %s/%s is ready to use", vs.Name, pvc.Namespace, pvc.Name)
|
||||
|
||||
return vs, nil
|
||||
}
|
||||
|
||||
func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer {
|
||||
@@ -523,9 +730,25 @@ func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kubeClient, err := f.KubeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientConfig, err := f.ClientConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
csiSnapshotClient, err := snapshotter.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &pvcRestoreItemAction{
|
||||
log: logger,
|
||||
crClient: crClient,
|
||||
log: logger,
|
||||
crClient: crClient,
|
||||
kubeClient: kubeClient,
|
||||
csiSnapshotClient: csiSnapshotClient,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,12 +28,15 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
"k8s.io/client-go/rest"
|
||||
crclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
|
||||
@@ -371,6 +374,7 @@ func TestExecute(t *testing.T) {
|
||||
backup *velerov1api.Backup
|
||||
restore *velerov1api.Restore
|
||||
pvc *corev1api.PersistentVolumeClaim
|
||||
pv *corev1api.PersistentVolume
|
||||
pvcFromBackup *corev1api.PersistentVolumeClaim
|
||||
vs *snapshotv1api.VolumeSnapshot
|
||||
dataUploadResult *corev1api.ConfigMap
|
||||
@@ -378,9 +382,11 @@ func TestExecute(t *testing.T) {
|
||||
expectedDataDownload *velerov2alpha1.DataDownload
|
||||
expectedPVC *corev1api.PersistentVolumeClaim
|
||||
preCreatePVC bool
|
||||
kubeClientObj []runtime.Object
|
||||
}{
|
||||
{
|
||||
name: "Don't restore PV",
|
||||
backup: builder.ForBackup("velero", "testBackup").Result(),
|
||||
restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").RestorePVs(false).Result(),
|
||||
pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(),
|
||||
expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).VolumeName("").Result(),
|
||||
@@ -486,6 +492,47 @@ func TestExecute(t *testing.T) {
|
||||
pvc: builder.ForPersistentVolumeClaim("restore", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(),
|
||||
preCreatePVC: true,
|
||||
},
|
||||
{
|
||||
name: "PVC exists and in-place restore set",
|
||||
backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(),
|
||||
restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ExistingVolumeDataPolicy(string(velerov1api.VolumeDataPolicyTypeFull)).ItemOperationTimeout(time.Minute * 10).ObjectMeta(builder.WithUID("uid")).Result(),
|
||||
pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(),
|
||||
pv: builder.ForPersistentVolume("testPV").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(),
|
||||
dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(),
|
||||
preCreatePVC: true,
|
||||
kubeClientObj: []runtime.Object{
|
||||
builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(),
|
||||
},
|
||||
expectedDataDownload: func() *velerov2alpha1.DataDownload {
|
||||
d := builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero", PV: "testPV"}).
|
||||
ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}),
|
||||
builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}),
|
||||
builder.WithGenerateName("testRestore-")).Result()
|
||||
d.Spec.RestoreType = "full"
|
||||
return d
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "PVC exists and in-place incremental restore set, createVolumeSnapshot fails",
|
||||
backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(),
|
||||
restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ExistingVolumeDataPolicy(string(velerov1api.VolumeDataPolicyTypeIncremental)).ItemOperationTimeout(time.Minute * 10).ObjectMeta(builder.WithUID("uid")).Result(),
|
||||
pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(),
|
||||
pv: builder.ForPersistentVolume("testPV").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(),
|
||||
dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{\"DataMover\":\"velero-block\", \"SnapshotClass\":\"test-snapclass\"}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(),
|
||||
preCreatePVC: true,
|
||||
kubeClientObj: []runtime.Object{
|
||||
builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(),
|
||||
},
|
||||
expectedDataDownload: func() *velerov2alpha1.DataDownload {
|
||||
d := builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero", PV: "testPV"}).
|
||||
ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}),
|
||||
builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}),
|
||||
builder.WithGenerateName("testRestore-")).Result()
|
||||
d.Spec.RestoreType = "full"
|
||||
d.Spec.DataMover = "velero-block"
|
||||
return d
|
||||
}(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -499,6 +546,10 @@ func TestExecute(t *testing.T) {
|
||||
object = append(object, tc.vs)
|
||||
}
|
||||
|
||||
if tc.pv != nil {
|
||||
object = append(object, tc.pv)
|
||||
}
|
||||
|
||||
input := new(velero.RestoreItemActionExecuteInput)
|
||||
|
||||
if tc.pvc != nil {
|
||||
@@ -524,8 +575,9 @@ func TestExecute(t *testing.T) {
|
||||
}
|
||||
|
||||
pvcRIA := pvcRestoreItemAction{
|
||||
log: logrus.New(),
|
||||
crClient: velerotest.NewFakeControllerRuntimeClient(t, object...),
|
||||
log: logrus.New(),
|
||||
crClient: velerotest.NewFakeControllerRuntimeClient(t, object...),
|
||||
kubeClient: fake.NewSimpleClientset(tc.kubeClientObj...),
|
||||
}
|
||||
|
||||
output, err := pvcRIA.Execute(input)
|
||||
@@ -567,6 +619,128 @@ func TestExecute(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrepareForInplaceRestoreSelectedNode verifies that prepareForInplaceRestore captures
|
||||
// the selected-node annotation from the existing PVC into the Velero-internal carrier
|
||||
// annotation (not the Kubernetes annotation) on the target PVC, before deleting the PVC.
|
||||
func TestPrepareForInplaceRestoreSelectedNode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
existingPVC *corev1api.PersistentVolumeClaim
|
||||
expectedCarrier string
|
||||
expectCarrierSet bool
|
||||
expectKubeAnnoSet bool
|
||||
}{
|
||||
{
|
||||
name: "existing PVC with selected-node sets carrier annotation only",
|
||||
existingPVC: builder.ForPersistentVolumeClaim("ns-1", "pvc-1").
|
||||
ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node-1")).
|
||||
VolumeName("pv-1").
|
||||
Phase(corev1api.ClaimBound).Result(),
|
||||
expectedCarrier: "node-1",
|
||||
expectCarrierSet: true,
|
||||
expectKubeAnnoSet: false,
|
||||
},
|
||||
{
|
||||
name: "existing PVC without selected-node sets neither annotation",
|
||||
existingPVC: builder.ForPersistentVolumeClaim("ns-1", "pvc-1").
|
||||
VolumeName("pv-1").
|
||||
Phase(corev1api.ClaimBound).Result(),
|
||||
expectCarrierSet: false,
|
||||
expectKubeAnnoSet: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
pv := builder.ForPersistentVolume("pv-1").Result()
|
||||
kubeClient := fake.NewSimpleClientset(tc.existingPVC, pv)
|
||||
pvcRIA := pvcRestoreItemAction{
|
||||
log: logrus.New(),
|
||||
crClient: velerotest.NewFakeControllerRuntimeClient(t, pv),
|
||||
kubeClient: kubeClient,
|
||||
}
|
||||
|
||||
targetPVC := builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()
|
||||
returnedPV, err := pvcRIA.deleteExistingPVC(
|
||||
t.Context(), logrus.New().WithField("test", tc.name),
|
||||
targetPVC, tc.existingPVC, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "pv-1", returnedPV.Name)
|
||||
|
||||
carrier, carrierOK := targetPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]
|
||||
require.Equal(t, tc.expectCarrierSet, carrierOK)
|
||||
if tc.expectCarrierSet {
|
||||
require.Equal(t, tc.expectedCarrier, carrier)
|
||||
}
|
||||
_, kubeAnnoOK := targetPVC.Annotations[AnnSelectedNode]
|
||||
require.Equal(t, tc.expectKubeAnnoSet, kubeAnnoOK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecuteInplaceRestore exercises the public Execute() entry for an in-place restore
|
||||
// with an existing PVC: the carrier annotation must be emitted on the returned item, the
|
||||
// Kubernetes selected-node annotation must not be set by this RIA, the existing PVC must be
|
||||
// deleted, and a DataDownload with the in-place restoreType must be created.
|
||||
func TestExecuteInplaceRestore(t *testing.T) {
|
||||
existingPVC := builder.ForPersistentVolumeClaim("velero", "testPVC").
|
||||
ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node-1")).
|
||||
VolumeName("testPV").
|
||||
Phase(corev1api.ClaimBound).Result()
|
||||
existingPV := builder.ForPersistentVolume("testPV").Result()
|
||||
backup := builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result()
|
||||
restore := builder.ForRestore("velero", "testRestore").Backup("testBackup").
|
||||
ObjectMeta(builder.WithUID("uid")).ExistingVolumeDataPolicy("full").Result()
|
||||
pvcFromBackup := builder.ForPersistentVolumeClaim("velero", "testPVC").
|
||||
ObjectMeta(builder.WithAnnotations(
|
||||
velerov1api.VolumeSnapshotLabel, "vsName",
|
||||
velerov1api.DataUploadNameAnnotation, "velero/testDU",
|
||||
)).Result()
|
||||
dataUploadResult := builder.ForConfigMap("velero", "testCM").Data("uid", "{}").
|
||||
ObjectMeta(builder.WithLabels(
|
||||
velerov1api.RestoreUIDLabel, "uid",
|
||||
velerov1api.PVCNamespaceNameLabel, "velero.testPVC",
|
||||
velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)),
|
||||
)).Result()
|
||||
|
||||
pvcRIA := pvcRestoreItemAction{
|
||||
log: logrus.New(),
|
||||
crClient: velerotest.NewFakeControllerRuntimeClient(t, existingPVC, existingPV, backup, dataUploadResult),
|
||||
kubeClient: fake.NewSimpleClientset(existingPVC, existingPV),
|
||||
}
|
||||
|
||||
pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvcFromBackup.DeepCopy())
|
||||
require.NoError(t, err)
|
||||
pvcFromBackupMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvcFromBackup)
|
||||
require.NoError(t, err)
|
||||
|
||||
output, err := pvcRIA.Execute(&velero.RestoreItemActionExecuteInput{
|
||||
Item: &unstructured.Unstructured{Object: pvcMap},
|
||||
ItemFromBackup: &unstructured.Unstructured{Object: pvcFromBackupMap},
|
||||
Restore: restore,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
updatedPVC := new(corev1api.PersistentVolumeClaim)
|
||||
require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(
|
||||
output.UpdatedItem.UnstructuredContent(), updatedPVC))
|
||||
|
||||
// Carrier annotation carries the captured value; the Kubernetes annotation is not set by this RIA.
|
||||
require.Equal(t, "node-1", updatedPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation])
|
||||
require.NotContains(t, updatedPVC.Annotations, AnnSelectedNode)
|
||||
|
||||
// The existing PVC is deleted so the exposer can bind a temporary PVC to the PV.
|
||||
_, err = pvcRIA.kubeClient.CoreV1().PersistentVolumeClaims("velero").Get(t.Context(), "testPVC", metav1.GetOptions{})
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
// A DataDownload with the in-place restoreType referencing the existing PV is created.
|
||||
dataDownloadList := new(velerov2alpha1.DataDownloadList)
|
||||
require.NoError(t, pvcRIA.crClient.List(t.Context(), dataDownloadList, &crclient.ListOptions{}))
|
||||
require.Len(t, dataDownloadList.Items, 1)
|
||||
require.Equal(t, "full", dataDownloadList.Items[0].Spec.RestoreType)
|
||||
require.Equal(t, "testPV", dataDownloadList.Items[0].Spec.TargetVolume.PV)
|
||||
}
|
||||
|
||||
func TestPVCAppliesTo(t *testing.T) {
|
||||
p := pvcRestoreItemAction{
|
||||
log: logrus.StandardLogger(),
|
||||
@@ -596,6 +770,8 @@ func TestNewPvcRestoreItemAction(t *testing.T) {
|
||||
|
||||
f1 := &factorymocks.Factory{}
|
||||
f1.On("KubebuilderClient").Return(crClient, nil)
|
||||
f1.On("KubeClient").Return(nil, nil)
|
||||
f1.On("ClientConfig").Return(&rest.Config{}, nil)
|
||||
plugin1 := NewPvcRestoreItemAction(f1)
|
||||
_, err1 := plugin1(logger)
|
||||
require.NoError(t, err1)
|
||||
|
||||
+34
-4
@@ -1636,6 +1636,19 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
|
||||
return warnings, errs, itemExists
|
||||
}
|
||||
|
||||
// Strip any pre-existing Velero-internal in-place restore carrier annotation coming from
|
||||
// the backup metadata before RestoreItemActions run. The carrier is only trusted when it
|
||||
// is set by a RestoreItemAction (the PVC CSI RIA) during this restore; a stale carrier
|
||||
// baked into the backup must not be translated into the Kubernetes "selected-node"
|
||||
// annotation, which could pin a newly provisioned PVC to a stale node.
|
||||
if annotations := obj.GetAnnotations(); annotations != nil {
|
||||
if _, present := annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]; present {
|
||||
restoreLogger.Infof("Removing pre-existing %q annotation from backup metadata", velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
delete(annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
obj.SetAnnotations(annotations)
|
||||
}
|
||||
}
|
||||
|
||||
restoreLogger.Infof("restore status includes excludes: %+v", ctx.resourceStatusIncludesExcludes)
|
||||
|
||||
for _, action := range ctx.getApplicableActions(groupResource, namespace) {
|
||||
@@ -1768,6 +1781,23 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
|
||||
}
|
||||
}
|
||||
|
||||
// Translate the Velero-internal carrier annotation (set by the PVC CSI RestoreItemAction
|
||||
// during an in-place volume data restore) back to the Kubernetes "selected-node" annotation.
|
||||
// This runs after all RestoreItemActions so the result does not depend on the order in which
|
||||
// the actions executed: the generic PVC RIA unconditionally strips the Kubernetes annotation,
|
||||
// while the carrier annotation passes through untouched. The carrier itself is always
|
||||
// stripped so it never lands on the cluster.
|
||||
if annotations := obj.GetAnnotations(); annotations != nil {
|
||||
if selectedNode, present := annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]; present {
|
||||
if selectedNode != "" {
|
||||
restoreLogger.Infof("Restoring %q annotation with value %q from in-place restore carrier annotation", kube.KubeAnnSelectedNode, selectedNode)
|
||||
annotations[kube.KubeAnnSelectedNode] = selectedNode
|
||||
}
|
||||
delete(annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
obj.SetAnnotations(annotations)
|
||||
}
|
||||
}
|
||||
|
||||
// This comes after running item actions because we have built-in actions that restore
|
||||
// a PVC's associated PV (if applicable). As part of the PV being restored, the 'pvsToProvision'
|
||||
// set may be inserted into, and this needs to happen *before* running the following block of logic.
|
||||
@@ -1943,7 +1973,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
|
||||
if err != nil {
|
||||
warnings.Add(namespace, err)
|
||||
// check if there is existingResourcePolicy and if it is set to update policy
|
||||
if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate {
|
||||
if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.ResourcePolicyTypeUpdate {
|
||||
// remove restore labels so that we apply the latest backup/restore names on the object via patch
|
||||
removeRestoreLabels(fromCluster)
|
||||
//try patching just the backup/restore labels
|
||||
@@ -1963,14 +1993,14 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
|
||||
restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for changed resource %s %s", resourcePolicy, fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
|
||||
|
||||
// existingResourcePolicy is set as none, add warning
|
||||
if resourcePolicy == velerov1api.PolicyTypeNone {
|
||||
if resourcePolicy == velerov1api.ResourcePolicyTypeNone {
|
||||
e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version",
|
||||
obj.GetKind(), obj.GetName())
|
||||
warnings.Add(namespace, e)
|
||||
itemStatus.action = ItemRestoreResultSkipped
|
||||
ctx.restoredItems[itemKey] = itemStatus
|
||||
// existingResourcePolicy is set as update, attempt patch on the resource and add warning if it fails
|
||||
} else if resourcePolicy == velerov1api.PolicyTypeUpdate {
|
||||
} else if resourcePolicy == velerov1api.ResourcePolicyTypeUpdate {
|
||||
// processing update as existingResourcePolicy
|
||||
warningsFromUpdateRP, errsFromUpdateRP := ctx.processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj, namespace, resourceClient)
|
||||
if warningsFromUpdateRP.IsEmpty() && errsFromUpdateRP.IsEmpty() {
|
||||
@@ -1993,7 +2023,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
|
||||
}
|
||||
|
||||
//update backup/restore labels on the unchanged resources if existingResourcePolicy is set as update
|
||||
if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate {
|
||||
if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.ResourcePolicyTypeUpdate {
|
||||
resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy
|
||||
restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for unchanged resource %s %s ", resourcePolicy, obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
|
||||
// remove restore labels so that we apply the latest backup/restore names on the object via patch
|
||||
|
||||
@@ -44,6 +44,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/client-go/dynamic"
|
||||
k8sfake "k8s.io/client-go/kubernetes/fake"
|
||||
kubetesting "k8s.io/client-go/testing"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
@@ -60,6 +61,7 @@ import (
|
||||
vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1"
|
||||
"github.com/vmware-tanzu/velero/pkg/podvolume"
|
||||
uploadermocks "github.com/vmware-tanzu/velero/pkg/podvolume/mocks"
|
||||
riav1 "github.com/vmware-tanzu/velero/pkg/restore/actions"
|
||||
"github.com/vmware-tanzu/velero/pkg/test"
|
||||
"github.com/vmware-tanzu/velero/pkg/types"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/kube"
|
||||
@@ -2852,6 +2854,185 @@ func TestRestoreMustIncludeAdditionalItems(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestRestoreInplaceSelectedNodeCarrierAnnotation verifies the engine translates the
|
||||
// Velero-internal in-place restore carrier annotation into the Kubernetes selected-node
|
||||
// annotation after all RestoreItemActions have run, and always strips the carrier.
|
||||
func TestRestoreInplaceSelectedNodeCarrierAnnotation(t *testing.T) {
|
||||
t.Run("carrier annotation is translated to selected-node and stripped", func(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.AddItems(t, test.PVCs())
|
||||
|
||||
data := &Request{
|
||||
Log: h.log,
|
||||
Restore: defaultRestore().Result(),
|
||||
Backup: defaultBackup().Result(),
|
||||
BackupReader: test.NewTarWriter(t).
|
||||
AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()).
|
||||
Done(),
|
||||
}
|
||||
warnings, errs := h.restorer.Restore(
|
||||
data,
|
||||
[]riav2.RestoreItemAction{
|
||||
// Simulates the PVC CSI RIA setting the carrier during an in-place restore.
|
||||
&pluggableAction{
|
||||
executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
item := input.Item.(*unstructured.Unstructured)
|
||||
annotations := item.GetAnnotations()
|
||||
if annotations == nil {
|
||||
annotations = map[string]string{}
|
||||
}
|
||||
annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = "node-1"
|
||||
item.SetAnnotations(annotations)
|
||||
return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil
|
||||
},
|
||||
},
|
||||
// The real generic PVC RIA (velero.io/pvc), which unconditionally strips the
|
||||
// Kubernetes selected-node annotation. Running it after the carrier-setting
|
||||
// action proves the carrier survives the real strip regardless of action order.
|
||||
&pluggableAction{
|
||||
executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
clientset := k8sfake.NewSimpleClientset()
|
||||
return riav1.NewPVCAction(
|
||||
h.log,
|
||||
clientset.CoreV1().ConfigMaps("velero"),
|
||||
clientset.CoreV1().Nodes(),
|
||||
).Execute(input)
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
|
||||
assertEmptyResults(t, warnings, errs)
|
||||
|
||||
got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
annotations := got.GetAnnotations()
|
||||
assert.Equal(t, "node-1", annotations["volume.kubernetes.io/selected-node"])
|
||||
assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
})
|
||||
|
||||
t.Run("empty carrier annotation is stripped without setting selected-node", func(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.AddItems(t, test.PVCs())
|
||||
|
||||
data := &Request{
|
||||
Log: h.log,
|
||||
Restore: defaultRestore().Result(),
|
||||
Backup: defaultBackup().Result(),
|
||||
BackupReader: test.NewTarWriter(t).
|
||||
AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()).
|
||||
Done(),
|
||||
}
|
||||
warnings, errs := h.restorer.Restore(
|
||||
data,
|
||||
[]riav2.RestoreItemAction{
|
||||
&pluggableAction{
|
||||
executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
item := input.Item.(*unstructured.Unstructured)
|
||||
annotations := item.GetAnnotations()
|
||||
if annotations == nil {
|
||||
annotations = map[string]string{}
|
||||
}
|
||||
annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = ""
|
||||
item.SetAnnotations(annotations)
|
||||
return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
|
||||
assertEmptyResults(t, warnings, errs)
|
||||
|
||||
got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
annotations := got.GetAnnotations()
|
||||
assert.NotContains(t, annotations, "volume.kubernetes.io/selected-node")
|
||||
assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
})
|
||||
|
||||
t.Run("no carrier annotation leaves selected-node stripped (PVC-absent fallback)", func(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.AddItems(t, test.PVCs())
|
||||
|
||||
data := &Request{
|
||||
Log: h.log,
|
||||
Restore: defaultRestore().Result(),
|
||||
Backup: defaultBackup().Result(),
|
||||
BackupReader: test.NewTarWriter(t).
|
||||
AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").
|
||||
ObjectMeta(builder.WithAnnotations("volume.kubernetes.io/selected-node", "stale-node")).Result()).
|
||||
Done(),
|
||||
}
|
||||
warnings, errs := h.restorer.Restore(
|
||||
data,
|
||||
[]riav2.RestoreItemAction{
|
||||
// Simulates the generic PVC RIA stripping the annotation; no action sets the
|
||||
// carrier (as when the target PVC does not exist and Velero falls back to
|
||||
// provisioning a new PVC).
|
||||
&pluggableAction{
|
||||
executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
item := input.Item.(*unstructured.Unstructured)
|
||||
annotations := item.GetAnnotations()
|
||||
delete(annotations, "volume.kubernetes.io/selected-node")
|
||||
item.SetAnnotations(annotations)
|
||||
return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
|
||||
assertEmptyResults(t, warnings, errs)
|
||||
|
||||
got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, got.GetAnnotations(), "volume.kubernetes.io/selected-node")
|
||||
})
|
||||
|
||||
t.Run("carrier annotation baked into backup metadata is not trusted when no action sets it", func(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.AddItems(t, test.PVCs())
|
||||
|
||||
data := &Request{
|
||||
Log: h.log,
|
||||
Restore: defaultRestore().Result(),
|
||||
Backup: defaultBackup().Result(),
|
||||
BackupReader: test.NewTarWriter(t).
|
||||
AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").
|
||||
ObjectMeta(builder.WithAnnotations(velerov1api.InplaceRestoreSelectedNodeAnnotation, "stale-node")).Result()).
|
||||
Done(),
|
||||
}
|
||||
warnings, errs := h.restorer.Restore(
|
||||
data,
|
||||
// No action sets the carrier during this restore (as in the PVC-absent fallback
|
||||
// path where a new PVC is dynamically provisioned), so the carrier from the
|
||||
// backup metadata must be stripped and never translated into selected-node.
|
||||
[]riav2.RestoreItemAction{
|
||||
&pluggableAction{
|
||||
executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
|
||||
item := input.Item.(*unstructured.Unstructured)
|
||||
// The stale carrier from the backup must already be gone before
|
||||
// RestoreItemActions execute.
|
||||
assert.NotContains(t, item.GetAnnotations(), velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
|
||||
assertEmptyResults(t, warnings, errs)
|
||||
|
||||
got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
annotations := got.GetAnnotations()
|
||||
assert.NotContains(t, annotations, "volume.kubernetes.io/selected-node")
|
||||
assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation)
|
||||
})
|
||||
}
|
||||
|
||||
// TestShouldRestore runs the ShouldRestore function for various permutations of
|
||||
// existing/nonexisting/being-deleted PVs, PVCs, and namespaces, and verifies the
|
||||
// result/error matches expectations.
|
||||
|
||||
@@ -205,18 +205,44 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull
|
||||
}
|
||||
|
||||
// Restore restore specific sourcePath with given snapshotID and update progress
|
||||
func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
|
||||
func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
|
||||
log.Info("Start to restore...")
|
||||
|
||||
snapshot, err := rep.GetSnapshot(ctx, udmrepo.ID(snapshotID))
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "Unable to load snapshot %v", snapshotID)
|
||||
}
|
||||
log.Infof("Restore from snapshot %s, incremental %v, cbt source %v, description %s, created time %v, tags %v", snapshotID, incremental, cbtSource, snapshot.Description, snapshot.EndTime, snapshot.Tags)
|
||||
|
||||
log.Infof("Restore from snapshot %s, description %s, created time %v, tags %v", snapshotID, snapshot.Description, snapshot.EndTime, snapshot.Tags)
|
||||
var volumeSnapshot, changeID, volumeID string
|
||||
if incremental {
|
||||
if snapshot.Tags == nil {
|
||||
log.Warnf("No tag from snapshot %s, fallback to full restore", snapshotID)
|
||||
incremental = false
|
||||
} else if snapshot.Tags[uploader.CBTChangeIDTag] == "" {
|
||||
log.Warnf("No ChangeID tag from snapshot %s, fallback to full restore", snapshotID)
|
||||
incremental = false
|
||||
} else if snapshot.Tags[uploader.CBTVolumeIDTag] == "" {
|
||||
log.Warnf("No VolumeID tag from snapshot %s, fallback to full restore", snapshotID)
|
||||
incremental = false
|
||||
} else if snapshot.Tags[uploader.CBTVolumeIDTag] != cbtSource.VolumeID {
|
||||
log.Warnf("VolumeID %s from snapshot %s is not expected as %s, fallback to full restore", snapshot.Tags[uploader.CBTVolumeIDTag], snapshotID, cbtSource.VolumeID)
|
||||
incremental = false
|
||||
} else {
|
||||
volumeSnapshot = cbtSource.Snapshot
|
||||
changeID = snapshot.Tags[uploader.CBTChangeIDTag]
|
||||
volumeID = snapshot.Tags[uploader.CBTVolumeIDTag]
|
||||
}
|
||||
}
|
||||
|
||||
bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), "", "", "")
|
||||
bitmap.SetFull()
|
||||
bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), volumeSnapshot, changeID, volumeID)
|
||||
if incremental {
|
||||
if err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap); err != nil {
|
||||
log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to full restore", cbtSource)
|
||||
}
|
||||
} else {
|
||||
bitmap.SetFull()
|
||||
}
|
||||
|
||||
destPath, err := filepath.Abs(dest)
|
||||
if err != nil {
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
|
||||
udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks"
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader"
|
||||
@@ -123,6 +124,23 @@ func TestBackup(t *testing.T) {
|
||||
assert.Positive(t, info.Size)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "success with CBT",
|
||||
setupOpenDev: func(t *testing.T) *os.File {
|
||||
t.Helper()
|
||||
return tempFile(t, "test-block-data")
|
||||
},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
|
||||
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
|
||||
repo.On("Flush", mock.Anything).Return(nil)
|
||||
},
|
||||
checkInfo: func(t *testing.T, info uploader.SnapshotInfo) {
|
||||
t.Helper()
|
||||
assert.Equal(t, "snap-001", info.ID)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
@@ -186,6 +204,7 @@ func TestSnapshotSource(t *testing.T) {
|
||||
expectedErrStr string
|
||||
expectedSnapID string
|
||||
expectedSize int64
|
||||
cbtService func(t *testing.T) cbtservice.Service
|
||||
}{
|
||||
{
|
||||
name: "uploader Backup error",
|
||||
@@ -218,7 +237,10 @@ func TestSnapshotSource(t *testing.T) {
|
||||
{
|
||||
name: "success with nil cbtService falls back to full bitmap",
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
blkup.On("Backup", mock.Anything, mock.Anything, mock.MatchedBy(func(iter cbttypes.Iterator) bool {
|
||||
// In full mode, the iterator should cover the whole range if it's a full backup
|
||||
return iter != nil
|
||||
}), mock.Anything).
|
||||
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil)
|
||||
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil)
|
||||
repo.On("Flush", mock.Anything).Return(nil)
|
||||
@@ -241,6 +263,46 @@ func TestSnapshotSource(t *testing.T) {
|
||||
},
|
||||
expectedSnapID: "snap-tags",
|
||||
},
|
||||
{
|
||||
name: "success with cbtService getting allocated blocks",
|
||||
cbtService: func(t *testing.T) cbtservice.Service {
|
||||
t.Helper()
|
||||
m := cbtservicemocks.NewService(t)
|
||||
m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
record := args.Get(2).(func([]cbtservice.Range) error)
|
||||
record([]cbtservice.Range{{Offset: 0, Length: 1024}})
|
||||
}).Return(nil)
|
||||
return m
|
||||
},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil)
|
||||
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil)
|
||||
repo.On("Flush", mock.Anything).Return(nil)
|
||||
},
|
||||
expectedSnapID: "snap-cbt-alloc",
|
||||
expectedSize: 1024,
|
||||
},
|
||||
{
|
||||
name: "cbtService error falls back to full",
|
||||
cbtService: func(t *testing.T) cbtservice.Service {
|
||||
t.Helper()
|
||||
m := cbtservicemocks.NewService(t)
|
||||
m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).
|
||||
Return(errors.New("CBT error"))
|
||||
return m
|
||||
},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
// Should be called with parentObject as empty because of fallback
|
||||
blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything).
|
||||
Return(udmrepo.Snapshot{}, int64(2048), nil)
|
||||
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil)
|
||||
repo.On("Flush", mock.Anything).Return(nil)
|
||||
},
|
||||
expectedSnapID: "snap-cbt-fallback",
|
||||
expectedSize: 2048,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
@@ -251,14 +313,19 @@ func TestSnapshotSource(t *testing.T) {
|
||||
|
||||
tc.setupMocks(mockBlkup, mockRepo)
|
||||
|
||||
cbtSrc := cbtservice.SourceInfo{ChangeID: "cid-1", VolumeID: "vid-1"}
|
||||
cbtSrc := cbtservice.SourceInfo{Snapshot: "snap-1", ChangeID: "cid-1", VolumeID: "vid-1"}
|
||||
snapshotTags := map[string]string{"custom": "val"}
|
||||
|
||||
var cbtSvc cbtservice.Service
|
||||
if tc.cbtService != nil {
|
||||
cbtSvc = tc.cbtService(t)
|
||||
}
|
||||
|
||||
snapID, size, err := snapshotSource(
|
||||
ctx, mockRepo, mockBlkup,
|
||||
baseSource,
|
||||
true, "",
|
||||
cbtSrc, nil,
|
||||
cbtSrc, cbtSvc,
|
||||
snapshotTags, map[string]string{},
|
||||
testLog(), "Block Uploader",
|
||||
)
|
||||
@@ -601,6 +668,9 @@ func TestRestore(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
incremental bool
|
||||
cbtSource cbtservice.SourceInfo
|
||||
cbtService func(t *testing.T) cbtservice.Service
|
||||
setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
|
||||
setupOpenDev func(t *testing.T) *os.File
|
||||
expectedErrStr string
|
||||
@@ -637,7 +707,7 @@ func TestRestore(t *testing.T) {
|
||||
expectedErrStr: "error restoring to block dev",
|
||||
},
|
||||
{
|
||||
name: "success returns size",
|
||||
name: "success returns size (full restore)",
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).
|
||||
Return(storedSnap, nil)
|
||||
@@ -650,6 +720,102 @@ func TestRestore(t *testing.T) {
|
||||
},
|
||||
expectedSize: 4096,
|
||||
},
|
||||
{
|
||||
name: "incremental restore success",
|
||||
incremental: true,
|
||||
cbtSource: cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"},
|
||||
cbtService: func(t *testing.T) cbtservice.Service {
|
||||
t.Helper()
|
||||
m := cbtservicemocks.NewService(t)
|
||||
m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
record := args.Get(3).(func([]cbtservice.Range) error)
|
||||
record([]cbtservice.Range{{Offset: 0, Length: 512}})
|
||||
}).Return(nil)
|
||||
return m
|
||||
},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
snapWithTags := udmrepo.Snapshot{
|
||||
Tags: map[string]string{
|
||||
uploader.CBTChangeIDTag: "cid-1",
|
||||
uploader.CBTVolumeIDTag: "vol-1",
|
||||
},
|
||||
TotalSize: 1024,
|
||||
}
|
||||
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
|
||||
blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(int64(512), int64(512), nil)
|
||||
},
|
||||
setupOpenDev: func(t *testing.T) *os.File {
|
||||
t.Helper()
|
||||
return tempFile(t, "")
|
||||
},
|
||||
expectedSize: 512,
|
||||
},
|
||||
{
|
||||
name: "incremental restore fallback - missing tags",
|
||||
incremental: true,
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(storedSnap, nil)
|
||||
blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(int64(4096), int64(4096), nil)
|
||||
},
|
||||
setupOpenDev: func(t *testing.T) *os.File {
|
||||
t.Helper()
|
||||
return tempFile(t, "")
|
||||
},
|
||||
expectedSize: 4096,
|
||||
},
|
||||
{
|
||||
name: "incremental restore fallback - VolumeID mismatch",
|
||||
incremental: true,
|
||||
cbtSource: cbtservice.SourceInfo{VolumeID: "vol-actual"},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
snapWithTags := udmrepo.Snapshot{
|
||||
Tags: map[string]string{
|
||||
uploader.CBTChangeIDTag: "cid-1",
|
||||
uploader.CBTVolumeIDTag: "vol-expected",
|
||||
},
|
||||
}
|
||||
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
|
||||
blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(int64(4096), int64(4096), nil)
|
||||
},
|
||||
setupOpenDev: func(t *testing.T) *os.File {
|
||||
t.Helper()
|
||||
return tempFile(t, "")
|
||||
},
|
||||
expectedSize: 4096,
|
||||
},
|
||||
{
|
||||
name: "incremental restore fallback - CBT service error",
|
||||
incremental: true,
|
||||
cbtSource: cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"},
|
||||
cbtService: func(t *testing.T) cbtservice.Service {
|
||||
t.Helper()
|
||||
m := cbtservicemocks.NewService(t)
|
||||
m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything).
|
||||
Return(errors.New("CBT error"))
|
||||
return m
|
||||
},
|
||||
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
|
||||
snapWithTags := udmrepo.Snapshot{
|
||||
Tags: map[string]string{
|
||||
uploader.CBTChangeIDTag: "cid-1",
|
||||
uploader.CBTVolumeIDTag: "vol-1",
|
||||
},
|
||||
TotalSize: 1024,
|
||||
}
|
||||
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
|
||||
blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(int64(1024), int64(1024), nil)
|
||||
},
|
||||
setupOpenDev: func(t *testing.T) *os.File {
|
||||
t.Helper()
|
||||
return tempFile(t, "")
|
||||
},
|
||||
expectedSize: 1024,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
@@ -671,7 +837,12 @@ func TestRestore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", map[string]string{}, testLog())
|
||||
var cbtSvc cbtservice.Service
|
||||
if tc.cbtService != nil {
|
||||
cbtSvc = tc.cbtService(t)
|
||||
}
|
||||
|
||||
size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", tc.incremental, tc.cbtSource, cbtSvc, map[string]string{}, testLog())
|
||||
|
||||
if tc.expectedErrStr != "" {
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -389,7 +389,7 @@ func (o *fileSystemRestoreOutput) Terminate() error {
|
||||
}
|
||||
|
||||
// Restore restore specific sourcePath with given snapshotID and update progress
|
||||
func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
|
||||
func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
|
||||
log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
log.Info("Start to restore...")
|
||||
|
||||
@@ -421,7 +421,7 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
|
||||
}
|
||||
|
||||
restoreConcurrency := runtime.NumCPU()
|
||||
|
||||
deleteExtra := false
|
||||
if len(uploaderCfg) > 0 {
|
||||
writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg)
|
||||
if err != nil {
|
||||
@@ -438,9 +438,14 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
|
||||
if concurrency > 0 {
|
||||
restoreConcurrency = concurrency
|
||||
}
|
||||
|
||||
deleteExtra, err = uploaderutil.GetDeleteExtraFiles(uploaderCfg)
|
||||
if err != nil {
|
||||
return 0, 0, errors.Wrap(err, "failed to get delete extra files config")
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("Restore filesystem output %v, concurrency %d", fsOutput, restoreConcurrency)
|
||||
log.Debugf("Restore filesystem output %v, concurrency %d, incremental %v, delete extra %v", fsOutput, restoreConcurrency, incremental, deleteExtra)
|
||||
|
||||
err = fsOutput.Init(ctx)
|
||||
if err != nil {
|
||||
@@ -448,14 +453,22 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
|
||||
}
|
||||
|
||||
var output RestoreOutput
|
||||
// kopiaOutput is the output passed to Kopia's restore.Entry function.
|
||||
// We must pass the unwrapped fsOutput (*restore.FilesystemOutput) directly for file system restores.
|
||||
// This is because Kopia internally uses a strict type assertion (c.output.(*FilesystemOutput))
|
||||
// to determine if it should execute the deleteExtra logic. If we pass the wrapped
|
||||
// fileSystemRestoreOutput, the type assertion fails and extra files are not deleted.
|
||||
var kopiaOutput restore.Output
|
||||
if volMode == uploader.PersistentVolumeBlock {
|
||||
output = &BlockOutput{
|
||||
FilesystemOutput: fsOutput,
|
||||
}
|
||||
kopiaOutput = output
|
||||
} else {
|
||||
output = &fileSystemRestoreOutput{
|
||||
FilesystemOutput: fsOutput,
|
||||
}
|
||||
kopiaOutput = fsOutput
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -464,8 +477,10 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
|
||||
}
|
||||
}()
|
||||
|
||||
stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{
|
||||
stat, err := restoreEntryFunc(kopiaCtx, rep, kopiaOutput, rootEntry, restore.Options{
|
||||
Parallel: restoreConcurrency,
|
||||
Incremental: incremental,
|
||||
DeleteExtra: deleteExtra,
|
||||
RestoreDirEntryAtDepth: math.MaxInt32,
|
||||
Cancel: cancleCh,
|
||||
ProgressCallback: func(ctx context.Context, stats restore.Stats) {
|
||||
|
||||
@@ -681,6 +681,7 @@ func TestRestore(t *testing.T) {
|
||||
expectedCount int32
|
||||
expectedError error
|
||||
volMode uploader.PersistentVolumeMode
|
||||
incremental bool
|
||||
}
|
||||
|
||||
// Define test cases
|
||||
@@ -818,7 +819,7 @@ func TestRestore(t *testing.T) {
|
||||
repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil)
|
||||
|
||||
progress := new(Progress)
|
||||
bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, map[string]string{}, logrus.New(), nil)
|
||||
bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.incremental, tc.volMode, map[string]string{}, logrus.New(), nil)
|
||||
|
||||
// Check if the returned error matches the expected error
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -163,6 +163,8 @@ func (bp *blockProvider) RunRestore(
|
||||
ctx context.Context,
|
||||
snapshotID string,
|
||||
volumePath string,
|
||||
incremental bool,
|
||||
cbtParam CBTParam,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderCfg map[string]string,
|
||||
updater uploader.ProgressUpdater) (int64, error) {
|
||||
@@ -178,7 +180,7 @@ func (bp *blockProvider) RunRestore(
|
||||
|
||||
blkUploader := block.NewUploader(ctx, bp.bkRepo, updater, log)
|
||||
|
||||
size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, uploaderCfg, log)
|
||||
size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, incremental, cbtParam.Source, cbtParam.Service, uploaderCfg, log)
|
||||
|
||||
// errors.Is, not ==: see the equivalent comment on the backup path above.
|
||||
if errors.Is(err, block.ErrCanceled) {
|
||||
|
||||
@@ -412,7 +412,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
|
||||
t.Run("restore", func(t *testing.T) {
|
||||
orig := blockRestoreFunc
|
||||
defer func() { blockRestoreFunc = orig }()
|
||||
blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
|
||||
blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ bool, _ cbtservice.SourceInfo, _ cbtservice.Service, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
|
||||
return 0, errors.Wrap(block.ErrCanceled, "error restoring bdev")
|
||||
}
|
||||
|
||||
@@ -422,7 +422,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
|
||||
log: logrus.New(),
|
||||
}
|
||||
|
||||
_, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda",
|
||||
_, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda", false, CBTParam{},
|
||||
uploader.PersistentVolumeBlock, map[string]string{}, &blockMockProgressUpdater{})
|
||||
|
||||
require.ErrorIs(t, err, ErrorCanceled)
|
||||
@@ -496,9 +496,9 @@ func TestBlockProviderRunRestore(t *testing.T) {
|
||||
var capturedSnapshotID string
|
||||
var capturedVolumePath string
|
||||
|
||||
blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, snapshotID string, volumePath string, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
|
||||
blockRestoreFunc = func(ctx context.Context, blkUp block.Uploader, rep udmrepo.BackupRepo, snapshotID string, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
|
||||
capturedSnapshotID = snapshotID
|
||||
capturedVolumePath = volumePath
|
||||
capturedVolumePath = dest
|
||||
return tc.mockRestoreSize, tc.mockRestoreErr
|
||||
}
|
||||
|
||||
@@ -511,6 +511,8 @@ func TestBlockProviderRunRestore(t *testing.T) {
|
||||
t.Context(),
|
||||
tc.snapshotID,
|
||||
tc.volumePath,
|
||||
false,
|
||||
CBTParam{},
|
||||
uploader.PersistentVolumeBlock,
|
||||
map[string]string{},
|
||||
tc.updater,
|
||||
|
||||
@@ -211,6 +211,8 @@ func (kp *kopiaProvider) RunRestore(
|
||||
ctx context.Context,
|
||||
snapshotID string,
|
||||
volumePath string,
|
||||
incremental bool,
|
||||
_ CBTParam,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderCfg map[string]string,
|
||||
updater uploader.ProgressUpdater) (int64, error) {
|
||||
@@ -234,7 +236,7 @@ func (kp *kopiaProvider) RunRestore(
|
||||
// We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore.
|
||||
// Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error.
|
||||
// Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way.
|
||||
size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel)
|
||||
size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, incremental, volMode, uploaderCfg, log, restoreCancel)
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "Failed to run kopia restore")
|
||||
|
||||
@@ -119,20 +119,21 @@ func TestRunBackup(t *testing.T) {
|
||||
func TestRunRestore(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
|
||||
hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
|
||||
notError bool
|
||||
volMode uploader.PersistentVolumeMode
|
||||
incremental bool
|
||||
}{
|
||||
{
|
||||
name: "normal restore",
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
return 0, 0, nil
|
||||
},
|
||||
notError: true,
|
||||
},
|
||||
{
|
||||
name: "normal block mode restore",
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
return 0, 0, nil
|
||||
},
|
||||
volMode: uploader.PersistentVolumeBlock,
|
||||
@@ -140,7 +141,7 @@ func TestRunRestore(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "failed to restore",
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
|
||||
return 0, 0, errors.New("failed to restore")
|
||||
},
|
||||
notError: false,
|
||||
@@ -157,7 +158,7 @@ func TestRunRestore(t *testing.T) {
|
||||
tc.volMode = uploader.PersistentVolumeFilesystem
|
||||
}
|
||||
kopiaRestoreFunc = tc.hookRestoreFunc
|
||||
_, err := kp.RunRestore(t.Context(), "", "/var", tc.volMode, map[string]string{}, &updater)
|
||||
_, err := kp.RunRestore(t.Context(), "", "/var", tc.incremental, CBTParam{}, tc.volMode, map[string]string{}, &updater)
|
||||
if tc.notError {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
||||
@@ -223,8 +223,8 @@ func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, pa
|
||||
}
|
||||
|
||||
// RunRestore provides a mock function for the type Provider
|
||||
func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) {
|
||||
ret := _mock.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
|
||||
func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) {
|
||||
ret := _mock.Called(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunRestore")
|
||||
@@ -232,16 +232,16 @@ func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volume
|
||||
|
||||
var r0 int64
|
||||
var r1 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok {
|
||||
return returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok {
|
||||
return returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
|
||||
}
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
|
||||
r0 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
|
||||
r0 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
|
||||
r1 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
|
||||
if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
|
||||
r1 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
@@ -257,14 +257,16 @@ type Provider_RunRestore_Call struct {
|
||||
// - ctx context.Context
|
||||
// - snapshotID string
|
||||
// - volumePath string
|
||||
// - incremental bool
|
||||
// - cbtParam provider.CBTParam
|
||||
// - volMode uploader.PersistentVolumeMode
|
||||
// - uploaderConfig map[string]string
|
||||
// - updater uploader.ProgressUpdater
|
||||
func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call {
|
||||
return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)}
|
||||
func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, incremental interface{}, cbtParam interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call {
|
||||
return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)}
|
||||
}
|
||||
|
||||
func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call {
|
||||
func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
@@ -278,17 +280,25 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID
|
||||
if args[2] != nil {
|
||||
arg2 = args[2].(string)
|
||||
}
|
||||
var arg3 uploader.PersistentVolumeMode
|
||||
var arg3 bool
|
||||
if args[3] != nil {
|
||||
arg3 = args[3].(uploader.PersistentVolumeMode)
|
||||
arg3 = args[3].(bool)
|
||||
}
|
||||
var arg4 map[string]string
|
||||
var arg4 provider.CBTParam
|
||||
if args[4] != nil {
|
||||
arg4 = args[4].(map[string]string)
|
||||
arg4 = args[4].(provider.CBTParam)
|
||||
}
|
||||
var arg5 uploader.ProgressUpdater
|
||||
var arg5 uploader.PersistentVolumeMode
|
||||
if args[5] != nil {
|
||||
arg5 = args[5].(uploader.ProgressUpdater)
|
||||
arg5 = args[5].(uploader.PersistentVolumeMode)
|
||||
}
|
||||
var arg6 map[string]string
|
||||
if args[6] != nil {
|
||||
arg6 = args[6].(map[string]string)
|
||||
}
|
||||
var arg7 uploader.ProgressUpdater
|
||||
if args[7] != nil {
|
||||
arg7 = args[7].(uploader.ProgressUpdater)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
@@ -297,6 +307,8 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID
|
||||
arg3,
|
||||
arg4,
|
||||
arg5,
|
||||
arg6,
|
||||
arg7,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
@@ -307,7 +319,7 @@ func (_c *Provider_RunRestore_Call) Return(n int64, err error) *Provider_RunRest
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call {
|
||||
func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -64,6 +64,8 @@ type Provider interface {
|
||||
ctx context.Context,
|
||||
snapshotID string,
|
||||
volumePath string,
|
||||
incremental bool,
|
||||
cbtParam CBTParam,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderConfig map[string]string,
|
||||
updater uploader.ProgressUpdater) (int64, error)
|
||||
|
||||
@@ -28,6 +28,7 @@ const (
|
||||
ParallelFilesUpload = "ParallelFilesUpload"
|
||||
WriteSparseFiles = "WriteSparseFiles"
|
||||
RestoreConcurrency = "ParallelFilesDownload"
|
||||
DeleteExtraFiles = "DeleteExtraFiles"
|
||||
)
|
||||
|
||||
func StoreBackupConfig(config *velerov1api.UploaderConfigForBackup) map[string]string {
|
||||
@@ -47,6 +48,13 @@ func StoreRestoreConfig(config *velerov1api.UploaderConfigForRestore) map[string
|
||||
if config.ParallelFilesDownload > 0 {
|
||||
data[RestoreConcurrency] = strconv.Itoa(config.ParallelFilesDownload)
|
||||
}
|
||||
|
||||
if config.DeleteExtraFiles != nil {
|
||||
data[DeleteExtraFiles] = strconv.FormatBool(*config.DeleteExtraFiles)
|
||||
} else {
|
||||
data[DeleteExtraFiles] = strconv.FormatBool(false)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
@@ -85,3 +93,15 @@ func GetRestoreConcurrency(uploaderCfg map[string]string) (int, error) {
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func GetDeleteExtraFiles(uploaderCfg map[string]string) (bool, error) {
|
||||
deleteExtraFiles, ok := uploaderCfg[DeleteExtraFiles]
|
||||
if ok {
|
||||
deleteExtraFilesBool, err := strconv.ParseBool(deleteExtraFiles)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to parse DeleteExtraFiles config")
|
||||
}
|
||||
return deleteExtraFilesBool, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ func TestStoreRestoreConfig(t *testing.T) {
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "true",
|
||||
DeleteExtraFiles: "false",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -67,6 +68,7 @@ func TestStoreRestoreConfig(t *testing.T) {
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "false",
|
||||
DeleteExtraFiles: "false",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -76,6 +78,7 @@ func TestStoreRestoreConfig(t *testing.T) {
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "false", // Assuming default value is false for nil case
|
||||
DeleteExtraFiles: "false",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -86,6 +89,37 @@ func TestStoreRestoreConfig(t *testing.T) {
|
||||
expectedData: map[string]string{
|
||||
RestoreConcurrency: "5",
|
||||
WriteSparseFiles: "false",
|
||||
DeleteExtraFiles: "false",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "DeleteExtraFiles is true",
|
||||
config: &velerov1api.UploaderConfigForRestore{
|
||||
DeleteExtraFiles: &boolTrue,
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "false",
|
||||
DeleteExtraFiles: "true",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "DeleteExtraFiles is false",
|
||||
config: &velerov1api.UploaderConfigForRestore{
|
||||
DeleteExtraFiles: &boolFalse,
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "false",
|
||||
DeleteExtraFiles: "false",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "DeleteExtraFiles is nil",
|
||||
config: &velerov1api.UploaderConfigForRestore{
|
||||
DeleteExtraFiles: nil,
|
||||
},
|
||||
expectedData: map[string]string{
|
||||
WriteSparseFiles: "false",
|
||||
DeleteExtraFiles: "false", // Assuming default value is false for nil case
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -240,3 +274,51 @@ func TestGetRestoreConcurrency(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDeleteExtraFiles(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uploaderCfg map[string]string
|
||||
expectedResult bool
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Valid DeleteExtraFiles (true)",
|
||||
uploaderCfg: map[string]string{DeleteExtraFiles: "true"},
|
||||
expectedResult: true,
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Valid DeleteExtraFiles (false)",
|
||||
uploaderCfg: map[string]string{DeleteExtraFiles: "false"},
|
||||
expectedResult: false,
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Invalid DeleteExtraFiles (not a boolean)",
|
||||
uploaderCfg: map[string]string{DeleteExtraFiles: "invalid"},
|
||||
expectedResult: false,
|
||||
expectedError: errors.Wrap(errors.New("strconv.ParseBool: parsing \"invalid\": invalid syntax"), "failed to parse DeleteExtraFiles config"),
|
||||
},
|
||||
{
|
||||
name: "Missing DeleteExtraFiles",
|
||||
uploaderCfg: map[string]string{},
|
||||
expectedResult: false,
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result, err := GetDeleteExtraFiles(test.uploaderCfg)
|
||||
|
||||
if result != test.expectedResult {
|
||||
t.Errorf("Expected result %t, but got %t", test.expectedResult, result)
|
||||
}
|
||||
|
||||
if (err == nil && test.expectedError != nil) || (err != nil && test.expectedError == nil) || (err != nil && test.expectedError != nil && err.Error() != test.expectedError.Error()) {
|
||||
t.Errorf("Expected error '%v', but got '%v'", test.expectedError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/util"
|
||||
)
|
||||
|
||||
// CBTInfo define the info for CBT
|
||||
type CBTInfo struct {
|
||||
ChangeID string
|
||||
VolumeID string
|
||||
SnapshotID string
|
||||
}
|
||||
|
||||
// GetCBTInfo returns the CBT info for a snapshot
|
||||
func GetCBTInfo(ctx context.Context, kubeClient kubernetes.Interface, log logrus.FieldLogger, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (CBTInfo, error) {
|
||||
cbtInfo := CBTInfo{}
|
||||
if vs == nil || vsc == nil {
|
||||
return cbtInfo, errors.New("vs or vsc is nil")
|
||||
}
|
||||
|
||||
cbtInfo.SnapshotID = vs.Name
|
||||
|
||||
if vs.Annotations != nil &&
|
||||
(vs.Annotations[util.VSphereCNSChangeIDAnno] != "" ||
|
||||
vs.Annotations[util.VSphereCNSSnapshotAnno] != "") {
|
||||
cbtInfo.ChangeID = vs.Annotations[util.VSphereCNSChangeIDAnno]
|
||||
|
||||
splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+")
|
||||
if len(splitSnapshotAnno) >= 2 {
|
||||
cbtInfo.VolumeID = splitSnapshotAnno[0]
|
||||
}
|
||||
log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.VolumeID, cbtInfo.ChangeID)
|
||||
} else {
|
||||
pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err)
|
||||
}
|
||||
|
||||
if vsc.Status != nil && vsc.Status.SnapshotHandle != nil {
|
||||
cbtInfo.ChangeID = *vsc.Status.SnapshotHandle
|
||||
}
|
||||
|
||||
if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" {
|
||||
cbtInfo.VolumeID = pv.Spec.CSI.VolumeHandle
|
||||
}
|
||||
log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.VolumeID, cbtInfo.ChangeID)
|
||||
}
|
||||
|
||||
if cbtInfo.VolumeID == "" {
|
||||
return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT")
|
||||
}
|
||||
|
||||
return cbtInfo, nil
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
|
||||
crclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
storagev1api "k8s.io/api/storage/v1"
|
||||
storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1"
|
||||
)
|
||||
@@ -95,6 +96,10 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if tmpPVC.Status.Phase != corev1api.ClaimBound {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
updated = tmpPVC
|
||||
|
||||
return true, nil
|
||||
@@ -112,6 +117,16 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
|
||||
return pv, err
|
||||
}
|
||||
|
||||
// DeletePVCIfAny deletes a PVC by namespace and name if it exists, and log an error when the deletion fails
|
||||
func DeletePVCIfAny(ctx context.Context, client corev1client.CoreV1Interface, pvcName, pvcNamespace string, ensureTimeout time.Duration, log logrus.FieldLogger) {
|
||||
if err := EnsureDeletePVC(ctx, client, pvcName, pvcNamespace, ensureTimeout); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return
|
||||
}
|
||||
log.Warnf("failed to delete pvc %s/%s with err %v", pvcNamespace, pvcName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeletePVIfAny deletes a PV by name if it exists, and log an error when the deletion fails
|
||||
func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, log logrus.FieldLogger) {
|
||||
err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{})
|
||||
@@ -124,6 +139,47 @@ func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, p
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDeleteVolumeSnapshotIfAny deletes a VolumeSnapshot by namespace and name if it exists, and log an error when the deletion fails
|
||||
func EnsureDeleteVolumeSnapshotIfAny(ctx context.Context, client crclient.Client, namespace, name string, ensureTimeout time.Duration, log logrus.FieldLogger) {
|
||||
if err := client.Delete(ctx, &snapshotv1api.VolumeSnapshot{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
}); err != nil && !apierrors.IsNotFound(err) {
|
||||
log.WithError(err).Errorf("Failed to delete the VolumeSnapshot %s/%s", namespace, name)
|
||||
}
|
||||
|
||||
if ensureTimeout == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var updated *snapshotv1api.VolumeSnapshot
|
||||
err := wait.PollUntilContextTimeout(ctx, waitInternal, ensureTimeout, true, func(ctx context.Context) (bool, error) {
|
||||
if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, updated); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, errors.Wrapf(err, "error to get VolumeSnapshot %s/%s", namespace, name)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
if updated == nil {
|
||||
log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted", namespace, name)
|
||||
} else {
|
||||
log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted, finalizers in VolumeSnapshot %v", namespace, name, updated.Finalizers)
|
||||
}
|
||||
} else {
|
||||
log.WithError(err).Errorf("Error to assure VolumeSnapshot %s/%s is deleted", namespace, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDeletePVC asserts the existence of a PVC by name, deletes it and waits for its disappearance and returns errors on any failure
|
||||
// If timeout is 0, it doesn't wait and return nil
|
||||
func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvcName string, namespace string, timeout time.Duration) error {
|
||||
|
||||
@@ -62,6 +62,9 @@ func TestWaitPVCBound(t *testing.T) {
|
||||
Spec: corev1api.PersistentVolumeClaimSpec{
|
||||
VolumeName: "fake-pv",
|
||||
},
|
||||
Status: corev1api.PersistentVolumeClaimStatus{
|
||||
Phase: corev1api.ClaimBound,
|
||||
},
|
||||
}
|
||||
|
||||
pvObj := &corev1api.PersistentVolume{
|
||||
@@ -304,6 +307,105 @@ func TestWaitPVCConsumed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDeletePVCIfAny(t *testing.T) {
|
||||
pvcObject := &corev1api.PersistentVolumeClaim{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "fake-kind-1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "fake-namespace",
|
||||
Name: "fake-pvc",
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
pvcName string
|
||||
pvcNamespace string
|
||||
kubeClientObj []runtime.Object
|
||||
kubeReactors []reactor
|
||||
logMessage string
|
||||
logLevel string
|
||||
ensureTimeout time.Duration
|
||||
}{
|
||||
{
|
||||
name: "pvc not found",
|
||||
pvcName: "fake-pvc",
|
||||
pvcNamespace: "fake-namespace",
|
||||
},
|
||||
{
|
||||
name: "failed to delete pvc",
|
||||
pvcName: "fake-pvc",
|
||||
pvcNamespace: "fake-namespace",
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "delete",
|
||||
resource: "persistentvolumeclaims",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
return true, nil, errors.New("fake-delete-error")
|
||||
},
|
||||
},
|
||||
},
|
||||
kubeClientObj: []runtime.Object{
|
||||
pvcObject,
|
||||
},
|
||||
logMessage: "failed to delete pvc fake-namespace/fake-pvc with err error to delete pvc fake-pvc: fake-delete-error",
|
||||
logLevel: "level=warning",
|
||||
},
|
||||
{
|
||||
name: "delete pvc success",
|
||||
pvcName: "fake-pvc",
|
||||
pvcNamespace: "fake-namespace",
|
||||
kubeClientObj: []runtime.Object{
|
||||
pvcObject,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "delete pvc success but wait fail",
|
||||
pvcName: "fake-pvc",
|
||||
pvcNamespace: "fake-namespace",
|
||||
kubeClientObj: []runtime.Object{
|
||||
pvcObject,
|
||||
},
|
||||
kubeReactors: []reactor{
|
||||
{
|
||||
verb: "delete",
|
||||
resource: "persistentvolumeclaims",
|
||||
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
return true, pvcObject, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
ensureTimeout: time.Second,
|
||||
logMessage: "failed to delete pvc fake-namespace/fake-pvc with err timeout to assure pvc fake-pvc is deleted, finalizers in pvc []",
|
||||
logLevel: "level=warning",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
|
||||
|
||||
for _, reactor := range test.kubeReactors {
|
||||
fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
|
||||
}
|
||||
|
||||
var kubeClient kubernetes.Interface = fakeKubeClient
|
||||
|
||||
logMessage := ""
|
||||
DeletePVCIfAny(t.Context(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, test.ensureTimeout, velerotest.NewSingleLogger(&logMessage))
|
||||
|
||||
if len(test.logMessage) > 0 {
|
||||
assert.Contains(t, logMessage, test.logMessage)
|
||||
}
|
||||
|
||||
if len(test.logLevel) > 0 {
|
||||
assert.Contains(t, logMessage, test.logLevel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletePVAndPVCIfAny(t *testing.T) {
|
||||
pvObject := &corev1api.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "fake-pv",
|
||||
|
||||
@@ -5,8 +5,14 @@ import (
|
||||
)
|
||||
|
||||
func IsResourcePolicyValid(resourcePolicy string) bool {
|
||||
if resourcePolicy == string(api.PolicyTypeNone) || resourcePolicy == string(api.PolicyTypeUpdate) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
return resourcePolicy == "" ||
|
||||
resourcePolicy == string(api.ResourcePolicyTypeNone) ||
|
||||
resourcePolicy == string(api.ResourcePolicyTypeUpdate)
|
||||
}
|
||||
|
||||
func IsVolumeDataPolicyValid(volumeDataPolicy string) bool {
|
||||
return volumeDataPolicy == "" ||
|
||||
volumeDataPolicy == string(api.VolumeDataPolicyTypeNone) ||
|
||||
volumeDataPolicy == string(api.VolumeDataPolicyTypeFull) ||
|
||||
volumeDataPolicy == string(api.VolumeDataPolicyTypeIncremental)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,16 @@ import (
|
||||
)
|
||||
|
||||
func TestIsResourcePolicyValid(t *testing.T) {
|
||||
require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeNone)))
|
||||
require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeUpdate)))
|
||||
require.False(t, IsResourcePolicyValid(""))
|
||||
require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeNone)))
|
||||
require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeUpdate)))
|
||||
require.True(t, IsResourcePolicyValid(""))
|
||||
require.False(t, IsResourcePolicyValid("invalid"))
|
||||
}
|
||||
|
||||
func TestIsVolumeDataPolicyValid(t *testing.T) {
|
||||
require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeNone)))
|
||||
require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeFull)))
|
||||
require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeIncremental)))
|
||||
require.True(t, IsVolumeDataPolicyValid(""))
|
||||
require.False(t, IsVolumeDataPolicyValid("invalid"))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user