Cherry pick the in-place restore implementation PRs from feature branch to main (#10415)

* Update CRDs and CLI to support in-place restore (#10038) Update CRDs(Restore, DataDownload, PodVolumeRestore) and restore create CLI to support in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore (#10066) Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Restore Exposer and PVC CSI to support in-place restore (#10104) 1. Update Restore Exposer to support exposing with existing PV for in-place restore 2. Update PVC CSI RIA to continue the restore process for in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Block uploader to support increase restore (#10244) Update Block uploader to support increase restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Exposer to recreate the target PV if the volume mode is different with the restore PVC (#10257) Update Exposer to recreate the target PV if the volume mode is different with t he restore PVC Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Preserve PVC selected-node annotation via carrier annotation for in-place restore For in-place volume data restore, the existing PVC is deleted and recreated. For StorageClasses with the WaitForFirstConsumer volume binding mode, losing the volume.kubernetes.io/selected-node annotation could let the scheduler place the recreated workload Pod in a different zone than the original PV, leaving it stuck in ContainerCreating. Instead of relying on RestoreItemAction execution order (the generic PVC RIA unconditionally strips the selected-node annotation), the PVC CSI RIA now captures the annotation from the existing PVC right before deleting it and carries it on the target PVC via the Velero-internal restore.velero.io/inplace-restore-selected-node annotation. The restore engine translates the carrier back to the Kubernetes annotation after all RestoreItemActions have run and always strips the carrier so it never lands on the cluster. This makes the behavior independent of RIA ordering: the Kubernetes annotation is stripped by default on every path (including when the target PVC does not exist and Velero falls back to provisioning a new PVC), and preservation only happens when the CSI RIA explicitly captured a value from the existing PVC. Signed-off-by: chlins <chlins.zhang@gmail.com> * Update the control path to make the in-place incremental restore with block data mover work E2E (#10410) Update the control path to make the in-place incremental restore with block data mover work E2E Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> --------- Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> Signed-off-by: chlins <chlins.zhang@gmail.com> Co-authored-by: chlins <chlins.zhang@gmail.com>
2026-09-13 11:34:54 +00:00 · 2026-08-26 10:21:32 -04:00
parent ac5744c7b4
commit b7d83a6f2b
54 changed files with 2164 additions and 335 deletions
@@ -0,0 +1,80 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cockroachdb/errors"
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/vmware-tanzu/velero/pkg/util"
+)
+
+// CBTInfo define the info for CBT
+type CBTInfo struct {
+	ChangeID   string
+	VolumeID   string
+	SnapshotID string
+}
+
+// GetCBTInfo returns the CBT info for a snapshot
+func GetCBTInfo(ctx context.Context, kubeClient kubernetes.Interface, log logrus.FieldLogger, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (CBTInfo, error) {
+	cbtInfo := CBTInfo{}
+	if vs == nil || vsc == nil {
+		return cbtInfo, errors.New("vs or vsc is nil")
+	}
+
+	cbtInfo.SnapshotID = vs.Name
+
+	if vs.Annotations != nil &&
+		(vs.Annotations[util.VSphereCNSChangeIDAnno] != "" ||
+			vs.Annotations[util.VSphereCNSSnapshotAnno] != "") {
+		cbtInfo.ChangeID = vs.Annotations[util.VSphereCNSChangeIDAnno]
+
+		splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+")
+		if len(splitSnapshotAnno) >= 2 {
+			cbtInfo.VolumeID = splitSnapshotAnno[0]
+		}
+		log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.VolumeID, cbtInfo.ChangeID)
+	} else {
+		pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{})
+		if err != nil {
+			return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err)
+		}
+
+		if vsc.Status != nil && vsc.Status.SnapshotHandle != nil {
+			cbtInfo.ChangeID = *vsc.Status.SnapshotHandle
+		}
+
+		if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" {
+			cbtInfo.VolumeID = pv.Spec.CSI.VolumeHandle
+		}
+		log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.VolumeID, cbtInfo.ChangeID)
+	}
+
+	if cbtInfo.VolumeID == "" {
+		return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT")
+	}
+
+	return cbtInfo, nil
+}
@@ -35,6 +35,7 @@ import (
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"

+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	storagev1api "k8s.io/api/storage/v1"
 	storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1"
 )
@@ -95,6 +96,10 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
 			return false, nil
 		}

+		if tmpPVC.Status.Phase != corev1api.ClaimBound {
+			return false, nil
+		}
+
 		updated = tmpPVC

 		return true, nil
@@ -112,6 +117,16 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
 	return pv, err
 }

+// DeletePVCIfAny deletes a PVC by namespace and name if it exists, and log an error when the deletion fails
+func DeletePVCIfAny(ctx context.Context, client corev1client.CoreV1Interface, pvcName, pvcNamespace string, ensureTimeout time.Duration, log logrus.FieldLogger) {
+	if err := EnsureDeletePVC(ctx, client, pvcName, pvcNamespace, ensureTimeout); err != nil {
+		if apierrors.IsNotFound(err) {
+			return
+		}
+		log.Warnf("failed to delete pvc %s/%s with err %v", pvcNamespace, pvcName, err)
+	}
+}
+
 // DeletePVIfAny deletes a PV by name if it exists, and log an error when the deletion fails
 func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, log logrus.FieldLogger) {
 	err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{})
@@ -124,6 +139,47 @@ func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, p
 	}
 }

+// EnsureDeleteVolumeSnapshotIfAny deletes a VolumeSnapshot by namespace and name if it exists, and log an error when the deletion fails
+func EnsureDeleteVolumeSnapshotIfAny(ctx context.Context, client crclient.Client, namespace, name string, ensureTimeout time.Duration, log logrus.FieldLogger) {
+	if err := client.Delete(ctx, &snapshotv1api.VolumeSnapshot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}); err != nil && !apierrors.IsNotFound(err) {
+		log.WithError(err).Errorf("Failed to delete the VolumeSnapshot %s/%s", namespace, name)
+	}
+
+	if ensureTimeout == 0 {
+		return
+	}
+
+	var updated *snapshotv1api.VolumeSnapshot
+	err := wait.PollUntilContextTimeout(ctx, waitInternal, ensureTimeout, true, func(ctx context.Context) (bool, error) {
+		if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, updated); err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+
+			return false, errors.Wrapf(err, "error to get VolumeSnapshot %s/%s", namespace, name)
+		}
+
+		return false, nil
+	})
+
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			if updated == nil {
+				log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted", namespace, name)
+			} else {
+				log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted, finalizers in VolumeSnapshot %v", namespace, name, updated.Finalizers)
+			}
+		} else {
+			log.WithError(err).Errorf("Error to assure VolumeSnapshot %s/%s is deleted", namespace, name)
+		}
+	}
+}
+
 // EnsureDeletePVC asserts the existence of a PVC by name, deletes it and waits for its disappearance and returns errors on any failure
 // If timeout is 0, it doesn't wait and return nil
 func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvcName string, namespace string, timeout time.Duration) error {
@@ -62,6 +62,9 @@ func TestWaitPVCBound(t *testing.T) {
 		Spec: corev1api.PersistentVolumeClaimSpec{
 			VolumeName: "fake-pv",
 		},
+		Status: corev1api.PersistentVolumeClaimStatus{
+			Phase: corev1api.ClaimBound,
+		},
 	}

 	pvObj := &corev1api.PersistentVolume{
@@ -304,6 +307,105 @@ func TestWaitPVCConsumed(t *testing.T) {
 }

 func TestDeletePVCIfAny(t *testing.T) {
+	pvcObject := &corev1api.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "fake-kind-1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "fake-namespace",
+			Name:      "fake-pvc",
+		},
+	}
+
+	tests := []struct {
+		name          string
+		pvcName       string
+		pvcNamespace  string
+		kubeClientObj []runtime.Object
+		kubeReactors  []reactor
+		logMessage    string
+		logLevel      string
+		ensureTimeout time.Duration
+	}{
+		{
+			name:         "pvc not found",
+			pvcName:      "fake-pvc",
+			pvcNamespace: "fake-namespace",
+		},
+		{
+			name:         "failed to delete pvc",
+			pvcName:      "fake-pvc",
+			pvcNamespace: "fake-namespace",
+			kubeReactors: []reactor{
+				{
+					verb:     "delete",
+					resource: "persistentvolumeclaims",
+					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
+						return true, nil, errors.New("fake-delete-error")
+					},
+				},
+			},
+			kubeClientObj: []runtime.Object{
+				pvcObject,
+			},
+			logMessage: "failed to delete pvc fake-namespace/fake-pvc with err error to delete pvc fake-pvc: fake-delete-error",
+			logLevel:   "level=warning",
+		},
+		{
+			name:         "delete pvc success",
+			pvcName:      "fake-pvc",
+			pvcNamespace: "fake-namespace",
+			kubeClientObj: []runtime.Object{
+				pvcObject,
+			},
+		},
+		{
+			name:         "delete pvc success but wait fail",
+			pvcName:      "fake-pvc",
+			pvcNamespace: "fake-namespace",
+			kubeClientObj: []runtime.Object{
+				pvcObject,
+			},
+			kubeReactors: []reactor{
+				{
+					verb:     "delete",
+					resource: "persistentvolumeclaims",
+					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
+						return true, pvcObject, nil
+					},
+				},
+			},
+			ensureTimeout: time.Second,
+			logMessage:    "failed to delete pvc fake-namespace/fake-pvc with err timeout to assure pvc fake-pvc is deleted, finalizers in pvc []",
+			logLevel:      "level=warning",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			for _, reactor := range test.kubeReactors {
+				fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
+			}
+
+			var kubeClient kubernetes.Interface = fakeKubeClient
+
+			logMessage := ""
+			DeletePVCIfAny(t.Context(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, test.ensureTimeout, velerotest.NewSingleLogger(&logMessage))
+
+			if len(test.logMessage) > 0 {
+				assert.Contains(t, logMessage, test.logMessage)
+			}
+
+			if len(test.logLevel) > 0 {
+				assert.Contains(t, logMessage, test.logLevel)
+			}
+		})
+	}
+}
+
+func TestDeletePVAndPVCIfAny(t *testing.T) {
 	pvObject := &corev1api.PersistentVolume{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "fake-pv",
@@ -5,8 +5,14 @@ import (
 )

 func IsResourcePolicyValid(resourcePolicy string) bool {
-	if resourcePolicy == string(api.PolicyTypeNone) || resourcePolicy == string(api.PolicyTypeUpdate) {
-		return true
-	}
-	return false
+	return resourcePolicy == "" ||
+		resourcePolicy == string(api.ResourcePolicyTypeNone) ||
+		resourcePolicy == string(api.ResourcePolicyTypeUpdate)
+}
+
+func IsVolumeDataPolicyValid(volumeDataPolicy string) bool {
+	return volumeDataPolicy == "" ||
+		volumeDataPolicy == string(api.VolumeDataPolicyTypeNone) ||
+		volumeDataPolicy == string(api.VolumeDataPolicyTypeFull) ||
+		volumeDataPolicy == string(api.VolumeDataPolicyTypeIncremental)
 }
@@ -9,7 +9,16 @@ import (
 )

 func TestIsResourcePolicyValid(t *testing.T) {
-	require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeNone)))
-	require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeUpdate)))
-	require.False(t, IsResourcePolicyValid(""))
+	require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeNone)))
+	require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeUpdate)))
+	require.True(t, IsResourcePolicyValid(""))
+	require.False(t, IsResourcePolicyValid("invalid"))
+}
+
+func TestIsVolumeDataPolicyValid(t *testing.T) {
+	require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeNone)))
+	require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeFull)))
+	require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeIncremental)))
+	require.True(t, IsVolumeDataPolicyValid(""))
+	require.False(t, IsVolumeDataPolicyValid("invalid"))
 }