diff --git a/changelogs/unreleased/9862-Lyndon-Li b/changelogs/unreleased/9862-Lyndon-Li new file mode 100644 index 000000000..04601d969 --- /dev/null +++ b/changelogs/unreleased/9862-Lyndon-Li @@ -0,0 +1 @@ +Enhance backup exposer for block data mover \ No newline at end of file diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 13be03994..f5c3deed2 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -936,7 +936,12 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get source PV %s", pvc.Spec.VolumeName) } - nodeOS := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) + nodeOS := "" + if du.Spec.DataMover == velerotypes.DataMoverTypeVeleroBlock { + nodeOS = kube.NodeOSLinux + } else { + nodeOS = kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) + } if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 079a4b527..ef7825a11 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -19,6 +19,7 @@ package exposer import ( "context" "fmt" + "maps" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" @@ -93,6 +94,12 @@ type CSISnapshotExposeParam struct { // PriorityClassName is the priority class name for the data mover pod PriorityClassName string + + // DataMover is the data mover type, e.g., velero-fs, velero-block + DataMover string + + // CSISnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + CSISnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService } // CSISnapshotExposeWaitParam define the input param for WaitExposed of CSI snapshots @@ -234,7 +241,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } - backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations) + backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover) if err != nil { return errors.Wrap(err, "error to create backup pvc") } @@ -264,6 +271,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O csiExposeParam.PriorityClassName, intoleratableNodes, volumeTopology, + csiExposeParam.CSISnapshotMetadataServiceConfigs, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -450,7 +458,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api. csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } -func getVolumeModeByAccessMode(accessMode string) (corev1api.PersistentVolumeMode, error) { +func getVolumeModeByAccessMode(accessMode string, dataMover string) (corev1api.PersistentVolumeMode, error) { + if dataMover == velerotypes.DataMoverTypeVeleroBlock { + return corev1api.PersistentVolumeBlock, nil + } + switch accessMode { case AccessModeFileSystem: return corev1api.PersistentVolumeFilesystem, nil @@ -488,10 +500,14 @@ func (e *csiSnapshotExposer) createBackupVS(ctx context.Context, ownerObject cor func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject corev1api.ObjectReference, snapshotVSC *snapshotv1api.VolumeSnapshotContent, vs *snapshotv1api.VolumeSnapshot) (*snapshotv1api.VolumeSnapshotContent, error) { backupVSCName := ownerObject.Name + anno := make(map[string]string) + maps.Copy(anno, snapshotVSC.Annotations) + anno[kube.KubeAnnAllowVolumeModeChange] = "true" + vsc := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: backupVSCName, - Annotations: snapshotVSC.Annotations, + Annotations: anno, Labels: map[string]string{}, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ @@ -524,10 +540,10 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } -func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string) (*corev1api.PersistentVolumeClaim, error) { +func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name - volumeMode, err := getVolumeModeByAccessMode(accessMode) + volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover) if err != nil { return nil, err } @@ -600,6 +616,7 @@ func (e *csiSnapshotExposer) createBackupPod( priorityClassName string, intoleratableNodes []string, volumeTopology *corev1api.NodeSelector, + csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -655,6 +672,12 @@ func (e *csiSnapshotExposer) createBackupPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + if csiSnapshotMetadataServiceConfigs != nil { + if csiSnapshotMetadataServiceConfigs.SAName != "" { + args = append(args, fmt.Sprintf("--csi-snapshot-metadata-service-sa=%s", csiSnapshotMetadataServiceConfigs.SAName)) + } + } + if affinity == nil { affinity = &kube.LoadAffinity{} } diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index d1ffa4700..8c3086f76 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -155,6 +155,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { tc.expectedPriorityClass, nil, nil, + nil, ) require.NoError(t, err, tc.description) @@ -241,6 +242,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { "", // empty priority class since config map is missing nil, nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index e1a9860eb..8e00654f9 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -18,6 +18,7 @@ package exposer import ( "fmt" + "maps" "testing" "time" @@ -1056,21 +1057,25 @@ func TestExpose(t *testing.T) { backupPVC, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - expectedVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + backupVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - expectedVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + backupVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - assert.Equal(t, expectedVS.Annotations, vsObject.Annotations) - assert.Equal(t, *expectedVS.Spec.VolumeSnapshotClassName, *vsObject.Spec.VolumeSnapshotClassName) - assert.Equal(t, expectedVSC.Name, *expectedVS.Spec.Source.VolumeSnapshotContentName) + assert.Equal(t, vsObject.Annotations, backupVS.Annotations) + assert.Equal(t, *vsObject.Spec.VolumeSnapshotClassName, *backupVS.Spec.VolumeSnapshotClassName) + assert.Equal(t, *backupVS.Spec.Source.VolumeSnapshotContentName, backupVSC.Name) - assert.Equal(t, expectedVSC.Annotations, vscObj.Annotations) - assert.Equal(t, expectedVSC.Labels, vscObj.Labels) - assert.Equal(t, expectedVSC.Spec.DeletionPolicy, vscObj.Spec.DeletionPolicy) - assert.Equal(t, expectedVSC.Spec.Driver, vscObj.Spec.Driver) - assert.Equal(t, *expectedVSC.Spec.VolumeSnapshotClassName, *vscObj.Spec.VolumeSnapshotClassName) + anno := make(map[string]string) + maps.Copy(anno, vscObj.Annotations) + anno[kube.KubeAnnAllowVolumeModeChange] = "true" + + assert.Equal(t, anno, backupVSC.Annotations) + assert.Equal(t, vscObj.Labels, backupVSC.Labels) + assert.Equal(t, vscObj.Spec.DeletionPolicy, backupVSC.Spec.DeletionPolicy) + assert.Equal(t, vscObj.Spec.Driver, backupVSC.Spec.Driver) + assert.Equal(t, *vscObj.Spec.VolumeSnapshotClassName, *backupVSC.Spec.VolumeSnapshotClassName) if test.expectedVolumeSize != nil { assert.Equal(t, *test.expectedVolumeSize, backupPVC.Spec.Resources.Requests[corev1api.ResourceStorage]) @@ -1514,7 +1519,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { APIVersion: tt.ownerBackup.APIVersion, } } - got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}) + got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "") if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } diff --git a/pkg/types/data_mover.go b/pkg/types/data_mover.go new file mode 100644 index 000000000..7850f498c --- /dev/null +++ b/pkg/types/data_mover.go @@ -0,0 +1,22 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +const ( + DataMoverTypeVeleroFs string = "velero-fs" + DataMoverTypeVeleroBlock string = "velero-block" +) diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index f456bbf55..42fe06f58 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -74,6 +74,10 @@ type CachePVC struct { ResidentThresholdInMB int64 `json:"residentThresholdInMB,omitempty"` } +type CSISnapshotMetadataService struct { + SAName string `json:"saName,omitempty"` +} + type NodeAgentConfigs struct { // LoadConcurrency is the config for data path load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` @@ -104,4 +108,7 @@ type NodeAgentConfigs struct { // PodLabels are labels to be added to pods created by node-agent, i.e., data mover pods. PodLabels map[string]string `json:"podLabels,omitempty"` + + // CSISnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + CSISnapshotMetadataServiceConfigs *CSISnapshotMetadataService `json:"csiSnapshotMetadataServiceConfigs,omitempty"` } diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 5e5e97603..d93effd7f 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -53,6 +53,7 @@ const ( KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" KubeAnnMigratedTo = "pv.kubernetes.io/migrated-to" KubeAnnSelectedNode = "volume.kubernetes.io/selected-node" + KubeAnnAllowVolumeModeChange = "snapshot.storage.kubernetes.io/allow-volume-mode-change" ) // VolumeSnapshotContentManagedByLabel is applied by the snapshot controller