diff --git a/changelogs/unreleased/10339-shubham-pampattiwar b/changelogs/unreleased/10339-shubham-pampattiwar new file mode 100644 index 000000000..9f23a6668 --- /dev/null +++ b/changelogs/unreleased/10339-shubham-pampattiwar @@ -0,0 +1 @@ +Add readWriteOncePod backupPVC config to enable mount-level SELinux labeling diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 65ebc4ce7..30e299380 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -246,6 +246,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O backupPVCStorageClass := csiExposeParam.StorageClass backupPVCReadOnly := false spcNoRelabeling := false + backupPVCReadWriteOncePod := false backupPVCAnnotations := map[string]string{} intoleratableNodes := []string{} if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { @@ -262,6 +263,14 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } + if value.ReadWriteOncePod { + if backupPVCReadOnly { + curLog.WithField("vs name", volumeSnapshot.Name).Warn("Ignoring readWriteOncePod for read-only volume") + } else { + backupPVCReadWriteOncePod = true + } + } + if len(value.Annotations) > 0 { backupPVCAnnotations = value.Annotations } @@ -276,7 +285,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } - backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover) + backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCReadWriteOncePod, backupPVCAnnotations, csiExposeParam.DataMover) if err != nil { return errors.Wrap(err, "error to create backup pvc") } @@ -632,7 +641,7 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } -func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { +func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, readWriteOncePod bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover) @@ -644,6 +653,8 @@ func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject co if readOnly { pvcAccessMode = corev1api.ReadOnlyMany + } else if readWriteOncePod { + pvcAccessMode = corev1api.ReadWriteOncePod } dataSource := &corev1api.TypedLocalObjectReference{ diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 13e5bd22f..7b5c2eb3e 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -219,6 +219,7 @@ func TestExpose(t *testing.T) { err string expectedVolumeSize *resource.Quantity expectedReadOnlyPVC bool + expectedRWOPPVC bool expectedBackupPVCStorageClass string expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string @@ -672,6 +673,95 @@ func TestExpose(t *testing.T) { }, }, }, + { + name: "backupPVC uses ReadWriteOncePod access mode", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedRWOPPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadOnly: true, + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedReadOnlyPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, { name: "backupPod mounts backupPVC with storageClass specified in backupPVC config", ownerBackup: backup, @@ -1150,6 +1240,12 @@ func TestExpose(t *testing.T) { assert.Equal(t, test.expectedReadOnlyPVC, gotReadOnlyAccessMode) } + if test.expectedRWOPPVC { + assert.Equal(t, []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOncePod}, backupPVC.Spec.AccessModes) + } else { + assert.NotContains(t, backupPVC.Spec.AccessModes, corev1api.ReadWriteOncePod) + } + if test.expectedBackupPVCStorageClass != "" { assert.Equal(t, test.expectedBackupPVCStorageClass, *backupPVC.Spec.StorageClassName) } @@ -1521,6 +1617,37 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { }, } + backupPVCReadWriteOncePod := corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{ + corev1api.ReadWriteOncePod, + }, + VolumeMode: &volumeMode, + DataSource: dataSource, + DataSourceRef: nil, + StorageClassName: ptr.To("fake-storage-class"), + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + tests := []struct { name string ownerBackup *velerov1.Backup @@ -1529,6 +1656,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { accessMode string resource resource.Quantity readOnly bool + readWriteOncePod bool kubeClientObj []runtime.Object snapshotClientObj []runtime.Object want *corev1api.PersistentVolumeClaim @@ -1556,6 +1684,30 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { want: &backupPVCReadOnly, wantErr: assert.NoError, }, + { + name: "backupPVC gets created with ReadWriteOncePod access mode when readWriteOncePod is set", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: false, + readWriteOncePod: true, + want: &backupPVCReadWriteOncePod, + wantErr: assert.NoError, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: true, + readWriteOncePod: true, + want: &backupPVCReadOnly, + wantErr: assert.NoError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1576,7 +1728,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { APIVersion: tt.ownerBackup.APIVersion, } } - got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "") + got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, tt.readWriteOncePod, map[string]string{}, "") if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index f162f55e2..16e50b283 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -57,6 +57,12 @@ type BackupPVC struct { // ignored if ReadOnly is false SPCNoRelabeling bool `json:"spcNoRelabeling,omitempty"` + // ReadWriteOncePod sets the backupPVC's access mode to ReadWriteOncePod so the kubelet can use + // mount-level SELinux labeling (-o context) instead of per-file relabeling, when the CSI driver + // advertises SELinux mount support. + // ignored if ReadOnly is true + ReadWriteOncePod bool `json:"readWriteOncePod,omitempty"` + // Annotations permits setting annotations for the backupPVC Annotations map[string]string `json:"annotations,omitempty"` diff --git a/site/content/docs/main/data-movement-backup-pvc-configuration.md b/site/content/docs/main/data-movement-backup-pvc-configuration.md index 956d03997..afcd7e50d 100644 --- a/site/content/docs/main/data-movement-backup-pvc-configuration.md +++ b/site/content/docs/main/data-movement-backup-pvc-configuration.md @@ -34,6 +34,12 @@ default the source PVC's storage class will be used. the SELinux point of view, this will be considered a "Super Privileged Container" which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- `readWriteOncePod`: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes. On + SELinux-enabled clusters the kubelet applies the SELinux label to a `ReadWriteOncePod` volume at mount time (`-o context=`) instead of recursively + relabeling every file on the volume, which can take hours on volumes with a high file count. It requires a CSI driver that advertises SELinux mount + support (`CSIDriver.spec.seLinuxMount: true`) and a storage class that supports creating `ReadWriteOncePod` PVCs from a snapshot. This field is ignored + if `readOnly` is `true`. + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` @@ -74,6 +80,9 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: "ocs-storagecluster-ceph-rbd-encrypted": { "secretNames": ["ceph-csi-kms-token"], "configMapNames": ["ceph-csi-kms-config"] + }, + "storage-class-5": { + "readWriteOncePod": true } } } @@ -93,6 +102,10 @@ this can be avoided by configuring a unique timeout (data movement prepare timeout value is 30m by default). - In an SELinux-enabled cluster, any time users set `readOnly=true` they must also set `spcNoRelabeling=true`. There is no need to set `spcNoRelabeling=true` if the volume is not readOnly. +- `readWriteOncePod` and `readOnly` are mutually exclusive. If both are set to `true`, `readOnly` wins, `readWriteOncePod` is ignored and a warning is logged. +- `readWriteOncePod` is an alternative to `readOnly`+`spcNoRelabeling` for SELinux-enabled clusters whose storage does not support `ReadOnlyMany` +(for example Ceph RBD in Filesystem mode or LVM). Users must make sure the storage class used for `backupPVC` supports creating a `ReadWriteOncePod` PVC from +a snapshot, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout. - If any of the above problems occur, then the DataUpload CR is `canceled` after timeout, and the backupPod and backupPVC will be deleted, and the backup will be marked as `PartiallyFailed`. diff --git a/site/content/docs/main/supported-configmaps/node-agent-configmap.md b/site/content/docs/main/supported-configmaps/node-agent-configmap.md index b1519b515..f8f8f2c5c 100644 --- a/site/content/docs/main/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/main/supported-configmaps/node-agent-configmap.md @@ -299,6 +299,7 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] - **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. - **`secretNames`**: List of secret names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). - **`configMapNames`**: List of configmap names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). +- **`readWriteOncePod`**: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes, so the kubelet labels the volume at mount time instead of relabeling every file. Requires a CSI driver with `seLinuxMount: true` and a storage class that supports `ReadWriteOncePod` PVCs from a snapshot. This field is ignored if `readOnly` is `true`. **Use Cases:** - Use read-only volumes for faster snapshot-to-volume conversion @@ -309,6 +310,7 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] **Important Notes:** - Ensure specified storage classes exist and support required access modes - In SELinux environments, always set `spcNoRelabeling: true` when using `readOnly: true` +- In SELinux environments where the storage does not support `ReadOnlyMany`, use `readWriteOncePod: true` instead; it is ignored when `readOnly: true` is also set - Failures result in DataUpload CR staying in `Accepted` phase until timeout (30m default) #### Storage Class Mapping