mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-07-27 18:42:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backup workflow for block data mover. (#10067)

Browse Source 
Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>
This commit is contained in:
Xun Jiang/Bruce Jiang
2026-07-27 15:36:44 +08:00
committed by GitHub
parent d2c46dad7a
commit 3905ccb0ea
12 changed files with 285 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelogs/unreleased/10067-blackpiglet
+1
View File
@@ -0,0 +1 @@
Backup workflow for block data mover.
config/crd/v2alpha1/bases/velero.io_datauploads.yaml
+7
View File
@@ -131,6 +131,13 @@ spec:
OperationTimeout specifies the time used to wait internal operations,
before returning error as timeout.
type: string
parentSnapshot:
description: |-
ParentSnapshot specifies the parent snapshot that current backup is based on.
If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent.
If its value is "none", the data mover will do a full backup
If its value is a specific snapshotID, the data mover finds the specific snapshot as parent.
type: string
snapshotType:
description: SnapshotType is the type of the snapshot to be backed
up.
config/crd/v2alpha1/crds/crds.go
+1 -1
View File
File diff suppressed because one or more lines are too long
pkg/apis/velero/shared/constants.go
+22
View File
@@ -0,0 +1,22 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package shared
const (
DataUploadParentSnapshotNone = "none"
DataUploadParentSnapshotAuto = "auto"
)
pkg/apis/velero/v2alpha1/data_upload_types.go
+6
View File
@@ -64,6 +64,12 @@ type DataUploadSpec struct {
// SourceFSType is the file system type of the source volume.
// +optional
SourceFSType string `json:"sourceFSType,omitempty"`
// ParentSnapshot specifies the parent snapshot that current backup is based on.
// If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent.
// If its value is "none", the data mover will do a full backup
// If its value is a specific snapshotID, the data mover finds the specific snapshot as parent.
ParentSnapshot string `json:"parentSnapshot,omitempty"`
}
type SnapshotType string
pkg/backup/actions/csi/pvc_action.go
+12
View File
@@ -42,6 +42,7 @@ import (
crclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
veleroclient "github.com/vmware-tanzu/velero/pkg/client"
@@ -535,6 +536,16 @@ func newDataUpload(
vsc *snapshotv1api.VolumeSnapshotContent,
fsType string,
) *velerov2alpha1.DataUpload {
var parentSnapshot string
switch backup.Spec.BackupType {
case velerov1api.BackupTypeFull:
parentSnapshot = veleroshared.DataUploadParentSnapshotNone
case velerov1api.BackupTypeIncremental:
parentSnapshot = veleroshared.DataUploadParentSnapshotAuto
default:
parentSnapshot = veleroshared.DataUploadParentSnapshotAuto
}
dataUpload := &velerov2alpha1.DataUpload{
TypeMeta: metav1.TypeMeta{
APIVersion: velerov2alpha1.SchemeGroupVersion.String(),
@@ -572,6 +583,7 @@ func newDataUpload(
SourceNamespace: pvc.Namespace,
OperationTimeout: backup.Spec.CSISnapshotTimeout,
SourceFSType: fsType,
ParentSnapshot: parentSnapshot,
},
}
pkg/backup/actions/csi/pvc_action_test.go
+140 -12
View File
@@ -23,40 +23,39 @@ import (
"testing"
"time"
"github.com/vmware-tanzu/velero/pkg/kuberesource"
volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2"
"github.com/stretchr/testify/assert"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/cockroachdb/errors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
"github.com/cockroachdb/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1api "k8s.io/api/core/v1"
storagev1api "k8s.io/api/storage/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/utils/ptr"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
"github.com/vmware-tanzu/velero/pkg/builder"
factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks"
"github.com/vmware-tanzu/velero/pkg/kuberesource"
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/vmware-tanzu/velero/pkg/plugin/velero"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
)
const testDriver = "csi.example.com"
@@ -163,6 +162,7 @@ func TestExecute(t *testing.T) {
SourcePVC: "testPVC",
SourceNamespace: "velero",
OperationTimeout: metav1.Duration{Duration: 1 * time.Minute},
ParentSnapshot: veleroshared.DataUploadParentSnapshotAuto,
},
},
},
@@ -2176,3 +2176,131 @@ func TestGetOrCreateVolumeHelper(t *testing.T) {
// The pvcPodCache should be the same instance
require.Same(t, cache1, action.pvcPodCache, "Expected same pvcPodCache instance on repeated calls")
}
func TestNewDataUpload(t *testing.T) {
tests := []struct {
name string
backupType velerov1api.BackupType
vsClassName *string
uploaderConfig *velerov1api.UploaderConfigForBackup
expectedParentSnap string
expectedDataMoverCfg map[string]string
}{
{
name: "Full backup type, no uploader config, no vs class name",
backupType: velerov1api.BackupTypeFull,
vsClassName: nil,
uploaderConfig: nil,
expectedParentSnap: "none",
expectedDataMoverCfg: nil,
},
{
name: "Incremental backup type, with uploader config, with vs class name",
backupType: velerov1api.BackupTypeIncremental,
vsClassName: ptr.To("test-vs-class"),
uploaderConfig: &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 10},
expectedParentSnap: "auto",
expectedDataMoverCfg: map[string]string{
uploaderUtil.ParallelFilesUpload: "10",
},
},
{
name: "Default backup type, uploader config with 0 parallel files",
backupType: "",
vsClassName: ptr.To("test-vs-class"),
uploaderConfig: &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 0},
expectedParentSnap: "auto",
expectedDataMoverCfg: nil,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
backup := &velerov1api.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-backup",
Namespace: "velero",
UID: types.UID("backup-uid"),
},
Spec: velerov1api.BackupSpec{
BackupType: tc.backupType,
DataMover: "velero",
StorageLocation: "default",
CSISnapshotTimeout: metav1.Duration{Duration: 10 * time.Minute},
UploaderConfig: tc.uploaderConfig,
},
}
vs := &snapshotv1api.VolumeSnapshot{
ObjectMeta: metav1.ObjectMeta{
Name: "test-vs",
},
Spec: snapshotv1api.VolumeSnapshotSpec{
VolumeSnapshotClassName: tc.vsClassName,
},
}
pvc := &corev1api.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pvc",
Namespace: "test-ns",
UID: types.UID("pvc-uid"),
},
Spec: corev1api.PersistentVolumeClaimSpec{
StorageClassName: ptr.To("test-storage-class"),
},
}
vsc := &snapshotv1api.VolumeSnapshotContent{
Spec: snapshotv1api.VolumeSnapshotContentSpec{
Driver: "test-driver",
},
}
operationID := "test-op-id"
fsType := "ext4"
du := newDataUpload(backup, vs, pvc, operationID, vsc, fsType)
require.NotNil(t, du)
assert.Equal(t, velerov2alpha1.SchemeGroupVersion.String(), du.APIVersion)
assert.Equal(t, "DataUpload", du.Kind)
assert.Equal(t, backup.Namespace, du.Namespace)
assert.Equal(t, backup.Name+"-", du.GenerateName)
require.Len(t, du.OwnerReferences, 1)
assert.Equal(t, velerov1api.SchemeGroupVersion.String(), du.OwnerReferences[0].APIVersion)
assert.Equal(t, "Backup", du.OwnerReferences[0].Kind)
assert.Equal(t, backup.Name, du.OwnerReferences[0].Name)
assert.Equal(t, backup.UID, du.OwnerReferences[0].UID)
assert.Equal(t, boolptr.True(), du.OwnerReferences[0].Controller)
expectedLabels := map[string]string{
velerov1api.BackupNameLabel: label.GetValidName(backup.Name),
velerov1api.BackupUIDLabel: string(backup.UID),
velerov1api.PVCUIDLabel: string(pvc.UID),
velerov1api.AsyncOperationIDLabel: operationID,
}
assert.Equal(t, expectedLabels, du.Labels)
assert.Equal(t, velerov2alpha1.SnapshotTypeCSI, du.Spec.SnapshotType)
assert.Equal(t, vs.Name, du.Spec.CSISnapshot.VolumeSnapshot)
assert.Equal(t, *pvc.Spec.StorageClassName, du.Spec.CSISnapshot.StorageClass)
assert.Equal(t, vsc.Spec.Driver, du.Spec.CSISnapshot.Driver)
if tc.vsClassName != nil {
assert.Equal(t, *tc.vsClassName, du.Spec.CSISnapshot.SnapshotClass)
} else {
assert.Empty(t, du.Spec.CSISnapshot.SnapshotClass)
}
assert.Equal(t, pvc.Name, du.Spec.SourcePVC)
assert.Equal(t, backup.Spec.DataMover, du.Spec.DataMover)
assert.Equal(t, backup.Spec.StorageLocation, du.Spec.BackupStorageLocation)
assert.Equal(t, pvc.Namespace, du.Spec.SourceNamespace)
assert.Equal(t, backup.Spec.CSISnapshotTimeout, du.Spec.OperationTimeout)
assert.Equal(t, fsType, du.Spec.SourceFSType)
assert.Equal(t, tc.expectedParentSnap, du.Spec.ParentSnapshot)
assert.Equal(t, tc.expectedDataMoverCfg, du.Spec.DataMoverConfig)
})
}
}
pkg/datamover/backup_micro_service.go
+1 -1
View File
@@ -204,7 +204,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
if err := dp.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.BackupStartParam{
RealSource: GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC),
ParentSnapshot: "",
ParentSnapshot: du.Spec.ParentSnapshot,
ForceFull: false,
Tags: tags,
VolumeID: r.volumeID,
pkg/datamover/util.go
+5 -2
View File
@@ -19,12 +19,15 @@ package datamover
import (
"fmt"
"github.com/vmware-tanzu/velero/pkg/uploader"
datamoverutil "github.com/vmware-tanzu/velero/pkg/util/datamover"
)
func GetUploaderType(dataMover string) string {
if datamoverutil.IsBuiltInDataMover(dataMover) {
return "kopia"
if datamoverutil.IsVeleroFSDataMover(dataMover) {
return uploader.KopiaType
} else if datamoverutil.IsVeleroBlockDataMover(dataMover) {
return uploader.BlockType
} else {
return dataMover
}
pkg/datamover/util_test.go
+10
View File
@@ -22,6 +22,16 @@ func TestGetUploaderType(t *testing.T) {
input: "velero",
want: "kopia",
},
{
name: "velero-fs dataMover is kopia",
input: "velero-fs",
want: "kopia",
},
{
name: "velero-block dataMover is velero-block",
input: "velero-block",
want: "velero-block",
},
{
name: "kopia dataMover is kopia",
input: "kopia",
pkg/util/datamover/datamover.go
+12 -1
View File
@@ -32,7 +32,18 @@ const (
// IsBuiltInDataMover reports whether the given data mover value refers to a
// Velero built-in data mover (an empty value or the default "velero" alias).
func IsBuiltInDataMover(dataMover string) bool {
return dataMover == "" || dataMover == DataMoverTypeVelero
return IsVeleroBlockDataMover(dataMover) || IsVeleroFSDataMover(dataMover)
}
func IsVeleroFSDataMover(dataMover string) bool {
if dataMover == "" || dataMover == DataMoverTypeVelero {
dataMover = DataMoverTypeVeleroFs
}
return dataMover == DataMoverTypeVeleroFs
}
func IsVeleroBlockDataMover(dataMover string) bool {
return dataMover == DataMoverTypeVeleroBlock
}
// GetDefaultBuiltInDataMover returns the data mover used when the default
pkg/util/datamover/datamover_test.go
+68
View File
@@ -38,6 +38,16 @@ func TestIsBuiltInDataMover(t *testing.T) {
dataMover: "velero",
want: true,
},
{
name: "velero-fs dataMover is builtin",
dataMover: "velero-fs",
want: true,
},
{
name: "velero-block dataMover is builtin",
dataMover: "velero-block",
want: true,
},
{
name: "kopia dataMover is not builtin",
dataMover: "kopia",
@@ -54,3 +64,61 @@ func TestIsBuiltInDataMover(t *testing.T) {
func TestGetDefaultBuiltInDataMover(t *testing.T) {
assert.Equal(t, DataMoverTypeVeleroFs, GetDefaultBuiltInDataMover())
}
func TestIsFSDataMover(t *testing.T) {
testcases := []struct {
name string
dataMover string
want bool
}{
{
name: "empty dataMover is fs",
dataMover: "",
want: true,
},
{
name: "velero dataMover is fs",
dataMover: "velero",
want: true,
},
{
name: "velero-fs dataMover is fs",
dataMover: "velero-fs",
want: true,
},
{
name: "velero-block dataMover is not fs",
dataMover: "velero-block",
want: false,
},
}
for _, tc := range testcases {
t.Run(tc.name, func(tt *testing.T) {
assert.Equal(tt, tc.want, IsVeleroFSDataMover(tc.dataMover))
})
}
}
func TestIsBlockDataMover(t *testing.T) {
testcases := []struct {
name string
dataMover string
want bool
}{
{
name: "velero-block dataMover is block",
dataMover: "velero-block",
want: true,
},
{
name: "velero-fs dataMover is not block",
dataMover: "velero-fs",
want: false,
},
}
for _, tc := range testcases {
t.Run(tc.name, func(tt *testing.T) {
assert.Equal(tt, tc.want, IsVeleroBlockDataMover(tc.dataMover))
})
}
}