Merge pull request #10479 from Lyndon-Li/report-incremental-fallback

Report incremental fallback message
This commit is contained in:
lyndon-li
2026-09-10 14:29:19 +08:00
committed by GitHub
26 changed files with 750 additions and 220 deletions
+1
View File
@@ -0,0 +1 @@
Enhance data mover progress update to include a message field so that critical activity messages such as incremental fallback could be saved to the message field of DU/DD/PVB/PVR
+15 -2
View File
@@ -42,7 +42,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
"github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/constant"
@@ -609,7 +608,21 @@ func (r *DataDownloadReconciler) OnDataDownloadProgress(ctx context.Context, nam
log := r.logger.WithField("datadownload", ddName) log := r.logger.WithField("datadownload", ddName)
if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: ddName}, log, func(dd *velerov2alpha1api.DataDownload) bool { if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: ddName}, log, func(dd *velerov2alpha1api.DataDownload) bool {
dd.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} if progress.TotalBytes != -1 {
dd.Status.Progress.TotalBytes = progress.TotalBytes
}
if progress.BytesDone != -1 {
dd.Status.Progress.BytesDone = progress.BytesDone
}
if progress.Message != "" {
message := progress.Message + ";"
if !strings.HasSuffix(dd.Status.Message, message) {
dd.Status.Message += message
}
}
return true return true
}); err != nil { }); err != nil {
log.WithError(err).Error("Failed to update progress") log.WithError(err).Error("Failed to update progress")
@@ -787,6 +787,15 @@ func TestOnDataDownloadProgress(t *testing.T) {
BytesDone: bytesDone, BytesDone: bytesDone,
}, },
}, },
{
name: "patch in progress phase with negative progress values and message",
dd: dataDownloadBuilder().Result(),
progress: uploader.Progress{
TotalBytes: -1,
BytesDone: -1,
Message: "some warning message",
},
},
{ {
name: "failed to get datadownload", name: "failed to get datadownload",
dd: dataDownloadBuilder().Result(), dd: dataDownloadBuilder().Result(),
@@ -815,20 +824,28 @@ func TestOnDataDownloadProgress(t *testing.T) {
require.NoError(t, r.client.Create(t.Context(), dd)) require.NoError(t, r.client.Create(t.Context(), dd))
// Create a Progress object // Create a Progress object
progress := &uploader.Progress{ progress := &test.progress
TotalBytes: totalBytes,
BytesDone: bytesDone,
}
// Call the OnDataDownloadProgress function // Call the OnDataDownloadProgress function
r.OnDataDownloadProgress(ctx, namespace, duName, progress) r.OnDataDownloadProgress(ctx, namespace, duName, progress)
if len(test.needErrs) != 0 && !test.needErrs[0] { if len(test.needErrs) != 0 && !test.needErrs[0] {
// Get the updated DataDownload object from the fake client // Get the updated DataDownload object from the fake client
updatedDu := &velerov2alpha1api.DataDownload{} updatedDd := &velerov2alpha1api.DataDownload{}
require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDd))
// Assert that the DataDownload object has been updated with the progress // Assert that the DataDownload object has been updated with the progress
assert.Equal(t, test.progress.TotalBytes, updatedDu.Status.Progress.TotalBytes) if progress.TotalBytes != -1 {
assert.Equal(t, test.progress.BytesDone, updatedDu.Status.Progress.BytesDone) assert.Equal(t, test.progress.TotalBytes, updatedDd.Status.Progress.TotalBytes)
} else {
assert.Equal(t, int64(0), updatedDd.Status.Progress.TotalBytes) // assuming default or original value
}
if progress.BytesDone != -1 {
assert.Equal(t, test.progress.BytesDone, updatedDd.Status.Progress.BytesDone)
} else {
assert.Equal(t, int64(0), updatedDd.Status.Progress.BytesDone) // assuming default or original value
}
if progress.Message != "" {
assert.Contains(t, updatedDd.Status.Message, progress.Message)
}
} }
}) })
} }
+15 -2
View File
@@ -42,7 +42,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
"github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/constant"
@@ -634,7 +633,21 @@ func (r *DataUploadReconciler) OnDataUploadProgress(ctx context.Context, namespa
log := r.logger.WithField("dataupload", duName) log := r.logger.WithField("dataupload", duName)
if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: duName}, log, func(du *velerov2alpha1api.DataUpload) bool { if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: duName}, log, func(du *velerov2alpha1api.DataUpload) bool {
du.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} if progress.TotalBytes != -1 {
du.Status.Progress.TotalBytes = progress.TotalBytes
}
if progress.BytesDone != -1 {
du.Status.Progress.BytesDone = progress.BytesDone
}
if progress.Message != "" {
message := progress.Message + ";"
if !strings.HasSuffix(du.Status.Message, message) {
du.Status.Message += message
}
}
return true return true
}); err != nil { }); err != nil {
log.WithError(err).Error("Failed to update progress") log.WithError(err).Error("Failed to update progress")
+23 -6
View File
@@ -809,6 +809,15 @@ func TestOnDataUploadProgress(t *testing.T) {
BytesDone: bytesDone, BytesDone: bytesDone,
}, },
}, },
{
name: "patch in progress phase with negative progress values and message",
du: dataUploadBuilder().Result(),
progress: uploader.Progress{
TotalBytes: -1,
BytesDone: -1,
Message: "some warning message",
},
},
{ {
name: "failed to get dataupload", name: "failed to get dataupload",
du: dataUploadBuilder().Result(), du: dataUploadBuilder().Result(),
@@ -837,10 +846,7 @@ func TestOnDataUploadProgress(t *testing.T) {
require.NoError(t, r.client.Create(t.Context(), du)) require.NoError(t, r.client.Create(t.Context(), du))
// Create a Progress object // Create a Progress object
progress := &uploader.Progress{ progress := &test.progress
TotalBytes: totalBytes,
BytesDone: bytesDone,
}
// Call the OnDataUploadProgress function // Call the OnDataUploadProgress function
r.OnDataUploadProgress(ctx, namespace, duName, progress) r.OnDataUploadProgress(ctx, namespace, duName, progress)
@@ -849,8 +855,19 @@ func TestOnDataUploadProgress(t *testing.T) {
updatedDu := &velerov2alpha1api.DataUpload{} updatedDu := &velerov2alpha1api.DataUpload{}
require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu))
// Assert that the DataUpload object has been updated with the progress // Assert that the DataUpload object has been updated with the progress
assert.Equal(t, test.progress.TotalBytes, updatedDu.Status.Progress.TotalBytes) if progress.TotalBytes != -1 {
assert.Equal(t, test.progress.BytesDone, updatedDu.Status.Progress.BytesDone) assert.Equal(t, test.progress.TotalBytes, updatedDu.Status.Progress.TotalBytes)
} else {
assert.Equal(t, int64(0), updatedDu.Status.Progress.TotalBytes) // assuming default or original value
}
if progress.BytesDone != -1 {
assert.Equal(t, test.progress.BytesDone, updatedDu.Status.Progress.BytesDone)
} else {
assert.Equal(t, int64(0), updatedDu.Status.Progress.BytesDone) // assuming default or original value
}
if progress.Message != "" {
assert.Contains(t, updatedDu.Status.Message, progress.Message)
}
} }
}) })
} }
+15 -2
View File
@@ -41,7 +41,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/reconcile"
veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/constant"
"github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/datapath"
@@ -628,7 +627,21 @@ func (r *PodVolumeBackupReconciler) OnDataPathProgress(ctx context.Context, name
log := r.logger.WithField("pvb", pvbName) log := r.logger.WithField("pvb", pvbName)
if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvbName}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvbName}, log, func(pvb *velerov1api.PodVolumeBackup) bool {
pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} if progress.TotalBytes != -1 {
pvb.Status.Progress.TotalBytes = progress.TotalBytes
}
if progress.BytesDone != -1 {
pvb.Status.Progress.BytesDone = progress.BytesDone
}
if progress.Message != "" {
message := progress.Message + ";"
if !strings.HasSuffix(pvb.Status.Message, message) {
pvb.Status.Message += message
}
}
return true return true
}); err != nil { }); err != nil {
log.WithError(err).Error("Failed to update progress") log.WithError(err).Error("Failed to update progress")
@@ -625,6 +625,15 @@ func TestOnPVBProgress(t *testing.T) {
BytesDone: bytesDone, BytesDone: bytesDone,
}, },
}, },
{
name: "patch in progress phase with negative progress values and message",
pvb: pvbBuilder().Result(),
progress: uploader.Progress{
TotalBytes: -1,
BytesDone: -1,
Message: "some warning message",
},
},
{ {
name: "failed to get pvb", name: "failed to get pvb",
pvb: pvbBuilder().Result(), pvb: pvbBuilder().Result(),
@@ -653,17 +662,25 @@ func TestOnPVBProgress(t *testing.T) {
require.NoError(t, r.client.Create(t.Context(), pvb)) require.NoError(t, r.client.Create(t.Context(), pvb))
// Create a Progress object // Create a Progress object
progress := &uploader.Progress{ progress := &test.progress
TotalBytes: totalBytes,
BytesDone: bytesDone,
}
r.OnDataPathProgress(ctx, namespace, pvbName, progress) r.OnDataPathProgress(ctx, namespace, pvbName, progress)
if len(test.needErrs) != 0 && !test.needErrs[0] { if len(test.needErrs) != 0 && !test.needErrs[0] {
updatedPvb := &velerov1api.PodVolumeBackup{} updatedPvb := &velerov1api.PodVolumeBackup{}
require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb))
assert.Equal(t, test.progress.TotalBytes, updatedPvb.Status.Progress.TotalBytes) if progress.TotalBytes != -1 {
assert.Equal(t, test.progress.BytesDone, updatedPvb.Status.Progress.BytesDone) assert.Equal(t, test.progress.TotalBytes, updatedPvb.Status.Progress.TotalBytes)
} else {
assert.Equal(t, int64(0), updatedPvb.Status.Progress.TotalBytes) // assuming default or original value
}
if progress.BytesDone != -1 {
assert.Equal(t, test.progress.BytesDone, updatedPvb.Status.Progress.BytesDone)
} else {
assert.Equal(t, int64(0), updatedPvb.Status.Progress.BytesDone) // assuming default or original value
}
if progress.Message != "" {
assert.Contains(t, updatedPvb.Status.Message, progress.Message)
}
} }
}) })
} }
@@ -44,7 +44,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/reconcile"
veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/constant"
"github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/datapath"
@@ -905,7 +904,21 @@ func (r *PodVolumeRestoreReconciler) OnDataPathProgress(ctx context.Context, nam
log := r.logger.WithField("PVR", pvrName) log := r.logger.WithField("PVR", pvrName)
if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvrName}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvrName}, log, func(pvr *velerov1api.PodVolumeRestore) bool {
pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} if progress.TotalBytes != -1 {
pvr.Status.Progress.TotalBytes = progress.TotalBytes
}
if progress.BytesDone != -1 {
pvr.Status.Progress.BytesDone = progress.BytesDone
}
if progress.Message != "" {
message := progress.Message + ";"
if !strings.HasSuffix(pvr.Status.Message, message) {
pvr.Status.Message += message
}
}
return true return true
}); err != nil { }); err != nil {
log.WithError(err).Error("Failed to update progress") log.WithError(err).Error("Failed to update progress")
@@ -1470,6 +1470,15 @@ func TestOnPodVolumeRestoreProgress(t *testing.T) {
BytesDone: bytesDone, BytesDone: bytesDone,
}, },
}, },
{
name: "patch in progress phase with negative progress values and message",
pvr: pvrBuilder().Result(),
progress: uploader.Progress{
TotalBytes: -1,
BytesDone: -1,
Message: "some warning message",
},
},
{ {
name: "failed to get pvr", name: "failed to get pvr",
pvr: pvrBuilder().Result(), pvr: pvrBuilder().Result(),
@@ -1498,17 +1507,25 @@ func TestOnPodVolumeRestoreProgress(t *testing.T) {
require.NoError(t, r.client.Create(t.Context(), pvr)) require.NoError(t, r.client.Create(t.Context(), pvr))
// Create a Progress object // Create a Progress object
progress := &uploader.Progress{ progress := &test.progress
TotalBytes: totalBytes,
BytesDone: bytesDone,
}
r.OnDataPathProgress(ctx, namespace, pvrName, progress) r.OnDataPathProgress(ctx, namespace, pvrName, progress)
if len(test.needErrs) != 0 && !test.needErrs[0] { if len(test.needErrs) != 0 && !test.needErrs[0] {
updatedPVR := &velerov1api.PodVolumeRestore{} updatedPVR := &velerov1api.PodVolumeRestore{}
require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR))
assert.Equal(t, test.progress.TotalBytes, updatedPVR.Status.Progress.TotalBytes) if progress.TotalBytes != -1 {
assert.Equal(t, test.progress.BytesDone, updatedPVR.Status.Progress.BytesDone) assert.Equal(t, test.progress.TotalBytes, updatedPVR.Status.Progress.TotalBytes)
} else {
assert.Equal(t, int64(0), updatedPVR.Status.Progress.TotalBytes) // assuming default or original value
}
if progress.BytesDone != -1 {
assert.Equal(t, test.progress.BytesDone, updatedPVR.Status.Progress.BytesDone)
} else {
assert.Equal(t, int64(0), updatedPVR.Status.Progress.BytesDone) // assuming default or original value
}
if progress.Message != "" {
assert.Contains(t, updatedPVR.Status.Message, progress.Message)
}
} }
}) })
} }
+5 -1
View File
@@ -278,7 +278,11 @@ func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, u
// UpdateProgress which implement ProgressUpdater interface to update progress status // UpdateProgress which implement ProgressUpdater interface to update progress status
func (dp *generalDataPath) UpdateProgress(p *uploader.Progress) { func (dp *generalDataPath) UpdateProgress(p *uploader.Progress) {
if dp.callbacks.OnProgress != nil { if dp.callbacks.OnProgress != nil {
dp.callbacks.OnProgress(context.Background(), dp.namespace, dp.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone}) dp.callbacks.OnProgress(context.Background(), dp.namespace, dp.jobName, &uploader.Progress{
TotalBytes: p.TotalBytes,
BytesDone: p.BytesDone,
Message: p.Message,
})
} }
} }
+104 -72
View File
@@ -40,6 +40,10 @@ type parentBackupInfo struct {
volumeID string volumeID string
} }
type backupInfo struct {
changeID string
}
// Backup backup specific sourcePath and update progress // Backup backup specific sourcePath and update progress
func Backup(ctx context.Context, blkUp Uploader, repoWriter udmrepo.BackupRepo, sourcePath string, realSource string, cbtSource cbtservice.SourceInfo, func Backup(ctx context.Context, blkUp Uploader, repoWriter udmrepo.BackupRepo, sourcePath string, realSource string, cbtSource cbtservice.SourceInfo,
forceFull bool, parentSnapshot string, cbtService cbtservice.Service, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { forceFull bool, parentSnapshot string, cbtService cbtservice.Service, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) {
@@ -106,14 +110,20 @@ func snapshotSource(
log.Info("Start to snapshot...") log.Info("Start to snapshot...")
snapshotStartTime := time.Now() snapshotStartTime := time.Now()
parentBackup := getParentBackupInfo(ctx, rep, forceFull, parentSnapshot, cbtSource.VolumeID, source.realSource, snapshotTags, log) bitmap := cbt.NewBitmap(blockSize, uint64(source.size), cbtSource.Snapshot, cbtSource.VolumeID)
bitmap := cbt.NewBitmap(blockSize, uint64(source.size), cbtSource.Snapshot, parentBackup.changeID, parentBackup.volumeID) parentBackup, err := getParentBackupInfo(ctx, rep, forceFull, parentSnapshot, cbtSource.VolumeID, source.realSource, snapshotTags, log)
if err != nil {
log.WithError(err).Warn("Failed to get parent backup info, fallback to full backup")
bitmap.SetError(errors.Wrap(err, "error getting parent backup info, fallback to full backup"))
} else {
bitmap.SetChangeID(parentBackup.changeID)
}
err := cbt.SetBitmapOrFull(ctx, cbtService, bitmap) err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap, false)
if err != nil { if err != nil {
parentBackup.parentObject = "" parentBackup.parentObject = ""
log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to real full backup", cbtSource) log.WithError(err).Warnf("Failed to create CBT with source %v", cbtSource)
} }
snap, backupSize, err := u.Backup(source, parentBackup.parentObject, bitmap.Iterator(), uploaderCfg) snap, backupSize, err := u.Backup(source, parentBackup.parentObject, bitmap.Iterator(), uploaderCfg)
@@ -147,55 +157,66 @@ func snapshotSource(
return string(snapID), backupSize, nil return string(snapID), backupSize, nil
} }
func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string, realSource string, snapshotTags map[string]string, log logrus.FieldLogger) parentBackupInfo { func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string,
var previous *udmrepo.Snapshot realSource string, snapshotTags map[string]string, log logrus.FieldLogger) (parentBackupInfo, error) {
if forceFull {
if !forceFull {
if parentSnapshot != "" {
snap, err := rep.GetSnapshot(ctx, udmrepo.ID(parentSnapshot))
if err != nil {
log.WithError(err).Warn("Failed to load previous snapshot, fallback to full backup")
} else {
previous = &snap
log.Infof("Using provided parent snapshot %s", parentSnapshot)
}
} else {
log.Infof("Searching for parent snapshot")
snap, err := findPreviousSnapshot(ctx, rep, realSource, snapshotTags, nil, log)
if err != nil {
log.WithError(err).Warn("Failed to search previous snapshot, fallback to full backup")
} else {
previous = &snap
log.Infof("Using previous snapshot %s", snap.ID)
}
}
} else {
log.Info("Forcing full snapshot") log.Info("Forcing full snapshot")
return parentBackupInfo{}, nil
} }
parentInfo := parentBackupInfo{} if volumeID == "" {
if previous != nil { return parentBackupInfo{}, errors.New("volumeID is not provided from the volume snapshot")
if previous.Tags == nil { }
log.Warnf("No tag from parent snapshot %s, fallback to full backup", previous.ID)
} else if previous.Tags[uploader.CBTChangeIDTag] == "" {
log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", previous.ID)
} else if previous.Tags[uploader.CBTVolumeIDTag] == "" {
log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", previous.ID)
} else if previous.Tags[uploader.CBTVolumeIDTag] != volumeID {
log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], previous.ID, volumeID)
} else if obj, err := loadObjectFromSnapshot(ctx, rep, previous); err != nil {
log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", previous.ID)
} else {
parentInfo.parentObject = obj
parentInfo.changeID = previous.Tags[uploader.CBTChangeIDTag]
parentInfo.volumeID = previous.Tags[uploader.CBTVolumeIDTag]
log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", previous.ID, previous.StartTime, previous.EndTime, previous.Description) var previous *udmrepo.Snapshot
if parentSnapshot != "" {
log.Infof("Loading provided parent snapshot %s", parentSnapshot)
snap, err := rep.GetSnapshot(ctx, udmrepo.ID(parentSnapshot))
if err != nil {
return parentBackupInfo{}, errors.Wrapf(err, "error loading previous snapshot")
} }
previous = &snap
} else {
log.Infof("Searching for parent snapshot")
snap, err := findPreviousSnapshot(ctx, rep, realSource, snapshotTags, nil, log)
if err != nil {
return parentBackupInfo{}, errors.Wrapf(err, "error searching previous snapshot")
}
previous = &snap
} }
return parentInfo if previous.Tags == nil {
return parentBackupInfo{}, errors.Errorf("no tag from parent snapshot %s", previous.ID)
}
if previous.Tags[uploader.CBTChangeIDTag] == "" {
return parentBackupInfo{}, errors.Errorf("no ChangeID tag from parent snapshot %s", previous.ID)
}
if previous.Tags[uploader.CBTVolumeIDTag] == "" {
return parentBackupInfo{}, errors.Errorf("no VolumeID tag from parent snapshot %s", previous.ID)
}
if previous.Tags[uploader.CBTVolumeIDTag] != volumeID {
return parentBackupInfo{}, errors.Errorf("VolumeID %s from parent snapshot %s is not expected as %s", previous.Tags[uploader.CBTVolumeIDTag], previous.ID, volumeID)
}
obj, err := loadObjectFromSnapshot(ctx, rep, previous)
if err != nil {
return parentBackupInfo{}, errors.Wrapf(err, "error loading object from parent snapshot %s", previous.ID)
}
log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", previous.ID, previous.StartTime, previous.EndTime, previous.Description)
return parentBackupInfo{
parentObject: obj,
changeID: previous.Tags[uploader.CBTChangeIDTag],
volumeID: previous.Tags[uploader.CBTVolumeIDTag],
}, nil
} }
// Restore restore specific sourcePath with given snapshotID and update progress // Restore restore specific sourcePath with given snapshotID and update progress
@@ -208,34 +229,19 @@ func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapsh
} }
log.Infof("Restore from snapshot %s, incremental %v, cbt source %v, description %s, created time %v, tags %v", snapshotID, incremental, cbtSource, snapshot.Description, snapshot.EndTime, snapshot.Tags) log.Infof("Restore from snapshot %s, incremental %v, cbt source %v, description %s, created time %v, tags %v", snapshotID, incremental, cbtSource, snapshot.Description, snapshot.EndTime, snapshot.Tags)
var volumeSnapshot, changeID, volumeID string bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), cbtSource.Snapshot, cbtSource.VolumeID)
if incremental {
if snapshot.Tags == nil {
log.Warnf("No tag from snapshot %s, fallback to full restore", snapshotID)
incremental = false
} else if snapshot.Tags[uploader.CBTChangeIDTag] == "" {
log.Warnf("No ChangeID tag from snapshot %s, fallback to full restore", snapshotID)
incremental = false
} else if snapshot.Tags[uploader.CBTVolumeIDTag] == "" {
log.Warnf("No VolumeID tag from snapshot %s, fallback to full restore", snapshotID)
incremental = false
} else if cbtSource.VolumeID == "" {
log.Warnf("No VolumeID in cbt source %v, fallback to full restore", cbtSource)
incremental = false
} else if snapshot.Tags[uploader.CBTVolumeIDTag] != cbtSource.VolumeID {
log.Warnf("VolumeID %s from snapshot %s is not expected as %s, fallback to full restore", snapshot.Tags[uploader.CBTVolumeIDTag], snapshotID, cbtSource.VolumeID)
incremental = false
} else {
volumeSnapshot = cbtSource.Snapshot
changeID = snapshot.Tags[uploader.CBTChangeIDTag]
volumeID = snapshot.Tags[uploader.CBTVolumeIDTag]
}
}
bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), volumeSnapshot, changeID, volumeID)
if incremental { if incremental {
if err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap); err != nil { if bkInfo, err := getBackupInfo(snapshot, cbtSource.VolumeID); err != nil {
log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to full restore", cbtSource) log.WithError(err).Warn("Failed to get backup info, fallback to full restore")
bitmap.SetError(errors.Wrap(err, "error getting backup info, fallback to full restore"))
bitmap.SetFull()
} else {
bitmap.SetChangeID(bkInfo.changeID)
if err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap, true); err != nil {
log.WithError(err).Warnf("Failed to create CBT with source %v", cbtSource)
}
} }
} else { } else {
bitmap.SetFull() bitmap.SetFull()
@@ -273,6 +279,32 @@ func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapsh
return incrementalBytes, totalSize, nil return incrementalBytes, totalSize, nil
} }
func getBackupInfo(snapshot udmrepo.Snapshot, volumeID string) (backupInfo, error) {
if snapshot.Tags == nil {
return backupInfo{}, errors.Errorf("no tag from snapshot %s", snapshot.ID)
}
if snapshot.Tags[uploader.CBTChangeIDTag] == "" {
return backupInfo{}, errors.Errorf("no ChangeID tag from snapshot %s", snapshot.ID)
}
if snapshot.Tags[uploader.CBTVolumeIDTag] == "" {
return backupInfo{}, errors.Errorf("no VolumeID tag from snapshot %s", snapshot.ID)
}
if volumeID == "" {
return backupInfo{}, errors.New("no VolumeID tag from the volume snapshot")
}
if snapshot.Tags[uploader.CBTVolumeIDTag] != volumeID {
return backupInfo{}, errors.Errorf("volumeID %s from snapshot %s is not expected as %s", snapshot.Tags[uploader.CBTVolumeIDTag], snapshot.ID, volumeID)
}
return backupInfo{
changeID: snapshot.Tags[uploader.CBTChangeIDTag],
}, nil
}
func findPreviousSnapshot(ctx context.Context, rep udmrepo.BackupRepo, path string, snapshotTags map[string]string, noLaterThan *time.Time, log logrus.FieldLogger) (udmrepo.Snapshot, error) { func findPreviousSnapshot(ctx context.Context, rep udmrepo.BackupRepo, path string, snapshotTags map[string]string, noLaterThan *time.Time, log logrus.FieldLogger) (udmrepo.Snapshot, error) {
snaps, err := rep.ListSnapshot(ctx, path) snaps, err := rep.ListSnapshot(ctx, path)
if err != nil { if err != nil {
+161 -13
View File
@@ -379,11 +379,12 @@ func TestGetParentBackupInfoLogsDiscoveredParentID(t *testing.T) {
SubObjects: []udmrepo.ObjectMetadata{{ID: udmrepo.ID("parent-obj")}}, SubObjects: []udmrepo.ObjectMetadata{{ID: udmrepo.ID("parent-obj")}},
}, nil) }, nil)
info := getParentBackupInfo( info, err := getParentBackupInfo(
context.Background(), repo, context.Background(), repo,
false, "", // no explicit parent -> discovery branch false, "", // no explicit parent -> discovery branch
volumeID, realSource, snapshotTags, logger, volumeID, realSource, snapshotTags, logger,
) )
require.NoError(t, err)
require.Equal(t, udmrepo.ID("parent-obj"), info.parentObject) require.Equal(t, udmrepo.ID("parent-obj"), info.parentObject)
@@ -408,6 +409,7 @@ func TestGetParentBackupInfo(t *testing.T) {
} }
validSnap := udmrepo.Snapshot{ validSnap := udmrepo.Snapshot{
ID: "snap-valid",
RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, RootObject: udmrepo.ObjectMetadata{ID: "root-obj"},
Tags: map[string]string{ Tags: map[string]string{
uploader.CBTChangeIDTag: "cid-abc", uploader.CBTChangeIDTag: "cid-abc",
@@ -421,7 +423,10 @@ func TestGetParentBackupInfo(t *testing.T) {
name string name string
forceFull bool forceFull bool
parentSnapshot string parentSnapshot string
emptyVolID bool
setupMocks func(repo *udmrepomocks.BackupRepo) setupMocks func(repo *udmrepomocks.BackupRepo)
expectErr bool
expectedErrStr string
expectEmpty bool expectEmpty bool
expectedParent udmrepo.ID expectedParent udmrepo.ID
expectedCID string expectedCID string
@@ -432,6 +437,13 @@ func TestGetParentBackupInfo(t *testing.T) {
forceFull: true, forceFull: true,
expectEmpty: true, expectEmpty: true,
}, },
{
name: "volumeID not provided",
emptyVolID: true,
expectEmpty: true,
expectErr: true,
expectedErrStr: "volumeID is not provided from the volume snapshot",
},
{ {
name: "GetSnapshot fails — falls back to full", name: "GetSnapshot fails — falls back to full",
parentSnapshot: "snap-parent", parentSnapshot: "snap-parent",
@@ -439,46 +451,69 @@ func TestGetParentBackupInfo(t *testing.T) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-parent")). repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-parent")).
Return(udmrepo.Snapshot{}, errors.New("not found")) Return(udmrepo.Snapshot{}, errors.New("not found"))
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "error loading previous snapshot",
}, },
{ {
name: "parent snapshot has nil tags — falls back to full", name: "parent snapshot has nil tags — falls back to full",
parentSnapshot: "snap-notags", parentSnapshot: "snap-notags",
setupMocks: func(repo *udmrepomocks.BackupRepo) { setupMocks: func(repo *udmrepomocks.BackupRepo) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-notags")). repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-notags")).
Return(udmrepo.Snapshot{Tags: nil}, nil) Return(udmrepo.Snapshot{ID: "snap-notags", Tags: nil}, nil)
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "no tag from parent snapshot snap-notags",
}, },
{ {
name: "parent snapshot missing ChangeID tag — falls back to full", name: "parent snapshot missing ChangeID tag — falls back to full",
parentSnapshot: "snap-nocid", parentSnapshot: "snap-nocid",
setupMocks: func(repo *udmrepomocks.BackupRepo) { setupMocks: func(repo *udmrepomocks.BackupRepo) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-nocid")). repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-nocid")).
Return(udmrepo.Snapshot{Tags: map[string]string{uploader.CBTVolumeIDTag: volumeID}}, nil) Return(udmrepo.Snapshot{ID: "snap-nocid", Tags: map[string]string{uploader.CBTVolumeIDTag: volumeID}}, nil)
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "no ChangeID tag from parent snapshot snap-nocid",
}, },
{ {
name: "parent snapshot missing VolumeID tag — falls back to full", name: "parent snapshot missing VolumeID tag — falls back to full",
parentSnapshot: "snap-novid", parentSnapshot: "snap-novid",
setupMocks: func(repo *udmrepomocks.BackupRepo) { setupMocks: func(repo *udmrepomocks.BackupRepo) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-novid")). repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-novid")).
Return(udmrepo.Snapshot{Tags: map[string]string{uploader.CBTChangeIDTag: "cid"}}, nil) Return(udmrepo.Snapshot{ID: "snap-novid", Tags: map[string]string{uploader.CBTChangeIDTag: "cid"}}, nil)
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "no VolumeID tag from parent snapshot snap-novid",
}, },
{ {
name: "parent snapshot VolumeID mismatch — falls back to full", name: "parent snapshot VolumeID mismatch — falls back to full",
parentSnapshot: "snap-vidmismatch", parentSnapshot: "snap-vidmismatch",
setupMocks: func(repo *udmrepomocks.BackupRepo) { setupMocks: func(repo *udmrepomocks.BackupRepo) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-vidmismatch")). repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-vidmismatch")).
Return(udmrepo.Snapshot{Tags: map[string]string{ Return(udmrepo.Snapshot{ID: "snap-vidmismatch", Tags: map[string]string{
uploader.CBTChangeIDTag: "cid", uploader.CBTChangeIDTag: "cid",
uploader.CBTVolumeIDTag: "different-vol", uploader.CBTVolumeIDTag: "different-vol",
}}, nil) }}, nil)
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "VolumeID different-vol from parent snapshot snap-vidmismatch is not expected as vol-123",
},
{
name: "loadObjectFromSnapshot fails — falls back to full",
parentSnapshot: "snap-valid",
setupMocks: func(repo *udmrepomocks.BackupRepo) {
repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-valid")).
Return(validSnap, nil)
repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")).
Return(nil, errors.New("read error"))
},
expectEmpty: true,
expectErr: true,
expectedErrStr: "error loading object from parent snapshot snap-valid",
}, },
{ {
name: "valid parent snapshot — returns parent info", name: "valid parent snapshot — returns parent info",
@@ -499,7 +534,9 @@ func TestGetParentBackupInfo(t *testing.T) {
repo.On("ListSnapshot", mock.Anything, realSource). repo.On("ListSnapshot", mock.Anything, realSource).
Return(nil, errors.New("list error")) Return(nil, errors.New("list error"))
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "error searching previous snapshot",
}, },
{ {
name: "no parentSnapshot — no matching snapshot — falls back to full", name: "no parentSnapshot — no matching snapshot — falls back to full",
@@ -507,7 +544,9 @@ func TestGetParentBackupInfo(t *testing.T) {
repo.On("ListSnapshot", mock.Anything, realSource). repo.On("ListSnapshot", mock.Anything, realSource).
Return([]udmrepo.Snapshot{{Tags: map[string]string{"other": "tag"}}}, nil) Return([]udmrepo.Snapshot{{Tags: map[string]string{"other": "tag"}}}, nil)
}, },
expectEmpty: true, expectEmpty: true,
expectErr: true,
expectedErrStr: "error searching previous snapshot",
}, },
{ {
name: "no parentSnapshot — matching snapshot found — returns parent info", name: "no parentSnapshot — matching snapshot found — returns parent info",
@@ -532,7 +571,21 @@ func TestGetParentBackupInfo(t *testing.T) {
tc.setupMocks(mockRepo) tc.setupMocks(mockRepo)
} }
info := getParentBackupInfo(ctx, mockRepo, tc.forceFull, tc.parentSnapshot, volumeID, realSource, snapshotTags, testLog()) volID := volumeID
if tc.emptyVolID {
volID = ""
}
info, err := getParentBackupInfo(ctx, mockRepo, tc.forceFull, tc.parentSnapshot, volID, realSource, snapshotTags, testLog())
if tc.expectErr {
require.Error(t, err)
if tc.expectedErrStr != "" {
assert.Contains(t, err.Error(), tc.expectedErrStr)
}
} else {
require.NoError(t, err)
}
if tc.expectEmpty { if tc.expectEmpty {
assert.Empty(t, info.parentObject) assert.Empty(t, info.parentObject)
@@ -547,6 +600,101 @@ func TestGetParentBackupInfo(t *testing.T) {
} }
} }
func TestGetBackupInfo(t *testing.T) {
const volumeID = "vol-123"
validSnap := udmrepo.Snapshot{
ID: "snap-valid",
Tags: map[string]string{
uploader.CBTChangeIDTag: "cid-abc",
uploader.CBTVolumeIDTag: volumeID,
},
}
testCases := []struct {
name string
snapshot udmrepo.Snapshot
volumeID string
expectErr bool
expectedErrStr string
expectedCID string
}{
{
name: "nil tags",
snapshot: udmrepo.Snapshot{ID: "snap-nil-tags"},
volumeID: volumeID,
expectErr: true,
expectedErrStr: "no tag from snapshot snap-nil-tags",
},
{
name: "missing ChangeID tag",
snapshot: udmrepo.Snapshot{
ID: "snap-no-cid",
Tags: map[string]string{uploader.CBTVolumeIDTag: volumeID},
},
volumeID: volumeID,
expectErr: true,
expectedErrStr: "no ChangeID tag from snapshot snap-no-cid",
},
{
name: "missing VolumeID tag",
snapshot: udmrepo.Snapshot{
ID: "snap-no-vid",
Tags: map[string]string{uploader.CBTChangeIDTag: "cid-abc"},
},
volumeID: volumeID,
expectErr: true,
expectedErrStr: "no VolumeID tag from snapshot snap-no-vid",
},
{
name: "empty volumeID parameter",
snapshot: udmrepo.Snapshot{
ID: "snap-valid",
Tags: map[string]string{
uploader.CBTChangeIDTag: "cid-abc",
uploader.CBTVolumeIDTag: volumeID,
},
},
volumeID: "",
expectErr: true,
expectedErrStr: "no VolumeID tag from the volume snapshot",
},
{
name: "volumeID mismatch",
snapshot: udmrepo.Snapshot{
ID: "snap-vid-mismatch",
Tags: map[string]string{
uploader.CBTChangeIDTag: "cid-abc",
uploader.CBTVolumeIDTag: "other-vol",
},
},
volumeID: volumeID,
expectErr: true,
expectedErrStr: "volumeID other-vol from snapshot snap-vid-mismatch is not expected as vol-123",
},
{
name: "valid snapshot",
snapshot: validSnap,
volumeID: volumeID,
expectErr: false,
expectedCID: "cid-abc",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
info, err := getBackupInfo(tc.snapshot, tc.volumeID)
if tc.expectErr {
require.Error(t, err)
assert.Contains(t, err.Error(), tc.expectedErrStr)
} else {
require.NoError(t, err)
assert.Equal(t, tc.expectedCID, info.changeID)
}
})
}
}
func TestFindPreviousSnapshot(t *testing.T) { func TestFindPreviousSnapshot(t *testing.T) {
snapshotTags := map[string]string{ snapshotTags := map[string]string{
uploader.SnapshotRequesterTag: "test-requester", uploader.SnapshotRequesterTag: "test-requester",
+12
View File
@@ -86,6 +86,12 @@ func (blkup *blockUploader) Backup(source sourceInfo, parentObject udmrepo.ID, b
return udmrepo.Snapshot{}, 0, errors.New("bitmap is not available") return udmrepo.Snapshot{}, 0, errors.New("bitmap is not available")
} }
if bitmap.Errors() != nil {
for _, err := range bitmap.Errors() {
blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: -1, TotalBytes: -1, Message: err.Error()})
}
}
backupMode := udmrepo.ObjectDataBackupModeInc backupMode := udmrepo.ObjectDataBackupModeInc
if parentObject == "" { if parentObject == "" {
backupMode = udmrepo.ObjectDataBackupModeFull backupMode = udmrepo.ObjectDataBackupModeFull
@@ -153,6 +159,12 @@ func (blkup *blockUploader) Restore(snapshot udmrepo.Snapshot, dest destInfo, bi
return 0, 0, errors.New("bitmap is not available") return 0, 0, errors.New("bitmap is not available")
} }
if bitmap.Errors() != nil {
for _, err := range bitmap.Errors() {
blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: -1, TotalBytes: -1, Message: err.Error()})
}
}
meta, err := blkup.repoWriter.ReadMetadata(blkup.ctx, snapshot.RootObject.ID) meta, err := blkup.repoWriter.ReadMetadata(blkup.ctx, snapshot.RootObject.ID)
if err != nil { if err != nil {
return 0, 0, errors.Wrapf(err, "error reading snapshot metadata for %s", snapshot.ID) return 0, 0, errors.Wrapf(err, "error reading snapshot metadata for %s", snapshot.ID)
+7
View File
@@ -295,6 +295,7 @@ func TestBlockUploaderBackup(t *testing.T) {
var iterator cbt.Iterator var iterator cbt.Iterator
if !tc.nilBitmap { if !tc.nilBitmap {
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
iterator = iterMock iterator = iterMock
backupMode := udmrepo.ObjectDataBackupModeInc backupMode := udmrepo.ObjectDataBackupModeInc
@@ -573,6 +574,7 @@ func TestRestoreData(t *testing.T) {
reader := bytes.NewReader(data) reader := bytes.NewReader(data)
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
iterMock.On("Count").Return(uint64(1)) iterMock.On("Count").Return(uint64(1))
iterMock.On("Next").Return(uint64(0), true).Once() iterMock.On("Next").Return(uint64(0), true).Once()
iterMock.On("Next").Return(uint64(0), false) iterMock.On("Next").Return(uint64(0), false)
@@ -603,6 +605,7 @@ func TestRestoreData(t *testing.T) {
reader := &errReader{err: errors.New("read error")} reader := &errReader{err: errors.New("read error")}
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
iterMock.On("Count").Return(uint64(1)) iterMock.On("Count").Return(uint64(1))
iterMock.On("Next").Return(uint64(0), true).Once() iterMock.On("Next").Return(uint64(0), true).Once()
iterMock.On("Next").Return(uint64(0), false) iterMock.On("Next").Return(uint64(0), false)
@@ -623,6 +626,7 @@ func TestBlockUploaderRestore(t *testing.T) {
repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(nil, errors.New("meta not found")) repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(nil, errors.New("meta not found"))
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
_, _, err := blkup.Restore(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root-id"}}, destInfo{}, iterMock, nil) _, _, err := blkup.Restore(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root-id"}}, destInfo{}, iterMock, nil)
require.Error(t, err) require.Error(t, err)
assert.Contains(t, err.Error(), "meta not found") assert.Contains(t, err.Error(), "meta not found")
@@ -680,6 +684,7 @@ func TestBlockUploaderRestore(t *testing.T) {
} }
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
iterMock.On("Count").Return(uint64(1)) iterMock.On("Count").Return(uint64(1))
iterMock.On("Next").Return(uint64(0), true).Once() iterMock.On("Next").Return(uint64(0), true).Once()
iterMock.On("Next").Return(uint64(0), false) iterMock.On("Next").Return(uint64(0), false)
@@ -708,6 +713,7 @@ func TestBlockUploaderRestore(t *testing.T) {
} }
dest := destInfo{size: 4194304, path: "/dev/target"} dest := destInfo{size: 4194304, path: "/dev/target"}
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
_, _, err := blkup.Restore(snap, dest, iterMock, nil) _, _, err := blkup.Restore(snap, dest, iterMock, nil)
require.Error(t, err) require.Error(t, err)
@@ -732,6 +738,7 @@ func TestBlockUploaderRestore(t *testing.T) {
} }
dest := destInfo{size: 512, path: "/dev/small"} dest := destInfo{size: 512, path: "/dev/small"}
iterMock := cbtmocks.NewIterator(t) iterMock := cbtmocks.NewIterator(t)
iterMock.On("Errors").Return(nil).Maybe()
_, _, err := blkup.Restore(snap, dest, iterMock, nil) _, _, err := blkup.Restore(snap, dest, iterMock, nil)
require.Error(t, err) require.Error(t, err)
+22 -2
View File
@@ -36,6 +36,7 @@ type bitmapImpl struct {
snapshot string snapshot string
changeID string changeID string
volumeID string volumeID string
cbtErrors []error
} }
type bitmapIterator struct { type bitmapIterator struct {
@@ -43,14 +44,13 @@ type bitmapIterator struct {
iterator roaring.IntPeekable iterator roaring.IntPeekable
} }
func NewBitmap(blockSize uint, length uint64, snapshot string, changeID string, volumeID string) types.Bitmap { func NewBitmap(blockSize uint, length uint64, snapshot string, volumeID string) types.Bitmap {
return &bitmapImpl{ return &bitmapImpl{
bitmap: roaring.New(), bitmap: roaring.New(),
blockSize: blockSize, blockSize: blockSize,
blockSizeLog: bits.Len(blockSize) - 1, blockSizeLog: bits.Len(blockSize) - 1,
length: length, length: length,
snapshot: snapshot, snapshot: snapshot,
changeID: changeID,
volumeID: volumeID, volumeID: volumeID,
} }
} }
@@ -81,6 +81,10 @@ func (c *bitmapImpl) Snapshot() string {
return c.snapshot return c.snapshot
} }
func (c *bitmapImpl) SetChangeID(id string) {
c.changeID = id
}
func (c *bitmapImpl) ChangeID() string { func (c *bitmapImpl) ChangeID() string {
return c.changeID return c.changeID
} }
@@ -89,6 +93,18 @@ func (c *bitmapImpl) VolumeID() string {
return c.volumeID return c.volumeID
} }
func (c *bitmapImpl) SetError(err error) {
if err == nil {
return
}
c.cbtErrors = append(c.cbtErrors, err)
}
func (c *bitmapImpl) Errors() []error {
return c.cbtErrors
}
func (c *bitmapImpl) Iterator() types.Iterator { func (c *bitmapImpl) Iterator() types.Iterator {
if c.bitmap == nil { if c.bitmap == nil {
return nil return nil
@@ -115,3 +131,7 @@ func (c *bitmapIterator) Count() uint64 {
func (c *bitmapIterator) BlockSize() uint { func (c *bitmapIterator) BlockSize() uint {
return c.blockSize return c.blockSize
} }
func (c *bitmapIterator) Errors() []error {
return c.cbtErrors
}
+18 -5
View File
@@ -17,6 +17,7 @@ limitations under the License.
package cbt package cbt
import ( import (
"errors"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -24,10 +25,18 @@ import (
) )
func TestBitmapProperties(t *testing.T) { func TestBitmapProperties(t *testing.T) {
b := NewBitmap(1024*1024, 10000*1024*1024, "snap-1", "change-1", "vol-1") b := NewBitmap(1024*1024, 10000*1024*1024, "snap-1", "vol-1")
assert.Equal(t, "snap-1", b.Snapshot()) assert.Equal(t, "snap-1", b.Snapshot())
assert.Equal(t, "change-1", b.ChangeID()) assert.Empty(t, b.ChangeID())
assert.Equal(t, "vol-1", b.VolumeID()) assert.Equal(t, "vol-1", b.VolumeID())
assert.Empty(t, b.Errors())
b.SetChangeID("change-1")
assert.Equal(t, "change-1", b.ChangeID())
err := errors.New("test error")
b.SetError(err)
assert.Equal(t, []error{err}, b.Errors())
} }
func TestBitmapSet(t *testing.T) { func TestBitmapSet(t *testing.T) {
@@ -138,7 +147,7 @@ func TestBitmapSet(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
b := NewBitmap(tt.blockSize, tt.totalLength, "snap-1", "change-1", "vol-1") b := NewBitmap(tt.blockSize, tt.totalLength, "snap-1", "vol-1")
for _, call := range tt.setCalls { for _, call := range tt.setCalls {
b.Set(call.offset, call.length) b.Set(call.offset, call.length)
@@ -174,7 +183,7 @@ func TestBitmapSetFull(t *testing.T) {
// block 0: 0 - 1MB // block 0: 0 - 1MB
// block 1: 1MB - 2MB // block 1: 1MB - 2MB
// block 2: 2MB - 3MB // block 2: 2MB - 3MB
b := NewBitmap(mb, 3*mb, "snap-1", "change-1", "vol-1") b := NewBitmap(mb, 3*mb, "snap-1", "vol-1")
b.SetFull() b.SetFull()
iter := b.Iterator() iter := b.Iterator()
@@ -199,7 +208,10 @@ func TestBitmapIterator(t *testing.T) {
const mb = 1024 * 1024 const mb = 1024 * 1024
const gb = 1024 * 1024 * 1024 const gb = 1024 * 1024 * 1024
b := NewBitmap(mb, 10*gb, "snap-1", "change-1", "vol-1") b := NewBitmap(mb, 10*gb, "snap-1", "vol-1")
b.SetChangeID("change-1")
err := errors.New("test error")
b.SetError(err)
// Set multiple ranges to test contiguous iteration // Set multiple ranges to test contiguous iteration
b.Set(mb, 100) // Block 1 b.Set(mb, 100) // Block 1
@@ -214,6 +226,7 @@ func TestBitmapIterator(t *testing.T) {
assert.Equal(t, "change-1", iter.ChangeID()) assert.Equal(t, "change-1", iter.ChangeID())
assert.Equal(t, "vol-1", iter.VolumeID()) assert.Equal(t, "vol-1", iter.VolumeID())
assert.Equal(t, uint(mb), iter.BlockSize()) assert.Equal(t, uint(mb), iter.BlockSize())
assert.Equal(t, []error{err}, iter.Errors())
assert.Equal(t, uint64(7), iter.Count()) // 1 + 5 + 1 = 7 blocks assert.Equal(t, uint64(7), iter.Count()) // 1 + 5 + 1 = 7 blocks
expectedOffsets := []uint64{ expectedOffsets := []uint64{
+48 -9
View File
@@ -26,36 +26,75 @@ import (
) )
// SetBitmapOrFull translates the allocated/changed blocks from CBT service to the given bitmap or set the bitmap to full when error happens // SetBitmapOrFull translates the allocated/changed blocks from CBT service to the given bitmap or set the bitmap to full when error happens
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap types.Bitmap) (err error) { func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap types.Bitmap, incOnly bool) (ret error) {
setFull := false
defer func() { defer func() {
if err != nil { bitmap.SetError(ret)
if setFull {
bitmap.SetFull() bitmap.SetFull()
} }
}() }()
if service == nil { if service == nil {
return errors.New("CBT service is absent") setFull = true
return errors.New("CBT service is absent, fallback to real full")
} }
if bitmap.Snapshot() == "" { if bitmap.Snapshot() == "" {
return errors.New("invalid snapshot") setFull = true
return errors.New("invalid snapshot, fallback to real full")
} }
if bitmap.ChangeID() == "" { if incOnly && bitmap.ChangeID() == "" {
return errors.Wrapf(service.GetAllocatedBlocks(ctx, bitmap.Snapshot(), func(blocks []cbtservice.Range) error { setFull = true
return errors.New("invalid changeID, fallback to real full")
}
var changedErr error
if bitmap.ChangeID() != "" {
err := service.GetChangedBlocks(ctx, bitmap.Snapshot(), bitmap.ChangeID(), func(blocks []cbtservice.Range) error {
for _, b := range blocks { for _, b := range blocks {
bitmap.Set(b.Offset, b.Length) bitmap.Set(b.Offset, b.Length)
} }
return nil return nil
}), "error getting allocated blocks from CBT service") })
if err == nil {
return nil
}
if incOnly {
setFull = true
return errors.Wrap(err, "error getting changed blocks from CBT service, fallback to real full")
}
changedErr = err
} }
return errors.Wrapf(service.GetChangedBlocks(ctx, bitmap.Snapshot(), bitmap.ChangeID(), func(blocks []cbtservice.Range) error { err := service.GetAllocatedBlocks(ctx, bitmap.Snapshot(), func(blocks []cbtservice.Range) error {
for _, b := range blocks { for _, b := range blocks {
bitmap.Set(b.Offset, b.Length) bitmap.Set(b.Offset, b.Length)
} }
return nil return nil
}), "error getting changed blocks from CBT service") })
if err != nil {
setFull = true
if changedErr != nil {
return errors.Wrap(err, "error getting both changed and allocated blocks from CBT service, fallback to real full")
} else {
return errors.Wrap(err, "error getting allocated blocks from CBT service, fallback to real full")
}
}
if changedErr != nil {
return errors.Wrap(changedErr, "error getting changed blocks from CBT service, fallback to full")
}
return nil
} }
+132 -65
View File
@@ -21,102 +21,148 @@ import (
"errors" "errors"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/cbtservice"
cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks" cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
cbtmocks "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types/mocks"
) )
func TestSetBitmapOrFull(t *testing.T) { func TestSetBitmapOrFull(t *testing.T) {
const mb = 1024 * 1024
tests := []struct { tests := []struct {
name string name string
nilService bool nilService bool
setupMocks func(*cbtservicemocks.Service, *cbtmocks.Bitmap) incOnly bool
snapshotID string
changeID string
setupMocks func(*cbtservicemocks.Service)
expectedErrStr string expectedErrStr string
expectedCount uint64
expectedNext []uint64
}{ }{
{ {
name: "nil service", name: "nil service",
nilService: true, nilService: true,
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) { snapshotID: "snap-1",
bmp.On("SetFull").Return() changeID: "change-1",
}, setupMocks: func(svc *cbtservicemocks.Service) {},
expectedErrStr: "CBT service is absent", expectedErrStr: "CBT service is absent, fallback to real full",
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
}, },
{ {
name: "invalid snapshot", name: "invalid snapshot",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) { snapshotID: "",
bmp.On("Snapshot").Return("") setupMocks: func(svc *cbtservicemocks.Service) {},
bmp.On("SetFull").Return() expectedErrStr: "invalid snapshot, fallback to real full",
}, expectedCount: 3,
expectedErrStr: "invalid snapshot", expectedNext: []uint64{0, mb, 2 * mb},
}, },
{ {
name: "allocated blocks success", name: "invalid changeID",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) { incOnly: true,
bmp.On("Snapshot").Return("snap-1") snapshotID: "snap-1",
bmp.On("ChangeID").Return("") changeID: "",
setupMocks: func(svc *cbtservicemocks.Service) {},
expectedErrStr: "invalid changeID, fallback to real full",
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "allocated blocks success",
snapshotID: "snap-1",
changeID: "",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(2).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 0, Length: uint64(mb)},
{Offset: uint64(2 * mb), Length: uint64(mb)},
})
}).Return(nil)
},
expectedCount: 2,
expectedNext: []uint64{0, 2 * mb},
},
{
name: "allocated blocks error",
snapshotID: "snap-1",
changeID: "",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
},
expectedErrStr: "error getting allocated blocks from CBT service, fallback to real full: mock alloc error",
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "changed blocks success",
snapshotID: "snap-1",
changeID: "change-1",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(3).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: uint64(mb), Length: uint64(mb)},
})
}).Return(nil)
},
expectedCount: 1,
expectedNext: []uint64{mb},
},
{
name: "changed blocks error with incOnly",
incOnly: true,
snapshotID: "snap-1",
changeID: "change-1",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
},
expectedErrStr: "error getting changed blocks from CBT service, fallback to real full: mock changed error",
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "both changed blocks error and allocated blocks error",
snapshotID: "snap-1",
changeID: "change-1",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
},
expectedErrStr: "error getting both changed and allocated blocks from CBT service, fallback to real full: mock alloc error",
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "changed blocks error fallback to full",
snapshotID: "snap-1",
changeID: "change-1",
setupMocks: func(svc *cbtservicemocks.Service) {
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) { svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(2).(func([]cbtservice.Range) error) record := args.Get(2).(func([]cbtservice.Range) error)
record([]cbtservice.Range{ record([]cbtservice.Range{
{Offset: 0, Length: 4096}, {Offset: 0, Length: uint64(mb)},
{Offset: 8192, Length: 4096}, {Offset: uint64(2 * mb), Length: uint64(mb)},
}) })
}).Return(nil) }).Return(nil)
bmp.On("Set", uint64(0), uint64(4096)).Return()
bmp.On("Set", uint64(8192), uint64(4096)).Return()
}, },
}, expectedErrStr: "error getting changed blocks from CBT service, fallback to full: mock changed error",
{ expectedCount: 2,
name: "allocated blocks error", expectedNext: []uint64{0, 2 * mb},
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("")
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting allocated blocks from CBT service: mock alloc error",
},
{
name: "changed blocks success",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(3).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 4096, Length: 4096},
})
}).Return(nil)
bmp.On("Set", uint64(4096), uint64(4096)).Return()
},
},
{
name: "changed blocks error",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting changed blocks from CBT service: mock changed error",
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
svcMock := new(cbtservicemocks.Service) svcMock := new(cbtservicemocks.Service)
bmpMock := new(cbtmocks.Bitmap)
if tt.setupMocks != nil { if tt.setupMocks != nil {
tt.setupMocks(svcMock, bmpMock) tt.setupMocks(svcMock)
} }
var svc cbtservice.Service var svc cbtservice.Service
@@ -124,7 +170,10 @@ func TestSetBitmapOrFull(t *testing.T) {
svc = svcMock svc = svcMock
} }
err := SetBitmapOrFull(context.Background(), svc, bmpMock) bmp := NewBitmap(mb, 3*mb, tt.snapshotID, "vol-1")
bmp.SetChangeID(tt.changeID)
err := SetBitmapOrFull(context.Background(), svc, bmp, tt.incOnly)
if tt.expectedErrStr != "" { if tt.expectedErrStr != "" {
require.Error(t, err) require.Error(t, err)
@@ -136,7 +185,25 @@ func TestSetBitmapOrFull(t *testing.T) {
if !tt.nilService { if !tt.nilService {
svcMock.AssertExpectations(t) svcMock.AssertExpectations(t)
} }
bmpMock.AssertExpectations(t)
iter := bmp.Iterator()
require.NotNil(t, iter)
assert.Equal(t, tt.expectedCount, iter.Count())
var actualOffsets []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
break
}
actualOffsets = append(actualOffsets, offset)
}
if len(tt.expectedNext) > 0 {
assert.Equal(t, tt.expectedNext, actualOffsets)
} else {
assert.Empty(t, actualOffsets)
}
}) })
} }
} }
+27
View File
@@ -292,3 +292,30 @@ func (_c *Bitmap_VolumeID_Call) RunAndReturn(run func() string) *Bitmap_VolumeID
_c.Call.Return(run) _c.Call.Return(run)
return _c return _c
} }
// SetChangeID provides a mock function for the type Bitmap
func (_mock *Bitmap) SetChangeID(id string) {
_mock.Called(id)
}
// SetError provides a mock function for the type Bitmap
func (_mock *Bitmap) SetError(err error) {
_mock.Called(err)
}
// Errors provides a mock function for the type Bitmap
func (_mock *Bitmap) Errors() []error {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Errors")
}
var r0 []error
if returnFunc, ok := ret.Get(0).(func() []error); ok {
r0 = returnFunc()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]error)
}
}
return r0
}
+17
View File
@@ -307,3 +307,20 @@ func (_c *Iterator_VolumeID_Call) RunAndReturn(run func() string) *Iterator_Volu
_c.Call.Return(run) _c.Call.Return(run)
return _c return _c
} }
// Errors provides a mock function for the type Iterator
func (_mock *Iterator) Errors() []error {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Errors")
}
var r0 []error
if returnFunc, ok := ret.Get(0).(func() []error); ok {
r0 = returnFunc()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]error)
}
}
return r0
}
+12
View File
@@ -35,6 +35,15 @@ type Bitmap interface {
// Iterator returns the iterator for the CBT Bitmap // Iterator returns the iterator for the CBT Bitmap
Iterator() Iterator Iterator() Iterator
// SetError sets CBT error when preparing this bitmap
SetError(error)
// Errors returns the CBT errors when preparing this bitmap
Errors() []error
// SetChangeID sets the changeID of the bitmap
SetChangeID(string)
} }
// Iterator defines the methods to iterate the CBT bitmap and query the associated information // Iterator defines the methods to iterate the CBT bitmap and query the associated information
@@ -56,4 +65,7 @@ type Iterator interface {
// Next returns the offset of the next set block and whether it comes to the end of the iteration // Next returns the offset of the next set block and whether it comes to the end of the iteration
Next() (uint64, bool) Next() (uint64, bool)
// Errors returns the CBT errors when preparing this bitmap
Errors() []error
} }
+19 -9
View File
@@ -153,7 +153,8 @@ func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snap
// Backup backup specific sourcePath and update progress // Backup backup specific sourcePath and update progress
func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string,
forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string,
updater uploader.ProgressUpdater, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
if fsUploader == nil { if fsUploader == nil {
return nil, false, errors.New("get empty kopia uploader") return nil, false, errors.New("get empty kopia uploader")
} }
@@ -189,7 +190,7 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re
kopiaCtx := kopia.SetupKopiaLog(ctx, log) kopiaCtx := kopia.SetupKopiaLog(ctx, log)
snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, log, "Kopia Uploader") snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, updater, log, "Kopia Uploader")
snapshotInfo := &uploader.SnapshotInfo{ snapshotInfo := &uploader.SnapshotInfo{
ID: snapID, ID: snapID,
Size: snapshotSize, Size: snapshotSize,
@@ -237,6 +238,7 @@ func SnapshotSource(
parentSnapshot string, parentSnapshot string,
snapshotTags map[string]string, snapshotTags map[string]string,
uploaderCfg map[string]string, uploaderCfg map[string]string,
updater uploader.ProgressUpdater,
log logrus.FieldLogger, log logrus.FieldLogger,
description string, description string,
) (string, int64, error) { ) (string, int64, error) {
@@ -248,21 +250,29 @@ func SnapshotSource(
if parentSnapshot != "" { if parentSnapshot != "" {
log.Infof("Using provided parent snapshot %s", parentSnapshot) log.Infof("Using provided parent snapshot %s", parentSnapshot)
mani, err := loadSnapshotFunc(ctx, rep, manifest.ID(parentSnapshot)) if mani, err := loadSnapshotFunc(ctx, rep, manifest.ID(parentSnapshot)); err != nil {
if err != nil {
log.WithError(err).Warnf("Failed to load previous snapshot %v from kopia, fallback to full backup", parentSnapshot) log.WithError(err).Warnf("Failed to load previous snapshot %v from kopia, fallback to full backup", parentSnapshot)
updater.UpdateProgress(&uploader.Progress{
BytesDone: -1,
TotalBytes: -1,
Message: fmt.Sprintf("Failed to load previous snapshot %v, fallback to full backup. Err: %v", parentSnapshot, err),
})
} else { } else {
previous = append(previous, mani) previous = append(previous, mani)
} }
} else { } else {
log.Infof("Searching for parent snapshot") log.Infof("Searching for parent snapshot")
pre, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, snapshotTags, nil, log) if pre, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, snapshotTags, nil, log); err != nil {
if err != nil { log.WithError(err).Warnf("Failed to find previous kopia snapshot manifests for si %v, fallback to full backup", sourceInfo)
return "", 0, errors.Wrapf(err, "Failed to find previous kopia snapshot manifests for si %v", sourceInfo) updater.UpdateProgress(&uploader.Progress{
BytesDone: -1,
TotalBytes: -1,
Message: fmt.Sprintf("Failed to find previous snapshots, fallback to full backup. Err: %v", err),
})
} else {
previous = pre
} }
previous = pre
} }
} else { } else {
log.Info("Forcing full snapshot") log.Info("Forcing full snapshot")
+3 -3
View File
@@ -200,7 +200,7 @@ func TestSnapshotSource(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
s := injectSnapshotFuncs() s := injectSnapshotFuncs()
MockFuncs(s, tc.args) MockFuncs(s, tc.args)
_, _, err = SnapshotSource(ctx, s.repoWriterMock, s.uploderMock, sourceInfo, rootDir, false, "/", nil, tc.uploaderCfg, log, "TestSnapshotSource") _, _, err = SnapshotSource(ctx, s.repoWriterMock, s.uploderMock, sourceInfo, rootDir, false, "/", nil, tc.uploaderCfg, &fakeProgressUpdater{}, log, "TestSnapshotSource")
if tc.notError { if tc.notError {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {
@@ -648,9 +648,9 @@ func TestBackup(t *testing.T) {
var snapshotInfo *uploader.SnapshotInfo var snapshotInfo *uploader.SnapshotInfo
var err error var err error
if tc.isEmptyUploader { if tc.isEmptyUploader {
snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &logrus.Logger{}) snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &fakeProgressUpdater{}, &logrus.Logger{})
} else { } else {
snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &logrus.Logger{}) snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &fakeProgressUpdater{}, &logrus.Logger{})
} }
// Check if the returned error matches the expected error // Check if the returned error matches the expected error
if tc.expectedError != nil { if tc.expectedError != nil {
+1 -1
View File
@@ -166,7 +166,7 @@ func (kp *kopiaProvider) RunBackup(
uploaderCfg[kopia.UploaderConfigMultipartKey] = "true" uploaderCfg[kopia.UploaderConfigMultipartKey] = "true"
} }
snapshotInfo, _, err := kopiaBackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, log) snapshotInfo, _, err := kopiaBackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, updater, log)
if err != nil { if err != nil {
snapshotID := "" snapshotID := ""
if snapshotInfo != nil { if snapshotInfo != nil {
+4 -4
View File
@@ -65,27 +65,27 @@ func (f *FakeRestoreProgressUpdater) UpdateProgress(p *uploader.Progress) {}
func TestRunBackup(t *testing.T) { func TestRunBackup(t *testing.T) {
testCases := []struct { testCases := []struct {
name string name string
hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, updater uploader.ProgressUpdater, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
volMode uploader.PersistentVolumeMode volMode uploader.PersistentVolumeMode
notError bool notError bool
}{ }{
{ {
name: "success to backup", name: "success to backup",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, updater uploader.ProgressUpdater, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, nil return &uploader.SnapshotInfo{}, false, nil
}, },
notError: true, notError: true,
}, },
{ {
name: "get error to backup", name: "get error to backup",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, updater uploader.ProgressUpdater, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, errors.New("failed to backup") return &uploader.SnapshotInfo{}, false, errors.New("failed to backup")
}, },
notError: false, notError: false,
}, },
{ {
name: "success to backup block mode volume", name: "success to backup block mode volume",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, updater uploader.ProgressUpdater, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, nil return &uploader.SnapshotInfo{}, false, nil
}, },
volMode: uploader.PersistentVolumeBlock, volMode: uploader.PersistentVolumeBlock,
+3 -2
View File
@@ -58,8 +58,9 @@ type SnapshotInfo struct {
// Progress which defined two variables to record progress // Progress which defined two variables to record progress
type Progress struct { type Progress struct {
TotalBytes int64 `json:"totalBytes,omitempty"` TotalBytes int64 `json:"totalBytes,omitempty"`
BytesDone int64 `json:"doneBytes,omitempty"` BytesDone int64 `json:"doneBytes,omitempty"`
Message string `json:"message,omitempty"`
} }
// UploaderProgress which defined generic interface to update progress // UploaderProgress which defined generic interface to update progress