mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-08 06:15:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

batch delete snapshot

Browse Source 
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
Lyndon-Li
2024-01-09 13:32:11 +08:00
parent 72f2da92b7
commit 32d92ca964
12 changed files with 274 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
pkg/controller/backup_deletion_controller.go
View File

@@ -68,6 +68,7 @@ type backupDeletionReconciler struct {
newPluginManager func(logrus.FieldLogger) clientmgmt.Manager
backupStoreGetter persistence.ObjectBackupStoreGetter
credentialStore credentials.FileStore
repoEnsurer *repository.Ensurer
}
// NewBackupDeletionReconciler creates a new backup deletion reconciler.
@@ -81,6 +82,7 @@ func NewBackupDeletionReconciler(
newPluginManager func(logrus.FieldLogger) clientmgmt.Manager,
backupStoreGetter persistence.ObjectBackupStoreGetter,
credentialStore credentials.FileStore,
repoEnsurer *repository.Ensurer,
) *backupDeletionReconciler {
return &backupDeletionReconciler{
Client: client,
@@ -93,6 +95,7 @@ func NewBackupDeletionReconciler(
newPluginManager: newPluginManager,
backupStoreGetter: backupStoreGetter,
credentialStore: credentialStore,
repoEnsurer: repoEnsurer,
}
}
@@ -502,18 +505,12 @@ func (r *backupDeletionReconciler) deletePodVolumeSnapshots(ctx context.Context,
return nil
}
snapshots, err := getSnapshotsInBackup(ctx, backup, r.Client)
directSnapshots, err := getSnapshotsInBackup(ctx, backup, r.Client)
if err != nil {
return []error{err}
}
var errs []error
for _, snapshot := range snapshots {
if err := r.repoMgr.Forget(ctx, snapshot); err != nil {
errs = append(errs, err)
}
}
return errs
return r.batchDeleteSnapshots(ctx, directSnapshots, backup)
}
func (r *backupDeletionReconciler) deleteMovedSnapshots(ctx context.Context, backup *velerov1api.Backup) []error {
@@ -532,6 +529,7 @@ func (r *backupDeletionReconciler) deleteMovedSnapshots(ctx context.Context, bac
return []error{errors.Wrapf(err, "failed to retrieve config for snapshot info")}
}
var errs []error
directSnapshots := map[string][]repository.SnapshotIdentifier{}
for i := range list.Items {
cm := list.Items[i]
snapshot := repository.SnapshotIdentifier{}
@@ -544,15 +542,24 @@ func (r *backupDeletionReconciler) deleteMovedSnapshots(ctx context.Context, bac
errs = append(errs, errors.Wrapf(err, "failed to unmarshal snapshot info"))
continue
}
if err := r.repoMgr.Forget(ctx, snapshot); err != nil {
errs = append(errs, errors.Wrapf(err, "failed to delete snapshot %s, namespace: %s", snapshot.SnapshotID, snapshot.VolumeNamespace))
if directSnapshots[snapshot.VolumeNamespace] == nil {
directSnapshots[snapshot.VolumeNamespace] = []repository.SnapshotIdentifier{}
}
directSnapshots[snapshot.VolumeNamespace] = append(directSnapshots[snapshot.VolumeNamespace], snapshot)
r.logger.Infof("Deleted snapshot %s, namespace: %s, repo type: %s", snapshot.SnapshotID, snapshot.VolumeNamespace, snapshot.RepositoryType)
if err := r.Client.Delete(ctx, &cm); err != nil {
r.logger.Warnf("Failed to delete snapshot info configmap %s/%s: %v", cm.Namespace, cm.Name, err)
}
}
return errs
if len(errs) > 0 {
return errs
}
return r.batchDeleteSnapshots(ctx, directSnapshots, backup)
}
func (r *backupDeletionReconciler) patchDeleteBackupRequest(ctx context.Context, req *velerov1api.DeleteBackupRequest, mutate func(*velerov1api.DeleteBackupRequest)) (*velerov1api.DeleteBackupRequest, error) {
@@ -592,7 +599,7 @@ func (r *backupDeletionReconciler) patchBackup(ctx context.Context, backup *vele
// getSnapshotsInBackup returns a list of all pod volume snapshot ids associated with
// a given Velero backup.
func getSnapshotsInBackup(ctx context.Context, backup *velerov1api.Backup, kbClient client.Client) ([]repository.SnapshotIdentifier, error) {
func getSnapshotsInBackup(ctx context.Context, backup *velerov1api.Backup, kbClient client.Client) (map[string][]repository.SnapshotIdentifier, error) {
podVolumeBackups := &velerov1api.PodVolumeBackupList{}
options := &client.ListOptions{
LabelSelector: labels.Set(map[string]string{
@@ -607,3 +614,29 @@ func getSnapshotsInBackup(ctx context.Context, backup *velerov1api.Backup, kbCli
return podvolume.GetSnapshotIdentifier(podVolumeBackups), nil
}
func (r *backupDeletionReconciler) batchDeleteSnapshots(ctx context.Context, directSnapshots map[string][]repository.SnapshotIdentifier, backup *velerov1api.Backup) []error {
var errs []error
for volumeNamespace, snapshots := range directSnapshots {
batchForget := []string{}
for _, snapshot := range snapshots {
batchForget = append(batchForget, snapshot.SnapshotID)
}
// For volumes in one backup, the BSL and repositoryType should always be the same
repo, err := r.repoEnsurer.EnsureRepo(ctx, backup.Namespace, volumeNamespace, backup.Spec.StorageLocation, snapshots[0].RepositoryType)
if err != nil {
errs = append(errs, errors.Wrapf(err, "error to ensure repo %s-%s-%s, skip deleting PVB snapshots %v", backup.Spec.StorageLocation, volumeNamespace, snapshots[0].RepositoryType, batchForget))
continue
}
if forgetErrs := r.repoMgr.BatchForget(ctx, repo, batchForget); len(forgetErrs) > 0 {
errs = append(errs, forgetErrs...)
continue
}
r.logger.Infof("Batch deleted snapshots %v", batchForget)
}
return errs
}

73
pkg/controller/backup_deletion_controller_test.go
View File

@@ -20,7 +20,7 @@ import (
"bytes"
"fmt"
"io"
"sort"
"reflect"
"time"
"context"
@@ -95,6 +95,7 @@ func setupBackupDeletionControllerTest(t *testing.T, req *velerov1api.DeleteBack
func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
NewFakeSingleObjectBackupStoreGetter(backupStore),
velerotest.NewFakeCredentialsFileStore("", nil),
nil,
),
req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: req.Namespace, Name: req.Name}},
}
@@ -695,13 +696,13 @@ func TestGetSnapshotsInBackup(t *testing.T) {
tests := []struct {
name string
podVolumeBackups []velerov1api.PodVolumeBackup
expected []repository.SnapshotIdentifier
expected map[string][]repository.SnapshotIdentifier
longBackupNameEnabled bool
}{
{
name: "no pod volume backups",
podVolumeBackups: nil,
expected: nil,
expected: map[string][]repository.SnapshotIdentifier{},
},
{
name: "no pod volume backups with matching label",
@@ -721,7 +722,7 @@ func TestGetSnapshotsInBackup(t *testing.T) {
Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-2"},
},
},
expected: nil,
expected: map[string][]repository.SnapshotIdentifier{},
},
{
name: "some pod volume backups with matching label",
@@ -762,16 +763,18 @@ func TestGetSnapshotsInBackup(t *testing.T) {
Status: velerov1api.PodVolumeBackupStatus{SnapshotID: ""},
},
},
expected: []repository.SnapshotIdentifier{
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-3",
RepositoryType: "restic",
},
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-4",
RepositoryType: "restic",
expected: map[string][]repository.SnapshotIdentifier{
"ns-1": {
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-3",
RepositoryType: "restic",
},
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-4",
RepositoryType: "restic",
},
},
},
},
@@ -815,11 +818,13 @@ func TestGetSnapshotsInBackup(t *testing.T) {
Status: velerov1api.PodVolumeBackupStatus{SnapshotID: ""},
},
},
expected: []repository.SnapshotIdentifier{
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-3",
RepositoryType: "restic",
expected: map[string][]repository.SnapshotIdentifier{
"ns-1": {
{
VolumeNamespace: "ns-1",
SnapshotID: "snap-3",
RepositoryType: "restic",
},
},
},
},
@@ -844,21 +849,27 @@ func TestGetSnapshotsInBackup(t *testing.T) {
res, err := getSnapshotsInBackup(context.TODO(), veleroBackup, clientBuilder.Build())
assert.NoError(t, err)
// sort to ensure good compare of slices
less := func(snapshots []repository.SnapshotIdentifier) func(i, j int) bool {
return func(i, j int) bool {
if snapshots[i].VolumeNamespace == snapshots[j].VolumeNamespace {
return snapshots[i].SnapshotID < snapshots[j].SnapshotID
}
return snapshots[i].VolumeNamespace < snapshots[j].VolumeNamespace
}
assert.True(t, reflect.DeepEqual(res, test.expected))
}
// for k, v := range res {
sort.Slice(test.expected, less(test.expected))
sort.Slice(res, less(res))
// }
assert.Equal(t, test.expected, res)
// // sort to ensure good compare of slices
// less := func(snapshots []repository.SnapshotIdentifier) func(i, j int) bool {
// return func(i, j int) bool {
// if snapshots[i].VolumeNamespace == snapshots[j].VolumeNamespace {
// return snapshots[i].SnapshotID < snapshots[j].SnapshotID
// }
// return snapshots[i].VolumeNamespace < snapshots[j].VolumeNamespace
// }
// }
// sort.Slice(test.expected, less(test.expected))
// sort.Slice(res, less(res))
// assert.Equal(t, test.expected, res)
})
}
}

2
pkg/controller/data_upload_controller.go
View File

@@ -344,7 +344,7 @@ func (r *DataUploadReconciler) runCancelableDataUpload(ctx context.Context, fsBa
velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel],
}
if err := fsBackup.StartBackup(path, fmt.Sprintf("%s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags, du.Spec.DataMoverConfig); err != nil {
if err := fsBackup.StartBackup(path, datamover.GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags, du.Spec.DataMoverConfig); err != nil {
return r.errorOut(ctx, du, err, "error starting data path backup", log)
}