mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-06 16:17:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #7630 from reasonerjt/restore-vol-info

Browse Source 
Track and persist restore volume info
This commit is contained in:
Wenkai Yin(尹文开)
2024-04-12 11:24:05 +08:00
committed by GitHub
parent ab57112347 0a280e5786
commit 3c377bc3ec
33 changed files with 1116 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/restore_controller.go
+37 -11
View File
@@ -107,6 +107,7 @@ type restoreReconciler struct {
newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager
backupStoreGetter persistence.ObjectBackupStoreGetter
globalCrClient client.Client
}
type backupInfo struct {
@@ -127,6 +128,7 @@ func NewRestoreReconciler(
logFormat logging.Format,
defaultItemOperationTimeout time.Duration,
disableInformerCache bool,
globalCrClient client.Client,
) *restoreReconciler {
r := &restoreReconciler{
ctx: ctx,
@@ -145,6 +147,8 @@ func NewRestoreReconciler(
// replaced with fakes for testing.
newPluginManager: newPluginManager,
backupStoreGetter: backupStoreGetter,
globalCrClient: globalCrClient,
}
// Move the periodical backup and restore metrics computing logic from controllers to here.
@@ -521,7 +525,7 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
return errors.Wrap(err, "fail to fetch CSI VolumeSnapshots metadata")
}
backupVolumeInfoMap := make(map[string]volume.VolumeInfo)
backupVolumeInfoMap := make(map[string]volume.BackupVolumeInfo)
volumeInfos, err := backupStore.GetBackupVolumeInfos(restore.Spec.BackupName)
if err != nil {
restoreLog.WithError(err).Errorf("fail to get VolumeInfos metadata file for backup %s", restore.Spec.BackupName)
@@ -540,16 +544,17 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
}
restoreReq := &pkgrestore.Request{
Log: restoreLog,
Restore: restore,
Backup: info.backup,
PodVolumeBackups: podVolumeBackups,
VolumeSnapshots: volumeSnapshots,
BackupReader: backupFile,
ResourceModifiers: resourceModifiers,
DisableInformerCache: r.disableInformerCache,
CSIVolumeSnapshots: csiVolumeSnapshots,
VolumeInfoMap: backupVolumeInfoMap,
Log: restoreLog,
Restore: restore,
Backup: info.backup,
PodVolumeBackups: podVolumeBackups,
VolumeSnapshots: volumeSnapshots,
BackupReader: backupFile,
ResourceModifiers: resourceModifiers,
DisableInformerCache: r.disableInformerCache,
CSIVolumeSnapshots: csiVolumeSnapshots,
BackupVolumeInfoMap: backupVolumeInfoMap,
RestoreVolumeInfoTracker: volume.NewRestoreVolInfoTracker(restore, restoreLog, r.globalCrClient),
}
restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, pluginManager)
@@ -640,6 +645,11 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
r.logger.WithError(err).Error("Error uploading restore item action operation resource list to backup storage")
}
restoreReq.RestoreVolumeInfoTracker.Populate(context.TODO(), restoreReq.RestoredResourceList())
if err := putRestoreVolumeInfoList(restore, restoreReq.RestoreVolumeInfoTracker.Result(), backupStore); err != nil {
r.logger.WithError(err).Error("Error uploading restored volume info to backup storage")
}
if restore.Status.Errors > 0 {
if inProgressOperations {
r.logger.Debug("Restore WaitingForPluginOperationsPartiallyFailed")
@@ -776,6 +786,22 @@ func putOperationsForRestore(restore *api.Restore, operations []*itemoperation.R
return nil
}
func putRestoreVolumeInfoList(restore *api.Restore, volInfoList []*volume.RestoreVolumeInfo, store persistence.BackupStore) error {
buf := new(bytes.Buffer)
gzw := gzip.NewWriter(buf)
defer gzw.Close()
if err := json.NewEncoder(gzw).Encode(volInfoList); err != nil {
return errors.Wrap(err, "error encoding restore volume info list to JSON")
}
if err := gzw.Close(); err != nil {
return errors.Wrap(err, "error closing gzip writer")
}
return store.PutRestoreVolumeInfo(restore.Name, buf)
}
func downloadToTempFile(backupName string, backupStore persistence.BackupStore, logger logrus.FieldLogger) (*os.File, error) {
readCloser, err := backupStore.GetBackupContents(backupName)
if err != nil {
pkg/controller/restore_controller_test.go
+33 -22
View File
@@ -91,11 +91,12 @@ func TestFetchBackupInfo(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
backupStore = &persistencemocks.BackupStore{}
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
backupStore = &persistencemocks.BackupStore{}
)
defer restorer.AssertExpectations(t)
@@ -114,6 +115,7 @@ func TestFetchBackupInfo(t *testing.T) {
formatFlag,
60*time.Minute,
false,
fakeGlobalClient,
)
if test.backupStoreError == nil {
@@ -170,9 +172,10 @@ func TestProcessQueueItemSkips(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
)
if test.restore != nil {
@@ -192,6 +195,7 @@ func TestProcessQueueItemSkips(t *testing.T) {
formatFlag,
60*time.Minute,
false,
fakeGlobalClient,
)
_, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{
@@ -432,11 +436,12 @@ func TestRestoreReconcile(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build()
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
backupStore = &persistencemocks.BackupStore{}
fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build()
fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
restorer = &fakeRestorer{kbClient: fakeClient}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
backupStore = &persistencemocks.BackupStore{}
)
defer restorer.AssertExpectations(t)
@@ -459,6 +464,7 @@ func TestRestoreReconcile(t *testing.T) {
formatFlag,
60*time.Minute,
false,
fakeGlobalClient,
)
r.clock = clocktesting.NewFakeClock(now)
@@ -500,10 +506,11 @@ func TestRestoreReconcile(t *testing.T) {
backupStore.On("PutRestoreResults", test.backup.Name, test.restore.Name, mock.Anything).Return(nil)
backupStore.On("PutRestoredResourceList", test.restore.Name, mock.Anything).Return(nil)
backupStore.On("PutRestoreItemOperations", mock.Anything, mock.Anything).Return(nil)
backupStore.On("PutRestoreVolumeInfo", test.restore.Name, mock.Anything).Return(nil)
if test.emptyVolumeInfo == true {
backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return(nil, nil)
} else {
backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return([]*volume.VolumeInfo{}, nil)
backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return([]*volume.BackupVolumeInfo{}, nil)
}
volumeSnapshots := []*volume.Snapshot{
@@ -626,10 +633,11 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) {
formatFlag := logging.FormatText
var (
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
backupStore = &persistencemocks.BackupStore{}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
backupStore = &persistencemocks.BackupStore{}
)
r := NewRestoreReconciler(
@@ -645,6 +653,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) {
formatFlag,
60*time.Minute,
false,
fakeGlobalClient,
)
restore := &velerov1api.Restore{
@@ -719,10 +728,11 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) {
formatFlag := logging.FormatText
var (
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
backupStore = &persistencemocks.BackupStore{}
logger = velerotest.NewLogger()
pluginManager = &pluginmocks.Manager{}
fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
backupStore = &persistencemocks.BackupStore{}
)
r := NewRestoreReconciler(
@@ -738,6 +748,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) {
formatFlag,
60*time.Minute,
false,
fakeGlobalClient,
)
restore := &velerov1api.Restore{
pkg/controller/restore_finalizer_controller.go
+3 -22
View File
@@ -19,7 +19,6 @@ package controller
import (
"context"
"fmt"
"regexp"
"sync"
"time"
@@ -39,7 +38,6 @@ import (
"github.com/vmware-tanzu/velero/pkg/metrics"
"github.com/vmware-tanzu/velero/pkg/persistence"
"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
"github.com/vmware-tanzu/velero/pkg/restore"
kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/results"
)
@@ -150,7 +148,7 @@ func (r *restoreFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, errors.Wrap(err, "error getting restoredResourceList")
}
restoredPVCList := getRestoredPVCFromRestoredResourceList(restoredResourceList)
restoredPVCList := volume.RestoredPVCFromRestoredResourceList(restoredResourceList)
finalizerCtx := &finalizerContext{
logger: log,
@@ -238,7 +236,7 @@ type finalizerContext struct {
logger logrus.FieldLogger
restore *velerov1api.Restore
crClient client.Client
volumeInfo []*volume.VolumeInfo
volumeInfo []*volume.BackupVolumeInfo
restoredPVCList map[string]struct{}
}
@@ -277,7 +275,7 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result
}
pvWaitGroup.Add(1)
go func(volInfo volume.VolumeInfo, restoredNamespace string) {
go func(volInfo volume.BackupVolumeInfo, restoredNamespace string) {
defer pvWaitGroup.Done()
semaphore <- struct{}{}
@@ -358,23 +356,6 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result
return errs
}
func getRestoredPVCFromRestoredResourceList(restoredResourceList map[string][]string) map[string]struct{} {
pvcKey := "v1/PersistentVolumeClaim"
pvcList := make(map[string]struct{})
for _, pvc := range restoredResourceList[pvcKey] {
// the format of pvc string in restoredResourceList is like: "namespace/pvcName(status)"
// extract the substring before "(created)" if the status in rightmost Parenthesis is "created"
r := regexp.MustCompile(`\(([^)]+)\)`)
matches := r.FindAllStringSubmatch(pvc, -1)
if len(matches) > 0 && matches[len(matches)-1][1] == restore.ItemRestoreResultCreated {
pvcList[pvc[:len(pvc)-len("(created)")]] = struct{}{}
}
}
return pvcList
}
func needPatch(newPV *v1.PersistentVolume, pvInfo *volume.PVInfo) bool {
if newPV.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimPolicy(pvInfo.ReclaimPolicy) {
return true
pkg/controller/restore_finalizer_controller_test.go
+9 -43
View File
@@ -210,7 +210,7 @@ func TestUpdateResult(t *testing.T) {
func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
tests := []struct {
name string
volumeInfo []*volume.VolumeInfo
volumeInfo []*volume.BackupVolumeInfo
restoredPVCNames map[string]struct{}
restore *velerov1api.Restore
restoredPVC []*corev1api.PersistentVolumeClaim
@@ -220,21 +220,21 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
}{
{
name: "no applicable volumeInfo",
volumeInfo: []*volume.VolumeInfo{{BackupMethod: "VeleroNativeSnapshot", PVCName: "pvc1"}},
volumeInfo: []*volume.BackupVolumeInfo{{BackupMethod: "VeleroNativeSnapshot", PVCName: "pvc1"}},
restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(),
expectedPatch: nil,
expectedErrNum: 0,
},
{
name: "no restored PVC",
volumeInfo: []*volume.VolumeInfo{{BackupMethod: "PodVolumeBackup", PVCName: "pvc1"}},
volumeInfo: []*volume.BackupVolumeInfo{{BackupMethod: "PodVolumeBackup", PVCName: "pvc1"}},
restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(),
expectedPatch: nil,
expectedErrNum: 0,
},
{
name: "no applicable pv patch",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -256,7 +256,7 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
},
{
name: "an applicable pv patch",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -281,7 +281,7 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
},
{
name: "a mapped namespace restore",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -306,7 +306,7 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
},
{
name: "two applicable pv patches",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -354,7 +354,7 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
},
{
name: "an applicable pv patch with bound error",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -375,7 +375,7 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
},
{
name: "two applicable pv patches with an error",
volumeInfo: []*volume.VolumeInfo{{
volumeInfo: []*volume.BackupVolumeInfo{{
BackupMethod: "PodVolumeBackup",
PVCName: "pvc1",
PVName: "pv1",
@@ -454,37 +454,3 @@ func TestPatchDynamicPVWithVolumeInfo(t *testing.T) {
}
}
}
func TestGetRestoredPVCFromRestoredResourceList(t *testing.T) {
// test empty list
restoredResourceList := map[string][]string{}
actual := getRestoredPVCFromRestoredResourceList(restoredResourceList)
assert.Empty(t, actual)
// test no match
restoredResourceList = map[string][]string{
"v1/PersistentVolumeClaim": {
"namespace1/pvc1(updated)",
},
"v1/PersistentVolume": {
"namespace1/pv(created)",
},
}
actual = getRestoredPVCFromRestoredResourceList(restoredResourceList)
assert.Empty(t, actual)
// test matches
restoredResourceList = map[string][]string{
"v1/PersistentVolumeClaim": {
"namespace1/pvc1(created)",
"namespace2/pvc2(updated)",
"namespace3/pvc(3)(created)",
},
}
expected := map[string]struct{}{
"namespace1/pvc1": {},
"namespace3/pvc(3)": {},
}
actual = getRestoredPVCFromRestoredResourceList(restoredResourceList)
assert.Equal(t, expected, actual)
}