mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-04 15:16:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

repo maintenance job out of repo manager

Browse Source 
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
Lyndon-Li
2025-01-06 16:25:33 +08:00
parent 912b116bdb
commit db69829fd7
9 changed files with 781 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/backup_repository_controller.go
+56 -21
View File
@@ -45,6 +45,7 @@ import (
repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config"
repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
)
const (
@@ -55,16 +56,22 @@ const (
type BackupRepoReconciler struct {
client.Client
namespace string
logger logrus.FieldLogger
clock clocks.WithTickerAndDelayedExecution
maintenanceFrequency time.Duration
backupRepoConfig string
repositoryManager repomanager.Manager
namespace string
logger logrus.FieldLogger
clock clocks.WithTickerAndDelayedExecution
maintenanceFrequency time.Duration
backupRepoConfig string
repositoryManager repomanager.Manager
keepLatestMaintenanceJobs int
repoMaintenanceConfig string
podResources kube.PodResources
logLevel logrus.Level
logFormat *logging.FormatFlag
}
func NewBackupRepoReconciler(namespace string, logger logrus.FieldLogger, client client.Client,
maintenanceFrequency time.Duration, backupRepoConfig string, repositoryManager repomanager.Manager) *BackupRepoReconciler {
func NewBackupRepoReconciler(namespace string, logger logrus.FieldLogger, client client.Client, repositoryManager repomanager.Manager,
maintenanceFrequency time.Duration, backupRepoConfig string, keepLatestMaintenanceJobs int, repoMaintenanceConfig string, podResources kube.PodResources,
logLevel logrus.Level, logFormat *logging.FormatFlag) *BackupRepoReconciler {
c := &BackupRepoReconciler{
client,
namespace,
@@ -73,6 +80,11 @@ func NewBackupRepoReconciler(namespace string, logger logrus.FieldLogger, client
maintenanceFrequency,
backupRepoConfig,
repositoryManager,
keepLatestMaintenanceJobs,
repoMaintenanceConfig,
podResources,
logLevel,
logFormat,
}
return c
@@ -212,7 +224,13 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, errors.Wrap(err, "error handling incomplete repo maintenance jobs")
}
return ctrl.Result{}, r.runMaintenanceIfDue(ctx, backupRepo, log)
if err := r.runMaintenanceIfDue(ctx, backupRepo, log); err != nil {
return ctrl.Result{}, errors.Wrap(err, "error check and run repo maintenance jobs")
}
if err := repository.DeleteOldMaintenanceJobs(r.Client, req.Name, r.keepLatestMaintenanceJobs); err != nil {
log.WithError(err).Warn("Failed to delete old maintenance jobs")
}
}
return ctrl.Result{}, nil
@@ -306,7 +324,7 @@ func ensureRepo(repo *velerov1api.BackupRepository, repoManager repomanager.Mana
}
func (r *BackupRepoReconciler) recallMaintenance(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error {
history, err := repository.WaitIncompleteMaintenance(ctx, r.Client, req, defaultMaintenanceStatusQueueLength, log)
history, err := repository.WaitAllMaintenanceJobComplete(ctx, r.Client, req, defaultMaintenanceStatusQueueLength, log)
if err != nil {
return errors.Wrapf(err, "error waiting incomplete repo maintenance job for repo %s", req.Name)
}
@@ -380,7 +398,11 @@ func getLastMaintenanceTimeFromHistory(history []velerov1api.BackupRepositoryMai
time := history[0].CompleteTimestamp
for i := range history {
if time.Before(history[i].CompleteTimestamp) {
if history[i].CompleteTimestamp == nil {
continue
}
if time == nil || time.Before(history[i].CompleteTimestamp) {
time = history[i].CompleteTimestamp
}
}
@@ -406,8 +428,13 @@ func isEarlierMaintenanceStatus(a, b velerov1api.BackupRepositoryMaintenanceStat
return a.StartTimestamp.Before(b.StartTimestamp)
}
var funcStartMaintenanceJob = repository.StartMaintenanceJob
var funcWaitMaintenanceJobComplete = repository.WaitMaintenanceJobComplete
func (r *BackupRepoReconciler) runMaintenanceIfDue(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error {
if !dueForMaintenance(req, r.clock.Now()) {
startTime := r.clock.Now()
if !dueForMaintenance(req, startTime) {
log.Debug("not due for maintenance")
return nil
}
@@ -418,31 +445,39 @@ func (r *BackupRepoReconciler) runMaintenanceIfDue(ctx context.Context, req *vel
// should not cause the repo to move to `NotReady`.
log.Debug("Pruning repo")
// when PruneRepo fails, the maintenance result will be left temporarily
// If the maintenenance still completes later, recallMaintenance recalls the left onces and update LastMaintenanceTime and history
status, err := r.repositoryManager.PruneRepo(req)
job, err := funcStartMaintenanceJob(r.Client, ctx, req, r.repoMaintenanceConfig, r.podResources, r.logLevel, r.logFormat, log)
if err != nil {
return errors.Wrapf(err, "error pruning repository")
log.WithError(err).Warn("Starting repo maintenance failed")
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, &metav1.Time{Time: startTime}, nil, fmt.Sprintf("Failed to start maintenance job, err: %v", err))
})
}
// when WaitMaintenanceJobComplete fails, the maintenance result will be left temporarily
// If the maintenenance still completes later, recallMaintenance recalls the left onces and update LastMaintenanceTime and history
status, err := funcWaitMaintenanceJobComplete(r.Client, ctx, job, r.namespace, log)
if err != nil {
return errors.Wrapf(err, "error waiting repo maintenance completion status")
}
if status.Result == velerov1api.BackupRepositoryMaintenanceFailed {
log.WithError(err).Warn("Pruning repository failed")
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, status.StartTimestamp.Time, status.CompleteTimestamp.Time, status.Message)
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, status.StartTimestamp, status.CompleteTimestamp, status.Message)
})
}
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
rr.Status.LastMaintenanceTime = &metav1.Time{Time: status.CompleteTimestamp.Time}
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceSucceeded, status.StartTimestamp.Time, status.CompleteTimestamp.Time, status.Message)
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceSucceeded, status.StartTimestamp, status.CompleteTimestamp, status.Message)
})
}
func updateRepoMaintenanceHistory(repo *velerov1api.BackupRepository, result velerov1api.BackupRepositoryMaintenanceResult, startTime time.Time, completionTime time.Time, message string) {
func updateRepoMaintenanceHistory(repo *velerov1api.BackupRepository, result velerov1api.BackupRepositoryMaintenanceResult, startTime, completionTime *metav1.Time, message string) {
latest := velerov1api.BackupRepositoryMaintenanceStatus{
Result: result,
StartTimestamp: &metav1.Time{Time: startTime},
CompleteTimestamp: &metav1.Time{Time: completionTime},
StartTimestamp: startTime,
CompleteTimestamp: completionTime,
Message: message,
}
pkg/controller/backup_repository_controller_test.go
+194 -14
View File
@@ -19,20 +19,30 @@ import (
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/repository"
repomokes "github.com/vmware-tanzu/velero/pkg/repository/mocks"
repotypes "github.com/vmware-tanzu/velero/pkg/repository/types"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/util/kube"
"github.com/vmware-tanzu/velero/pkg/util/logging"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
batchv1 "k8s.io/api/batch/v1"
)
const testMaintenanceFrequency = 10 * time.Minute
@@ -47,9 +57,14 @@ func mockBackupRepoReconciler(t *testing.T, mockOn string, arg interface{}, ret
velerov1api.DefaultNamespace,
velerotest.NewLogger(),
velerotest.NewFakeControllerRuntimeClient(t),
mgr,
testMaintenanceFrequency,
"fake-repo-config",
mgr,
3,
"",
kube.PodResources{},
logrus.InfoLevel,
nil,
)
}
@@ -104,24 +119,71 @@ func TestCheckNotReadyRepo(t *testing.T) {
assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier)
}
func startMaintenanceJobFail(client.Client, context.Context, *velerov1api.BackupRepository, string, kube.PodResources, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) {
return "", errors.New("fake-start-error")
}
func startMaintenanceJobSucceed(client.Client, context.Context, *velerov1api.BackupRepository, string, kube.PodResources, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) {
return "fake-job-name", nil
}
func waitMaintenanceJobCompleteFail(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) {
return velerov1api.BackupRepositoryMaintenanceStatus{}, errors.New("fake-wait-error")
}
func waitMaintenanceJobCompleteSucceed(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) {
return velerov1api.BackupRepositoryMaintenanceStatus{
StartTimestamp: &metav1.Time{Time: time.Now()},
CompleteTimestamp: &metav1.Time{Time: time.Now().Add(time.Hour)},
}, nil
}
func TestRunMaintenanceIfDue(t *testing.T) {
rr := mockBackupRepositoryCR()
reconciler := mockBackupRepoReconciler(t, "PruneRepo", rr, velerov1api.BackupRepositoryMaintenanceStatus{
StartTimestamp: &metav1.Time{},
CompleteTimestamp: &metav1.Time{},
}, nil)
reconciler := mockBackupRepoReconciler(t, "", rr, nil)
funcStartMaintenanceJob = startMaintenanceJobFail
err := reconciler.Client.Create(context.TODO(), rr)
assert.NoError(t, err)
lastTm := rr.Status.LastMaintenanceTime
err = reconciler.runMaintenanceIfDue(context.TODO(), rr, reconciler.logger)
assert.NoError(t, err)
assert.NotEqual(t, rr.Status.LastMaintenanceTime, lastTm)
rr.Status.LastMaintenanceTime = &metav1.Time{Time: time.Now()}
lastTm = rr.Status.LastMaintenanceTime
history := rr.Status.RecentMaintenance
err = reconciler.runMaintenanceIfDue(context.TODO(), rr, reconciler.logger)
assert.NoError(t, err)
assert.Equal(t, rr.Status.LastMaintenanceTime, lastTm)
assert.NotEqual(t, rr.Status.RecentMaintenance, history)
rr = mockBackupRepositoryCR()
reconciler = mockBackupRepoReconciler(t, "", rr, nil)
funcStartMaintenanceJob = startMaintenanceJobSucceed
funcWaitMaintenanceJobComplete = waitMaintenanceJobCompleteFail
err = reconciler.Client.Create(context.TODO(), rr)
assert.NoError(t, err)
lastTm = rr.Status.LastMaintenanceTime
history = rr.Status.RecentMaintenance
err = reconciler.runMaintenanceIfDue(context.TODO(), rr, reconciler.logger)
assert.EqualError(t, err, "error waiting repo maintenance completion status: fake-wait-error")
assert.Equal(t, rr.Status.LastMaintenanceTime, lastTm)
assert.Equal(t, rr.Status.RecentMaintenance, history)
rr = mockBackupRepositoryCR()
reconciler = mockBackupRepoReconciler(t, "", rr, nil)
funcStartMaintenanceJob = startMaintenanceJobSucceed
funcWaitMaintenanceJobComplete = waitMaintenanceJobCompleteSucceed
err = reconciler.Client.Create(context.TODO(), rr)
assert.NoError(t, err)
lastTm = rr.Status.LastMaintenanceTime
history = rr.Status.RecentMaintenance
err = reconciler.runMaintenanceIfDue(context.TODO(), rr, reconciler.logger)
assert.NoError(t, err)
assert.NotEqual(t, rr.Status.LastMaintenanceTime, lastTm)
assert.NotEqual(t, rr.Status.RecentMaintenance, history)
rr.Status.LastMaintenanceTime = &metav1.Time{Time: time.Now()}
lastTm = rr.Status.LastMaintenanceTime
history = rr.Status.RecentMaintenance
err = reconciler.runMaintenanceIfDue(context.TODO(), rr, reconciler.logger)
assert.NoError(t, err)
assert.Equal(t, rr.Status.LastMaintenanceTime, lastTm)
assert.Equal(t, rr.Status.RecentMaintenance, history)
}
func TestInitializeRepo(t *testing.T) {
@@ -251,9 +313,14 @@ func TestGetRepositoryMaintenanceFrequency(t *testing.T) {
velerov1api.DefaultNamespace,
velerotest.NewLogger(),
velerotest.NewFakeControllerRuntimeClient(t),
&mgr,
test.userDefinedFreq,
"",
&mgr,
3,
"",
kube.PodResources{},
logrus.InfoLevel,
nil,
)
freq := reconciler.getRepositoryMaintenanceFrequency(test.repo)
@@ -380,7 +447,14 @@ func TestNeedInvalidBackupRepo(t *testing.T) {
velerov1api.DefaultNamespace,
velerotest.NewLogger(),
velerotest.NewFakeControllerRuntimeClient(t),
time.Duration(0), "", nil)
nil,
time.Duration(0),
"",
3,
"",
kube.PodResources{},
logrus.InfoLevel,
nil)
need := reconciler.needInvalidBackupRepo(test.oldBSL, test.newBSL)
assert.Equal(t, test.expect, need)
@@ -656,7 +730,7 @@ func TestUpdateRepoMaintenanceHistory(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
updateRepoMaintenanceHistory(test.backupRepo, test.result, standardTime, standardTime.Add(time.Hour), "fake-message-0")
updateRepoMaintenanceHistory(test.backupRepo, test.result, &metav1.Time{Time: standardTime}, &metav1.Time{Time: standardTime.Add(time.Hour)}, "fake-message-0")
for at := range test.backupRepo.Status.RecentMaintenance {
assert.Equal(t, test.expectedHistory[at].StartTimestamp.Time, test.backupRepo.Status.RecentMaintenance[at].StartTimestamp.Time)
@@ -666,3 +740,109 @@ func TestUpdateRepoMaintenanceHistory(t *testing.T) {
})
}
}
func TestRecallMaintenance(t *testing.T) {
now := time.Now()
schemeFail := runtime.NewScheme()
velerov1api.AddToScheme(schemeFail)
scheme := runtime.NewScheme()
batchv1.AddToScheme(scheme)
corev1.AddToScheme(scheme)
velerov1api.AddToScheme(scheme)
jobSucceeded := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "job1",
Namespace: velerov1api.DefaultNamespace,
Labels: map[string]string{repository.RepositoryNameLabel: "repo"},
CreationTimestamp: metav1.Time{Time: now.Add(time.Hour)},
},
Status: batchv1.JobStatus{
StartTime: &metav1.Time{Time: now.Add(time.Hour)},
CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 2)},
Succeeded: 1,
},
}
jobPodSucceeded := builder.ForPod(velerov1api.DefaultNamespace, "job1").Labels(map[string]string{"job-name": "job1"}).ContainerStatuses(&v1.ContainerStatus{
State: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{},
},
}).Result()
tests := []struct {
name string
kubeClientObj []runtime.Object
runtimeScheme *runtime.Scheme
repoLastMatainTime metav1.Time
expectNewHistory bool
expectTimeUpdate bool
expectedErr string
}{
{
name: "wait completion error",
runtimeScheme: schemeFail,
expectedErr: "error waiting incomplete repo maintenance job for repo repo: error listing maintenance job for repo repo: no kind is registered for the type v1.JobList in scheme \"pkg/runtime/scheme.go:100\"",
},
{
name: "no consolidate result",
runtimeScheme: scheme,
},
{
name: "no update last time",
runtimeScheme: scheme,
kubeClientObj: []runtime.Object{
jobSucceeded,
jobPodSucceeded,
},
repoLastMatainTime: metav1.Time{Time: now.Add(time.Hour * 5)},
expectNewHistory: true,
},
{
name: "update last time",
runtimeScheme: scheme,
kubeClientObj: []runtime.Object{
jobSucceeded,
jobPodSucceeded,
},
expectNewHistory: true,
expectTimeUpdate: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
r := mockBackupRepoReconciler(t, "", nil, nil)
backupRepo := mockBackupRepositoryCR()
backupRepo.Status.LastMaintenanceTime = &test.repoLastMatainTime
test.kubeClientObj = append(test.kubeClientObj, backupRepo)
fakeClientBuilder := fake.NewClientBuilder()
fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme)
fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
r.Client = fakeClient
lastTm := backupRepo.Status.LastMaintenanceTime
history := backupRepo.Status.RecentMaintenance
err := r.recallMaintenance(context.TODO(), backupRepo, velerotest.NewLogger())
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
if test.expectNewHistory {
assert.NotEqual(t, history, backupRepo.Status.RecentMaintenance)
}
if test.expectTimeUpdate {
assert.NotEqual(t, lastTm, backupRepo.Status.LastMaintenanceTime)
}
}
})
}
}