Merge pull request #8532 from Lyndon-Li/isolate-message-in-backup-repo
Run the E2E test on kind / build (push) Failing after 4m51s
Run the E2E test on kind / setup-test-matrix (push) Successful in 12s
Run the E2E test on kind / run-e2e-test (push) Has been skipped
Main CI / Build (push) Failing after 40s
Close stale issues and PRs / stale (push) Successful in 7s
Trivy Nightly Scan / Trivy nightly scan (velero, main) (push) Failing after 1m8s
Trivy Nightly Scan / Trivy nightly scan (velero-restore-helper, main) (push) Failing after 1m2s

Add maintenance history for backupRepository CRs
This commit is contained in:
Daniel Jiang
2024-12-23 19:29:52 +08:00
committed by GitHub
7 changed files with 297 additions and 11 deletions
+25 -7
View File
@@ -46,8 +46,9 @@ import (
)
const (
repoSyncPeriod = 5 * time.Minute
defaultMaintainFrequency = 7 * 24 * time.Hour
repoSyncPeriod = 5 * time.Minute
defaultMaintainFrequency = 7 * 24 * time.Hour
defaultMaintenanceStatusQueueLength = 3
)
type BackupRepoReconciler struct {
@@ -299,9 +300,9 @@ func ensureRepo(repo *velerov1api.BackupRepository, repoManager repomanager.Mana
}
func (r *BackupRepoReconciler) runMaintenanceIfDue(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error {
now := r.clock.Now()
startTime := r.clock.Now()
if !dueForMaintenance(req, now) {
if !dueForMaintenance(req, startTime) {
log.Debug("not due for maintenance")
return nil
}
@@ -315,16 +316,33 @@ func (r *BackupRepoReconciler) runMaintenanceIfDue(ctx context.Context, req *vel
if err := r.repositoryManager.PruneRepo(req); err != nil {
log.WithError(err).Warn("error pruning repository")
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
rr.Status.Message = err.Error()
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, startTime, r.clock.Now(), err.Error())
})
}
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
rr.Status.Message = ""
rr.Status.LastMaintenanceTime = &metav1.Time{Time: now}
completionTime := r.clock.Now()
rr.Status.LastMaintenanceTime = &metav1.Time{Time: completionTime}
updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceSucceeded, startTime, completionTime, "")
})
}
func updateRepoMaintenanceHistory(repo *velerov1api.BackupRepository, result velerov1api.BackupRepositoryMaintenanceResult, startTime time.Time, completionTime time.Time, message string) {
latest := velerov1api.BackupRepositoryMaintenanceStatus{
Result: result,
StartTimestamp: &metav1.Time{Time: startTime},
CompleteTimestamp: &metav1.Time{Time: completionTime},
Message: message,
}
startingPos := 0
if len(repo.Status.RecentMaintenance) >= defaultMaintenanceStatusQueueLength {
startingPos = len(repo.Status.RecentMaintenance) - defaultMaintenanceStatusQueueLength + 1
}
repo.Status.RecentMaintenance = append(repo.Status.RecentMaintenance[startingPos:], latest)
}
func dueForMaintenance(req *velerov1api.BackupRepository, now time.Time) bool {
return req.Status.LastMaintenanceTime == nil || req.Status.LastMaintenanceTime.Add(req.Spec.MaintenanceFrequency.Duration).Before(now)
}
@@ -486,3 +486,180 @@ func TestGetBackupRepositoryConfig(t *testing.T) {
})
}
}
func TestUpdateRepoMaintenanceHistory(t *testing.T) {
standardTime := time.Now()
backupRepoWithoutHistory := &velerov1api.BackupRepository{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "repo",
},
}
backupRepoWithHistory := &velerov1api.BackupRepository{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "repo",
},
Status: velerov1api.BackupRepositoryStatus{
RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)},
Message: "fake-history-message-1",
},
},
},
}
backupRepoWithFullHistory := &velerov1api.BackupRepository{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "repo",
},
Status: velerov1api.BackupRepositoryStatus{
RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)},
Message: "fake-history-message-2",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)},
Message: "fake-history-message-3",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)},
Message: "fake-history-message-4",
},
},
},
}
backupRepoWithOverFullHistory := &velerov1api.BackupRepository{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "repo",
},
Status: velerov1api.BackupRepositoryStatus{
RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)},
Message: "fake-history-message-5",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)},
Message: "fake-history-message-6",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)},
Message: "fake-history-message-7",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 18)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 17)},
Message: "fake-history-message-8",
},
},
},
}
tests := []struct {
name string
backupRepo *velerov1api.BackupRepository
result velerov1api.BackupRepositoryMaintenanceResult
expectedHistory []velerov1api.BackupRepositoryMaintenanceStatus
}{
{
name: "empty history",
backupRepo: backupRepoWithoutHistory,
result: velerov1api.BackupRepositoryMaintenanceSucceeded,
expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)},
Message: "fake-message-0",
},
},
},
{
name: "less than history queue length",
backupRepo: backupRepoWithHistory,
result: velerov1api.BackupRepositoryMaintenanceSucceeded,
expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)},
Message: "fake-history-message-1",
},
{
StartTimestamp: &metav1.Time{Time: standardTime},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)},
Message: "fake-message-0",
},
},
},
{
name: "full history",
backupRepo: backupRepoWithFullHistory,
result: velerov1api.BackupRepositoryMaintenanceFailed,
expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)},
Message: "fake-history-message-3",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)},
Message: "fake-history-message-4",
},
{
StartTimestamp: &metav1.Time{Time: standardTime},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)},
Message: "fake-message-0",
},
},
},
{
name: "over full history",
backupRepo: backupRepoWithOverFullHistory,
result: velerov1api.BackupRepositoryMaintenanceFailed,
expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)},
Message: "fake-history-message-7",
},
{
StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 18)},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 17)},
Message: "fake-history-message-8",
},
{
StartTimestamp: &metav1.Time{Time: standardTime},
CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)},
Message: "fake-message-0",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
updateRepoMaintenanceHistory(test.backupRepo, test.result, standardTime, standardTime.Add(time.Hour), "fake-message-0")
for at := range test.backupRepo.Status.RecentMaintenance {
assert.Equal(t, test.expectedHistory[at].StartTimestamp.Time, test.backupRepo.Status.RecentMaintenance[at].StartTimestamp.Time)
assert.Equal(t, test.expectedHistory[at].CompleteTimestamp.Time, test.backupRepo.Status.RecentMaintenance[at].CompleteTimestamp.Time)
assert.Equal(t, test.expectedHistory[at].Message, test.backupRepo.Status.RecentMaintenance[at].Message)
}
})
}
}