Merge pull request #2560 from skriss/save-progress-updates

Save backup progress updates to object storage

pkg/apis/velero/v1/backup.go

@@ -277,6 +277,8 @@ type BackupStatus struct {
 	// Progress contains information about the backup's execution progress. Note
 	// that this information is best-effort only -- if Velero fails to update it
 	// during a backup for any reason, it may be inaccurate/stale.
+	// +optional
+	// +nullable
 	Progress *BackupProgress `json:"progress,omitempty"`
 }
 

pkg/backup/backup.go

@@ -294,6 +294,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 	items := collector.getAllItems()
 	log.WithField("progress", "").Infof("Collected %d items matching the backup spec from the Kubernetes API (actual number of items backed up may be more or less depending on velero.io/exclude-from-backup annotation, plugins returning additional related items to back up, etc.)", len(items))
 
+	backupRequest.Status.Progress = &velerov1api.BackupProgress{TotalItems: len(items)}
 	patch := fmt.Sprintf(`{"status":{"progress":{"totalItems":%d}}}`, len(items))
 	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
 		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress.totalItems")
@@ -345,6 +346,9 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 				lastUpdate = &val
 			case <-ticker.C:
 				if lastUpdate != nil {
+					backupRequest.Status.Progress.TotalItems = lastUpdate.totalItems
+					backupRequest.Status.Progress.ItemsBackedUp = lastUpdate.itemsBackedUp
+
 					patch := fmt.Sprintf(`{"status":{"progress":{"totalItems":%d,"itemsBackedUp":%d}}}`, lastUpdate.totalItems, lastUpdate.itemsBackedUp)
 					if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
 						log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")
@@ -421,6 +425,9 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 
 	// do a final update on progress since we may have just added some CRDs and may not have updated
 	// for the last few processed items.
+	backupRequest.Status.Progress.TotalItems = len(backupRequest.BackedUpItems)
+	backupRequest.Status.Progress.ItemsBackedUp = len(backupRequest.BackedUpItems)
+
 	patch = fmt.Sprintf(`{"status":{"progress":{"totalItems":%d,"itemsBackedUp":%d}}}`, len(backupRequest.BackedUpItems), len(backupRequest.BackedUpItems))
 	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
 		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")

pkg/backup/backup_test.go

@@ -114,6 +114,40 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) {
 	assertTarballContents(t, backupFile, append(expectedFiles, "metadata/version")...)
 }
 
+// TestBackupProgressIsUpdated verifies that after a backup has run, its
+// status.progress fields are updated to reflect the total number of items
+// backed up. It validates this by comparing their values to the length of
+// the request's BackedUpItems field.
+func TestBackupProgressIsUpdated(t *testing.T) {
+	h := newHarness(t)
+	req := &Request{Backup: defaultBackup().Result()}
+	backupFile := bytes.NewBuffer([]byte{})
+
+	apiResources := []*test.APIResource{
+		test.Pods(
+			builder.ForPod("foo", "bar").Result(),
+			builder.ForPod("zoo", "raz").Result(),
+		),
+		test.Deployments(
+			builder.ForDeployment("foo", "bar").Result(),
+			builder.ForDeployment("zoo", "raz").Result(),
+		),
+		test.PVs(
+			builder.ForPersistentVolume("bar").Result(),
+			builder.ForPersistentVolume("baz").Result(),
+		),
+	}
+	for _, resource := range apiResources {
+		h.addItems(t, resource)
+	}
+
+	h.backupper.Backup(h.log, req, backupFile, nil, nil)
+
+	require.NotNil(t, req.Status.Progress)
+	assert.Equal(t, len(req.BackedUpItems), req.Status.Progress.TotalItems)
+	assert.Equal(t, len(req.BackedUpItems), req.Status.Progress.ItemsBackedUp)
+}
+
 // TestBackupResourceFiltering runs backups with different combinations
 // of resource filters (included/excluded resources, included/excluded
 // namespaces, label selectors, "include cluster resources" flag), and

pkg/generated/crds/manifests/velero.io_backups.yaml

@@ -359,6 +359,7 @@ spec:
               description: Progress contains information about the backup's execution
                 progress. Note that this information is best-effort only -- if Velero
                 fails to update it during a backup for any reason, it may be inaccurate/stale.
+              nullable: true
               properties:
                 itemsBackedUp:
                   description: ItemsBackedUp is the number of items that have actually