print resource list metadata in velero backup describe --details (#1714)

* print resource list metadata in velero backup describe details

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* rewrite TestGetDownloadURL to test more scenarios

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* move backup printer helpers to backup_printer.go

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* move describe printer helpers back to backup_describer

and rename to prefix with describe* to indicate that they are used for the describe command

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* split backup and restore tests for TestGetDownloadURL

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* friendlier error message when backup resource list missing

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

pkg/apis/velero/v1/download_request.go

@@ -31,6 +31,7 @@ const (
 	DownloadTargetKindBackupLog             DownloadTargetKind = "BackupLog"
 	DownloadTargetKindBackupContents        DownloadTargetKind = "BackupContents"
 	DownloadTargetKindBackupVolumeSnapshots DownloadTargetKind = "BackupVolumeSnapshots"
+	DownloadTargetKindBackupResourceList    DownloadTargetKind = "BackupResourceList"
 	DownloadTargetKindRestoreLog            DownloadTargetKind = "RestoreLog"
 	DownloadTargetKindRestoreResults        DownloadTargetKind = "RestoreResults"
 )

pkg/cmd/util/downloadrequest/downloadrequest.go

@@ -32,6 +32,10 @@ import (
 	velerov1client "github.com/heptio/velero/pkg/generated/clientset/versioned/typed/velero/v1"
 )
 
+// ErrNotFound is exported for external packages to check for when a file is
+// not found
+var ErrNotFound = errors.New("file not found")
+
 func Stream(client velerov1client.DownloadRequestsGetter, namespace, name string, kind v1.DownloadTargetKind, w io.Writer, timeout time.Duration) error {
 	req := &v1.DownloadRequest{
 		ObjectMeta: metav1.ObjectMeta{
@@ -97,7 +101,7 @@ Loop:
 	}
 
 	if req.Status.DownloadURL == "" {
-		return errors.New("file not found")
+		return ErrNotFound
 	}
 
 	httpClient := new(http.Client)
@@ -124,6 +128,10 @@ Loop:
 			return errors.Wrapf(err, "request failed: unable to decode response body")
 		}
 
+		if resp.StatusCode == http.StatusNotFound {
+			return ErrNotFound
+		}
+
 		return errors.Errorf("request failed: %v", string(body))
 	}
 

pkg/cmd/util/output/backup_describer.go

@@ -233,6 +233,11 @@ func DescribeBackupStatus(d *Describer, backup *velerov1api.Backup, details bool
 	d.Printf("Expiration:\t%s\n", status.Expiration.Time)
 	d.Println()
 
+	if details {
+		describeBackupResourceList(d, backup, veleroClient)
+		d.Println()
+	}
+
 	if status.VolumeSnapshotsAttempted > 0 {
 		if !details {
 			d.Printf("Persistent Volumes:\t%d of %d snapshots completed successfully (specify --details for more information)\n", status.VolumeSnapshotsCompleted, status.VolumeSnapshotsAttempted)
@@ -253,7 +258,7 @@ func DescribeBackupStatus(d *Describer, backup *velerov1api.Backup, details bool
 
 		d.Printf("Persistent Volumes:\n")
 		for _, snap := range snapshots {
-			printSnapshot(d, snap.Spec.PersistentVolumeName, snap.Status.ProviderSnapshotID, snap.Spec.VolumeType, snap.Spec.VolumeAZ, snap.Spec.VolumeIOPS)
+			describeSnapshot(d, snap.Spec.PersistentVolumeName, snap.Status.ProviderSnapshotID, snap.Spec.VolumeType, snap.Spec.VolumeAZ, snap.Spec.VolumeIOPS)
 		}
 		return
 	}
@@ -261,7 +266,30 @@ func DescribeBackupStatus(d *Describer, backup *velerov1api.Backup, details bool
 	d.Printf("Persistent Volumes: <none included>\n")
 }
 
-func printSnapshot(d *Describer, pvName, snapshotID, volumeType, volumeAZ string, iops *int64) {
+func describeBackupResourceList(d *Describer, backup *velerov1api.Backup, veleroClient clientset.Interface) {
+	buf := new(bytes.Buffer)
+	if err := downloadrequest.Stream(veleroClient.VeleroV1(), backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout); err != nil {
+		if err == downloadrequest.ErrNotFound {
+			d.Println("Resource List:\t<backup resource list not found, this could be because this backup was taken prior to Velero 1.1.0>")
+		} else {
+			d.Printf("Resource List:\t<error getting backup resource list: %v>\n", err)
+		}
+		return
+	}
+
+	var resourceList map[string][]string
+	if err := json.NewDecoder(buf).Decode(&resourceList); err != nil {
+		d.Printf("Resource List:\t<error reading backup resource list: %v>\n", err)
+		return
+	}
+
+	d.Println("Resource List:")
+	for gvk, items := range resourceList {
+		d.Printf("\t%s:\n\t\t- %s\n", gvk, strings.Join(items, "\n\t\t- "))
+	}
+}
+
+func describeSnapshot(d *Describer, pvName, snapshotID, volumeType, volumeAZ string, iops *int64) {
 	d.Printf("\t%s:\n", pvName)
 	d.Printf("\t\tSnapshot ID:\t%s\n", snapshotID)
 	d.Printf("\t\tType:\t%s\n", volumeType)

pkg/persistence/object_store.go

@@ -438,6 +438,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupLogKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindBackupVolumeSnapshots:
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeSnapshotsKey(target.Name), DownloadURLTTL)
+	case velerov1api.DownloadTargetKindBackupResourceList:
+		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResourceListKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindRestoreLog:
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreLogKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindRestoreResults:

pkg/persistence/object_store_test.go

@@ -492,60 +492,87 @@ func TestDeleteBackup(t *testing.T) {
 
 func TestGetDownloadURL(t *testing.T) {
 	tests := []struct {
-		name        string
+		name              string
-		targetKind  velerov1api.DownloadTargetKind
+		targetName        string
-		targetName  string
+		expectedKeyByKind map[velerov1api.DownloadTargetKind]string
-		prefix      string
+		prefix            string
-		expectedKey string
 	}{
 		{
-			name:        "backup contents",
+			name:       "backup",
-			targetKind:  velerov1api.DownloadTargetKindBackupContents,
+			targetName: "my-backup",
-			targetName:  "my-backup",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "backups/my-backup/my-backup.tar.gz",
+				velerov1api.DownloadTargetKindBackupContents:        "backups/my-backup/my-backup.tar.gz",
+				velerov1api.DownloadTargetKindBackupLog:             "backups/my-backup/my-backup-logs.gz",
+				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup/my-backup-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupResourceList:    "backups/my-backup/my-backup-resource-list.json.gz",
+			},
 		},
 		{
-			name:        "backup log",
+			name:       "backup with prefix",
-			targetKind:  velerov1api.DownloadTargetKindBackupLog,
+			targetName: "my-backup",
-			targetName:  "my-backup",
+			prefix:     "velero-backups/",
-			expectedKey: "backups/my-backup/my-backup-logs.gz",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
+				velerov1api.DownloadTargetKindBackupContents:        "velero-backups/backups/my-backup/my-backup.tar.gz",
+				velerov1api.DownloadTargetKindBackupLog:             "velero-backups/backups/my-backup/my-backup-logs.gz",
+				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup/my-backup-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupResourceList:    "velero-backups/backups/my-backup/my-backup-resource-list.json.gz",
+			},
 		},
 		{
-			name:        "scheduled backup contents",
+			name:       "backup with multiple dashes",
-			targetKind:  velerov1api.DownloadTargetKindBackupContents,
+			targetName: "b-cool-20170913154901-20170913154902",
-			targetName:  "my-backup-20170913154901",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
+				velerov1api.DownloadTargetKindBackupContents:        "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902.tar.gz",
+				velerov1api.DownloadTargetKindBackupLog:             "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-logs.gz",
+				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupResourceList:    "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-resource-list.json.gz",
+			},
 		},
 		{
-			name:        "scheduled backup log",
+			name:       "scheduled backup",
-			targetKind:  velerov1api.DownloadTargetKindBackupLog,
+			targetName: "my-backup-20170913154901",
-			targetName:  "my-backup-20170913154901",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
+				velerov1api.DownloadTargetKindBackupContents:        "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
+				velerov1api.DownloadTargetKindBackupLog:             "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
+				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupResourceList:    "backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
+			},
 		},
 		{
-			name:        "backup contents with backup store prefix",
+			name:       "scheduled backup with prefix",
-			targetKind:  velerov1api.DownloadTargetKindBackupContents,
+			targetName: "my-backup-20170913154901",
-			targetName:  "my-backup",
+			prefix:     "velero-backups/",
-			prefix:      "velero-backups/",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "velero-backups/backups/my-backup/my-backup.tar.gz",
+				velerov1api.DownloadTargetKindBackupContents:        "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
+				velerov1api.DownloadTargetKindBackupLog:             "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
+				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupResourceList:    "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
+			},
 		},
 		{
-			name:        "restore log",
+			name:       "restore",
-			targetKind:  velerov1api.DownloadTargetKindRestoreLog,
+			targetName: "my-backup",
-			targetName:  "b-20170913154901",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "restores/b-20170913154901/restore-b-20170913154901-logs.gz",
+				velerov1api.DownloadTargetKindRestoreLog:     "restores/my-backup/restore-my-backup-logs.gz",
+				velerov1api.DownloadTargetKindRestoreResults: "restores/my-backup/restore-my-backup-results.gz",
+			},
 		},
 		{
-			name:        "restore results",
+			name:       "restore with prefix",
-			targetKind:  velerov1api.DownloadTargetKindRestoreResults,
+			targetName: "my-backup",
-			targetName:  "b-20170913154901",
+			prefix:     "velero-backups/",
-			expectedKey: "restores/b-20170913154901/restore-b-20170913154901-results.gz",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
+				velerov1api.DownloadTargetKindRestoreLog:     "velero-backups/restores/my-backup/restore-my-backup-logs.gz",
+				velerov1api.DownloadTargetKindRestoreResults: "velero-backups/restores/my-backup/restore-my-backup-results.gz",
+			},
 		},
 		{
-			name:        "restore results - backup has multiple dashes (e.g. restore of scheduled backup)",
+			name:       "restore with multiple dashes",
-			targetKind:  velerov1api.DownloadTargetKindRestoreResults,
+			targetName: "b-cool-20170913154901-20170913154902",
-			targetName:  "b-cool-20170913154901-20170913154902",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
-			expectedKey: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-results.gz",
+				velerov1api.DownloadTargetKindRestoreLog:     "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-logs.gz",
+				velerov1api.DownloadTargetKindRestoreResults: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-results.gz",
+			},
 		},
 	}
 
@@ -553,11 +580,15 @@ func TestGetDownloadURL(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			harness := newObjectBackupStoreTestHarness("test-bucket", test.prefix)
 
-			require.NoError(t, harness.objectStore.PutObject("test-bucket", test.expectedKey, newStringReadSeeker("foo")))
+			for kind, expectedKey := range test.expectedKeyByKind {
+				t.Run(string(kind), func(t *testing.T) {
+					require.NoError(t, harness.objectStore.PutObject("test-bucket", expectedKey, newStringReadSeeker("foo")))
 
-			url, err := harness.GetDownloadURL(velerov1api.DownloadTarget{Kind: test.targetKind, Name: test.targetName})
+					url, err := harness.GetDownloadURL(velerov1api.DownloadTarget{Kind: kind, Name: test.targetName})
-			require.NoError(t, err)
+					require.NoError(t, err)
-			assert.Equal(t, "a-url", url)
+					assert.Equal(t, "a-url", url)
+				})
+			}
 		})
 	}
 }