Adds <backup-name>-itemsnapshots.gz file to backup (when provided). (#4429)

* Adds <backup-name>-itemsnapshots.gz file to backup (when provided).  Also
adds DownloadTargetKindBackupItemSnapshots type to allow downloading.
Updated object store unit test

Fixes #3758

Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com>

* Removed redundant checks

Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com>

changelogs/unreleased/4429-dsmithuchida

@@ -0,0 +1,8 @@
+Added `<backup name>`-itemsnapshots.json.gz to the backup format.  This file exists
+when item snapshots are taken and contains an array of volume.Itemsnapshots 
+containing the information about the snapshots.  This will not be used unless 
+upload progress monitoring and item snapshots are enabled and an ItemSnapshot 
+plugin is used to take snapshots.
+
+Also added DownloadTargetKindBackupItemSnapshots for retrieving the signed URL to download only the `<backup name>`-itemsnapshots.json.gz part of a backup for use by 
+`velero backup describe`.

config/crd/v1/bases/velero.io_downloadrequests.yaml

@@ -46,6 +46,7 @@ spec:
                     - BackupLog
                     - BackupContents
                     - BackupVolumeSnapshots
+                    - BackupItemSnapshots
                     - BackupResourceList
                     - RestoreLog
                     - RestoreResults

pkg/apis/velero/v1/download_request_types.go

@@ -25,13 +25,14 @@ type DownloadRequestSpec struct {
 }
 
 // DownloadTargetKind represents what type of file to download.
-// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupResourceList;RestoreLog;RestoreResults
+// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemSnapshots;BackupResourceList;RestoreLog;RestoreResults
 type DownloadTargetKind string
 
 const (
 	DownloadTargetKindBackupLog             DownloadTargetKind = "BackupLog"
 	DownloadTargetKindBackupContents        DownloadTargetKind = "BackupContents"
 	DownloadTargetKindBackupVolumeSnapshots DownloadTargetKind = "BackupVolumeSnapshots"
+	DownloadTargetKindBackupItemSnapshots   DownloadTargetKind = "BackupItemSnapshots"
 	DownloadTargetKindBackupResourceList    DownloadTargetKind = "BackupResourceList"
 	DownloadTargetKindRestoreLog            DownloadTargetKind = "RestoreLog"
 	DownloadTargetKindRestoreResults        DownloadTargetKind = "RestoreResults"

pkg/persistence/mocks/backup_store.go

@@ -283,3 +283,7 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv
 	panic("Not implemented")
 	return nil, nil
 }
+
+func (_m *BackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) {
+	panic("implement me")
+}

pkg/persistence/object_store.go

@@ -44,6 +44,7 @@ type BackupInfo struct {
 	Log,
 	PodVolumeBackups,
 	VolumeSnapshots,
+	ItemSnapshots,
 	BackupResourceList,
 	CSIVolumeSnapshots,
 	CSIVolumeSnapshotContents io.Reader
@@ -58,6 +59,7 @@ type BackupStore interface {
 
 	PutBackup(info BackupInfo) error
 	GetBackupMetadata(name string) (*velerov1api.Backup, error)
+	GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error)
 	GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error)
 	GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error)
 	GetBackupContents(name string) (io.ReadCloser, error)
@@ -231,13 +233,6 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
 		s.logger.WithError(err).WithField("backup", info.Name).Error("Error uploading log file")
 	}
 
-	if info.Metadata == nil {
-		// If we don't have metadata, something failed, and there's no point in continuing. An object
-		// storage bucket that is missing the metadata file can't be restored, nor can its logs be
-		// viewed.
-		return nil
-	}
-
 	if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(info.Name), info.Metadata); err != nil {
 		// failure to upload metadata file is a hard-stop
 		return err
@@ -253,6 +248,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
 	var backupObjs = map[string]io.Reader{
 		s.layout.getPodVolumeBackupsKey(info.Name):          info.PodVolumeBackups,
 		s.layout.getBackupVolumeSnapshotsKey(info.Name):     info.VolumeSnapshots,
+		s.layout.getItemSnapshotsKey(info.Name):             info.ItemSnapshots,
 		s.layout.getBackupResourceListKey(info.Name):        info.BackupResourceList,
 		s.layout.getCSIVolumeSnapshotKey(info.Name):         info.CSIVolumeSnapshots,
 		s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents,
@@ -324,6 +320,27 @@ func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Sna
 	return volumeSnapshots, nil
 }
 
+func (s *objectBackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) {
+	// if the itemsnapshots file doesn't exist, we don't want to return an error, since
+	// a legacy backup or a backup with no snapshots would not have this file, so check for
+	// its existence before attempting to get its contents.
+	res, err := tryGet(s.objectStore, s.bucket, s.layout.getItemSnapshotsKey(name))
+	if err != nil {
+		return nil, err
+	}
+	if res == nil {
+		return nil, nil
+	}
+	defer res.Close()
+
+	var itemSnapshots []*volume.ItemSnapshot
+	if err := decode(res, &itemSnapshots); err != nil {
+		return nil, err
+	}
+
+	return itemSnapshots, nil
+}
+
 // tryGet returns the object with the given key if it exists, nil if it does not exist,
 // or an error if it was unable to check existence or get the object.
 func tryGet(objectStore velero.ObjectStore, bucket, key string) (io.ReadCloser, error) {
@@ -473,6 +490,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupLogKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindBackupVolumeSnapshots:
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeSnapshotsKey(target.Name), DownloadURLTTL)
+	case velerov1api.DownloadTargetKindBackupItemSnapshots:
+		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getItemSnapshotsKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindBackupResourceList:
 		return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResourceListKey(target.Name), DownloadURLTTL)
 	case velerov1api.DownloadTargetKindRestoreLog:

pkg/persistence/object_store_layout.go

@@ -88,6 +88,10 @@ func (l *ObjectStoreLayout) getBackupVolumeSnapshotsKey(backup string) string {
 	return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshots.json.gz", backup))
 }
 
+func (l *ObjectStoreLayout) getItemSnapshotsKey(backup string) string {
+	return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-itemsnapshots.json.gz", backup))
+}
+
 func (l *ObjectStoreLayout) getBackupResourceListKey(backup string) string {
 	return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-resource-list.json.gz", backup))
 }

pkg/persistence/object_store_test.go

@@ -223,6 +223,7 @@ func TestPutBackup(t *testing.T) {
 		log             io.Reader
 		podVolumeBackup io.Reader
 		snapshots       io.Reader
+		itemSnapshots   io.Reader
 		resourceList    io.Reader
 		expectedErr     string
 		expectedKeys    []string
@@ -234,6 +235,7 @@ func TestPutBackup(t *testing.T) {
 			log:             newStringReadSeeker("log"),
 			podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
 			snapshots:       newStringReadSeeker("snapshots"),
+			itemSnapshots:   newStringReadSeeker("itemSnapshots"),
 			resourceList:    newStringReadSeeker("resourceList"),
 			expectedErr:     "",
 			expectedKeys: []string{
@@ -242,6 +244,7 @@ func TestPutBackup(t *testing.T) {
 				"backups/backup-1/backup-1-logs.gz",
 				"backups/backup-1/backup-1-podvolumebackups.json.gz",
 				"backups/backup-1/backup-1-volumesnapshots.json.gz",
+				"backups/backup-1/backup-1-itemsnapshots.json.gz",
 				"backups/backup-1/backup-1-resource-list.json.gz",
 			},
 		},
@@ -253,6 +256,7 @@ func TestPutBackup(t *testing.T) {
 			log:             newStringReadSeeker("log"),
 			podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
 			snapshots:       newStringReadSeeker("snapshots"),
+			itemSnapshots:   newStringReadSeeker("itemSnapshots"),
 			resourceList:    newStringReadSeeker("resourceList"),
 			expectedErr:     "",
 			expectedKeys: []string{
@@ -261,6 +265,7 @@ func TestPutBackup(t *testing.T) {
 				"prefix-1/backups/backup-1/backup-1-logs.gz",
 				"prefix-1/backups/backup-1/backup-1-podvolumebackups.json.gz",
 				"prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz",
+				"prefix-1/backups/backup-1/backup-1-itemsnapshots.json.gz",
 				"prefix-1/backups/backup-1/backup-1-resource-list.json.gz",
 			},
 		},
@@ -271,19 +276,21 @@ func TestPutBackup(t *testing.T) {
 			log:             newStringReadSeeker("log"),
 			podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
 			snapshots:       newStringReadSeeker("snapshots"),
+			itemSnapshots:   newStringReadSeeker("itemSnapshots"),
 			resourceList:    newStringReadSeeker("resourceList"),
 			expectedErr:     "error readers return errors",
 			expectedKeys:    []string{"backups/backup-1/backup-1-logs.gz"},
 		},
 		{
-			name:         "error on data upload deletes metadata",
-			metadata:     newStringReadSeeker("metadata"),
-			contents:     new(errorReader),
-			log:          newStringReadSeeker("log"),
-			snapshots:    newStringReadSeeker("snapshots"),
-			resourceList: newStringReadSeeker("resourceList"),
-			expectedErr:  "error readers return errors",
-			expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
+			name:          "error on data upload deletes metadata",
+			metadata:      newStringReadSeeker("metadata"),
+			contents:      new(errorReader),
+			log:           newStringReadSeeker("log"),
+			snapshots:     newStringReadSeeker("snapshots"),
+			itemSnapshots: newStringReadSeeker("itemSnapshots"),
+			resourceList:  newStringReadSeeker("resourceList"),
+			expectedErr:   "error readers return errors",
+			expectedKeys:  []string{"backups/backup-1/backup-1-logs.gz"},
 		},
 		{
 			name:            "error on log upload is ok",
@@ -292,6 +299,7 @@ func TestPutBackup(t *testing.T) {
 			log:             new(errorReader),
 			podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
 			snapshots:       newStringReadSeeker("snapshots"),
+			itemSnapshots:   newStringReadSeeker("itemSnapshots"),
 			resourceList:    newStringReadSeeker("resourceList"),
 			expectedErr:     "",
 			expectedKeys: []string{
@@ -299,11 +307,12 @@ func TestPutBackup(t *testing.T) {
 				"backups/backup-1/backup-1.tar.gz",
 				"backups/backup-1/backup-1-podvolumebackups.json.gz",
 				"backups/backup-1/backup-1-volumesnapshots.json.gz",
+				"backups/backup-1/backup-1-itemsnapshots.json.gz",
 				"backups/backup-1/backup-1-resource-list.json.gz",
 			},
 		},
 		{
-			name:            "don't upload data when metadata is nil",
+			name:            "data should be uploaded even when metadata is nil",
 			metadata:        nil,
 			contents:        newStringReadSeeker("contents"),
 			log:             newStringReadSeeker("log"),
@@ -311,7 +320,13 @@ func TestPutBackup(t *testing.T) {
 			snapshots:       newStringReadSeeker("snapshots"),
 			resourceList:    newStringReadSeeker("resourceList"),
 			expectedErr:     "",
-			expectedKeys:    []string{"backups/backup-1/backup-1-logs.gz"},
+			expectedKeys: []string{
+				"backups/backup-1/backup-1.tar.gz",
+				"backups/backup-1/backup-1-logs.gz",
+				"backups/backup-1/backup-1-podvolumebackups.json.gz",
+				"backups/backup-1/backup-1-volumesnapshots.json.gz",
+				"backups/backup-1/backup-1-resource-list.json.gz",
+			},
 		},
 	}
 
@@ -326,6 +341,7 @@ func TestPutBackup(t *testing.T) {
 				Log:                tc.log,
 				PodVolumeBackups:   tc.podVolumeBackup,
 				VolumeSnapshots:    tc.snapshots,
+				ItemSnapshots:      tc.itemSnapshots,
 				BackupResourceList: tc.resourceList,
 			}
 			err := harness.PutBackup(backupInfo)
@@ -426,6 +442,48 @@ func TestGetBackupVolumeSnapshots(t *testing.T) {
 	assert.EqualValues(t, snapshots, res)
 }
 
+func TestGetItemSnapshots(t *testing.T) {
+	harness := newObjectBackupStoreTestHarness("test-bucket", "")
+
+	// volumesnapshots file not found should not error
+	harness.objectStore.PutObject(harness.bucket, "backups/test-backup/velero-backup.json", newStringReadSeeker("foo"))
+	res, err := harness.GetItemSnapshots("test-backup")
+	assert.NoError(t, err)
+	assert.Nil(t, res)
+
+	// volumesnapshots file containing invalid data should error
+	harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", newStringReadSeeker("foo"))
+	res, err = harness.GetItemSnapshots("test-backup")
+	assert.NotNil(t, err)
+
+	// volumesnapshots file containing gzipped json data should return correctly
+	snapshots := []*volume.ItemSnapshot{
+		{
+			Spec: volume.ItemSnapshotSpec{
+				BackupName:         "test-backup",
+				ResourceIdentifier: "item-1",
+			},
+		},
+		{
+			Spec: volume.ItemSnapshotSpec{
+				BackupName:         "test-backup",
+				ResourceIdentifier: "item-2",
+			},
+		},
+	}
+
+	obj := new(bytes.Buffer)
+	gzw := gzip.NewWriter(obj)
+
+	require.NoError(t, json.NewEncoder(gzw).Encode(snapshots))
+	require.NoError(t, gzw.Close())
+	require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", obj))
+
+	res, err = harness.GetItemSnapshots("test-backup")
+	assert.NoError(t, err)
+	assert.EqualValues(t, snapshots, res)
+}
+
 func TestGetBackupContents(t *testing.T) {
 	harness := newObjectBackupStoreTestHarness("test-bucket", "")
 
@@ -506,6 +564,7 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindBackupContents:        "backups/my-backup/my-backup.tar.gz",
 				velerov1api.DownloadTargetKindBackupLog:             "backups/my-backup/my-backup-logs.gz",
 				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup/my-backup-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupItemSnapshots:   "backups/my-backup/my-backup-itemsnapshots.json.gz",
 				velerov1api.DownloadTargetKindBackupResourceList:    "backups/my-backup/my-backup-resource-list.json.gz",
 			},
 		},
@@ -517,6 +576,7 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindBackupContents:        "velero-backups/backups/my-backup/my-backup.tar.gz",
 				velerov1api.DownloadTargetKindBackupLog:             "velero-backups/backups/my-backup/my-backup-logs.gz",
 				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup/my-backup-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupItemSnapshots:   "velero-backups/backups/my-backup/my-backup-itemsnapshots.json.gz",
 				velerov1api.DownloadTargetKindBackupResourceList:    "velero-backups/backups/my-backup/my-backup-resource-list.json.gz",
 			},
 		},
@@ -527,6 +587,7 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindBackupContents:        "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902.tar.gz",
 				velerov1api.DownloadTargetKindBackupLog:             "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-logs.gz",
 				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupItemSnapshots:   "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-itemsnapshots.json.gz",
 				velerov1api.DownloadTargetKindBackupResourceList:    "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-resource-list.json.gz",
 			},
 		},
@@ -537,6 +598,7 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindBackupContents:        "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
 				velerov1api.DownloadTargetKindBackupLog:             "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
 				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupItemSnapshots:   "backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz",
 				velerov1api.DownloadTargetKindBackupResourceList:    "backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
 			},
 		},
@@ -548,6 +610,7 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindBackupContents:        "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
 				velerov1api.DownloadTargetKindBackupLog:             "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
 				velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
+				velerov1api.DownloadTargetKindBackupItemSnapshots:   "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz",
 				velerov1api.DownloadTargetKindBackupResourceList:    "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
 			},
 		},

pkg/volume/item_snapshot.go

@@ -0,0 +1,57 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1"
+
+// ItemSnapshot stores information about an item snapshot (includes volumes and other Astrolabe objects) taken as
+// part of a Velero backup.
+type ItemSnapshot struct {
+	Spec ItemSnapshotSpec `json:"spec"`
+
+	Status ItemSnapshotStatus `json:"status"`
+}
+
+type ItemSnapshotSpec struct {
+	// ItemSnapshotter is the name of the ItemSnapshotter plugin that took the snapshot
+	ItemSnapshotter string `json:"itemSnapshotter"`
+
+	// BackupName is the name of the Velero backup this snapshot
+	// is associated with.
+	BackupName string `json:"backupName"`
+
+	// BackupUID is the UID of the Velero backup this snapshot
+	// is associated with.
+	BackupUID string `json:"backupUID"`
+
+	// Location is the name of the location where this snapshot is stored.
+	Location string `json:"location"`
+
+	// Kubernetes resource identifier for the item
+	ResourceIdentifier string "json:resourceIdentifier"
+}
+
+type ItemSnapshotStatus struct {
+	// ProviderSnapshotID is the ID of the snapshot taken by the ItemSnapshotter
+	ProviderSnapshotID string `json:"providerSnapshotID,omitempty"`
+
+	// Metadata is the metadata returned with the snapshot to be returned to the ItemSnapshotter at restore time
+	Metadata map[string]string `json:"metadata,omitempty"`
+
+	// Phase is the current state of the ItemSnapshot.
+	Phase isv1.SnapshotPhase `json:"phase,omitempty"`
+}