mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-25 09:24:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Adds <backup-name>-itemsnapshots.gz file to backup (when provided). (#4429)

Browse Source 
* Adds <backup-name>-itemsnapshots.gz file to backup (when provided).  Also
adds DownloadTargetKindBackupItemSnapshots type to allow downloading.
Updated object store unit test

Fixes #3758

Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com>

* Removed redundant checks

Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com>
This commit is contained in:
David L. Smith-Uchida
2021-12-13 14:47:50 -05:00
committed by GitHub
parent 3445c402a9
commit a1b48ceac5
9 changed files with 176 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/persistence/mocks/backup_store.go
+4
View File
@@ -283,3 +283,7 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv
panic("Not implemented")
return nil, nil
}
func (_m *BackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) {
panic("implement me")
}
pkg/persistence/object_store.go
+26 -7
View File
@@ -44,6 +44,7 @@ type BackupInfo struct {
Log,
PodVolumeBackups,
VolumeSnapshots,
ItemSnapshots,
BackupResourceList,
CSIVolumeSnapshots,
CSIVolumeSnapshotContents io.Reader
@@ -58,6 +59,7 @@ type BackupStore interface {
PutBackup(info BackupInfo) error
GetBackupMetadata(name string) (*velerov1api.Backup, error)
GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error)
GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error)
GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error)
GetBackupContents(name string) (io.ReadCloser, error)
@@ -231,13 +233,6 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
s.logger.WithError(err).WithField("backup", info.Name).Error("Error uploading log file")
}
if info.Metadata == nil {
// If we don't have metadata, something failed, and there's no point in continuing. An object
// storage bucket that is missing the metadata file can't be restored, nor can its logs be
// viewed.
return nil
}
if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(info.Name), info.Metadata); err != nil {
// failure to upload metadata file is a hard-stop
return err
@@ -253,6 +248,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
var backupObjs = map[string]io.Reader{
s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups,
s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots,
s.layout.getItemSnapshotsKey(info.Name): info.ItemSnapshots,
s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList,
s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots,
s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents,
@@ -324,6 +320,27 @@ func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Sna
return volumeSnapshots, nil
}
func (s *objectBackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) {
// if the itemsnapshots file doesn't exist, we don't want to return an error, since
// a legacy backup or a backup with no snapshots would not have this file, so check for
// its existence before attempting to get its contents.
res, err := tryGet(s.objectStore, s.bucket, s.layout.getItemSnapshotsKey(name))
if err != nil {
return nil, err
}
if res == nil {
return nil, nil
}
defer res.Close()
var itemSnapshots []*volume.ItemSnapshot
if err := decode(res, &itemSnapshots); err != nil {
return nil, err
}
return itemSnapshots, nil
}
// tryGet returns the object with the given key if it exists, nil if it does not exist,
// or an error if it was unable to check existence or get the object.
func tryGet(objectStore velero.ObjectStore, bucket, key string) (io.ReadCloser, error) {
@@ -473,6 +490,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s
return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupLogKey(target.Name), DownloadURLTTL)
case velerov1api.DownloadTargetKindBackupVolumeSnapshots:
return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeSnapshotsKey(target.Name), DownloadURLTTL)
case velerov1api.DownloadTargetKindBackupItemSnapshots:
return s.objectStore.CreateSignedURL(s.bucket, s.layout.getItemSnapshotsKey(target.Name), DownloadURLTTL)
case velerov1api.DownloadTargetKindBackupResourceList:
return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResourceListKey(target.Name), DownloadURLTTL)
case velerov1api.DownloadTargetKindRestoreLog:
pkg/persistence/object_store_layout.go
+4
View File
@@ -88,6 +88,10 @@ func (l *ObjectStoreLayout) getBackupVolumeSnapshotsKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshots.json.gz", backup))
}
func (l *ObjectStoreLayout) getItemSnapshotsKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-itemsnapshots.json.gz", backup))
}
func (l *ObjectStoreLayout) getBackupResourceListKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-resource-list.json.gz", backup))
}
pkg/persistence/object_store_test.go
+73 -10
View File
@@ -223,6 +223,7 @@ func TestPutBackup(t *testing.T) {
log io.Reader
podVolumeBackup io.Reader
snapshots io.Reader
itemSnapshots io.Reader
resourceList io.Reader
expectedErr string
expectedKeys []string
@@ -234,6 +235,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
itemSnapshots: newStringReadSeeker("itemSnapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
@@ -242,6 +244,7 @@ func TestPutBackup(t *testing.T) {
"backups/backup-1/backup-1-logs.gz",
"backups/backup-1/backup-1-podvolumebackups.json.gz",
"backups/backup-1/backup-1-volumesnapshots.json.gz",
"backups/backup-1/backup-1-itemsnapshots.json.gz",
"backups/backup-1/backup-1-resource-list.json.gz",
},
},
@@ -253,6 +256,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
itemSnapshots: newStringReadSeeker("itemSnapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
@@ -261,6 +265,7 @@ func TestPutBackup(t *testing.T) {
"prefix-1/backups/backup-1/backup-1-logs.gz",
"prefix-1/backups/backup-1/backup-1-podvolumebackups.json.gz",
"prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz",
"prefix-1/backups/backup-1/backup-1-itemsnapshots.json.gz",
"prefix-1/backups/backup-1/backup-1-resource-list.json.gz",
},
},
@@ -271,19 +276,21 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
itemSnapshots: newStringReadSeeker("itemSnapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "error readers return errors",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
},
{
name: "error on data upload deletes metadata",
metadata: newStringReadSeeker("metadata"),
contents: new(errorReader),
log: newStringReadSeeker("log"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "error readers return errors",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
name: "error on data upload deletes metadata",
metadata: newStringReadSeeker("metadata"),
contents: new(errorReader),
log: newStringReadSeeker("log"),
snapshots: newStringReadSeeker("snapshots"),
itemSnapshots: newStringReadSeeker("itemSnapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "error readers return errors",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
},
{
name: "error on log upload is ok",
@@ -292,6 +299,7 @@ func TestPutBackup(t *testing.T) {
log: new(errorReader),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
itemSnapshots: newStringReadSeeker("itemSnapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
@@ -299,11 +307,12 @@ func TestPutBackup(t *testing.T) {
"backups/backup-1/backup-1.tar.gz",
"backups/backup-1/backup-1-podvolumebackups.json.gz",
"backups/backup-1/backup-1-volumesnapshots.json.gz",
"backups/backup-1/backup-1-itemsnapshots.json.gz",
"backups/backup-1/backup-1-resource-list.json.gz",
},
},
{
name: "don't upload data when metadata is nil",
name: "data should be uploaded even when metadata is nil",
metadata: nil,
contents: newStringReadSeeker("contents"),
log: newStringReadSeeker("log"),
@@ -311,7 +320,13 @@ func TestPutBackup(t *testing.T) {
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
expectedKeys: []string{
"backups/backup-1/backup-1.tar.gz",
"backups/backup-1/backup-1-logs.gz",
"backups/backup-1/backup-1-podvolumebackups.json.gz",
"backups/backup-1/backup-1-volumesnapshots.json.gz",
"backups/backup-1/backup-1-resource-list.json.gz",
},
},
}
@@ -326,6 +341,7 @@ func TestPutBackup(t *testing.T) {
Log: tc.log,
PodVolumeBackups: tc.podVolumeBackup,
VolumeSnapshots: tc.snapshots,
ItemSnapshots: tc.itemSnapshots,
BackupResourceList: tc.resourceList,
}
err := harness.PutBackup(backupInfo)
@@ -426,6 +442,48 @@ func TestGetBackupVolumeSnapshots(t *testing.T) {
assert.EqualValues(t, snapshots, res)
}
func TestGetItemSnapshots(t *testing.T) {
harness := newObjectBackupStoreTestHarness("test-bucket", "")
// volumesnapshots file not found should not error
harness.objectStore.PutObject(harness.bucket, "backups/test-backup/velero-backup.json", newStringReadSeeker("foo"))
res, err := harness.GetItemSnapshots("test-backup")
assert.NoError(t, err)
assert.Nil(t, res)
// volumesnapshots file containing invalid data should error
harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", newStringReadSeeker("foo"))
res, err = harness.GetItemSnapshots("test-backup")
assert.NotNil(t, err)
// volumesnapshots file containing gzipped json data should return correctly
snapshots := []*volume.ItemSnapshot{
{
Spec: volume.ItemSnapshotSpec{
BackupName: "test-backup",
ResourceIdentifier: "item-1",
},
},
{
Spec: volume.ItemSnapshotSpec{
BackupName: "test-backup",
ResourceIdentifier: "item-2",
},
},
}
obj := new(bytes.Buffer)
gzw := gzip.NewWriter(obj)
require.NoError(t, json.NewEncoder(gzw).Encode(snapshots))
require.NoError(t, gzw.Close())
require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", obj))
res, err = harness.GetItemSnapshots("test-backup")
assert.NoError(t, err)
assert.EqualValues(t, snapshots, res)
}
func TestGetBackupContents(t *testing.T) {
harness := newObjectBackupStoreTestHarness("test-bucket", "")
@@ -506,6 +564,7 @@ func TestGetDownloadURL(t *testing.T) {
velerov1api.DownloadTargetKindBackupContents: "backups/my-backup/my-backup.tar.gz",
velerov1api.DownloadTargetKindBackupLog: "backups/my-backup/my-backup-logs.gz",
velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup/my-backup-volumesnapshots.json.gz",
velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/my-backup/my-backup-itemsnapshots.json.gz",
velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup/my-backup-resource-list.json.gz",
},
},
@@ -517,6 +576,7 @@ func TestGetDownloadURL(t *testing.T) {
velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup/my-backup.tar.gz",
velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup/my-backup-logs.gz",
velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup/my-backup-volumesnapshots.json.gz",
velerov1api.DownloadTargetKindBackupItemSnapshots: "velero-backups/backups/my-backup/my-backup-itemsnapshots.json.gz",
velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup/my-backup-resource-list.json.gz",
},
},
@@ -527,6 +587,7 @@ func TestGetDownloadURL(t *testing.T) {
velerov1api.DownloadTargetKindBackupContents: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902.tar.gz",
velerov1api.DownloadTargetKindBackupLog: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-logs.gz",
velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-volumesnapshots.json.gz",
velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-itemsnapshots.json.gz",
velerov1api.DownloadTargetKindBackupResourceList: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-resource-list.json.gz",
},
},
@@ -537,6 +598,7 @@ func TestGetDownloadURL(t *testing.T) {
velerov1api.DownloadTargetKindBackupContents: "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
velerov1api.DownloadTargetKindBackupLog: "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz",
velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
},
},
@@ -548,6 +610,7 @@ func TestGetDownloadURL(t *testing.T) {
velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz",
velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz",
velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz",
velerov1api.DownloadTargetKindBackupItemSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz",
velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz",
},
},