mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-05 04:55:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use backup storage location during restore

Browse Source 
Closes #740

Signed-off-by: Carlisia <carlisia@grokkingtech.io>
This commit is contained in:
Carlisia
2018-08-14 07:28:11 -07:00
committed by Steve Kriss
parent 20f89fbcef
commit 2750aa71b9
4 changed files with 326 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/cmd/server/server.go
View File

@@ -723,13 +723,13 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B
s.arkClient.ArkV1(),
s.arkClient.ArkV1(),
restorer,
config.BackupStorageProvider.CloudProviderConfig,
config.BackupStorageProvider.Bucket,
s.sharedInformerFactory.Ark().V1().Backups(),
s.sharedInformerFactory.Ark().V1().BackupStorageLocations(),
s.blockStore != nil,
s.logger,
s.logLevel,
s.pluginRegistry,
s.defaultBackupLocation,
s.metrics,
)

272
pkg/controller/restore_controller.go
View File

@@ -71,23 +71,24 @@ var nonRestorableResources = []string{
}
type restoreController struct {
namespace string
restoreClient arkv1client.RestoresGetter
backupClient arkv1client.BackupsGetter
restorer restore.Restorer
objectStoreConfig api.CloudProviderConfig
bucket string
pvProviderExists bool
backupLister listers.BackupLister
backupListerSynced cache.InformerSynced
restoreLister listers.RestoreLister
restoreListerSynced cache.InformerSynced
syncHandler func(restoreName string) error
queue workqueue.RateLimitingInterface
logger logrus.FieldLogger
logLevel logrus.Level
pluginRegistry plugin.Registry
metrics *metrics.ServerMetrics
namespace string
restoreClient arkv1client.RestoresGetter
backupClient arkv1client.BackupsGetter
restorer restore.Restorer
pvProviderExists bool
backupLister listers.BackupLister
backupListerSynced cache.InformerSynced
restoreLister listers.RestoreLister
restoreListerSynced cache.InformerSynced
backupLocationLister listers.BackupStorageLocationLister
backupLocationListerSynced cache.InformerSynced
syncHandler func(restoreName string) error
queue workqueue.RateLimitingInterface
logger logrus.FieldLogger
logLevel logrus.Level
pluginRegistry plugin.Registry
defaultBackupLocation string
metrics *metrics.ServerMetrics
getBackup cloudprovider.GetBackupFunc
downloadBackup cloudprovider.DownloadBackupFunc
@@ -102,33 +103,34 @@ func NewRestoreController(
restoreClient arkv1client.RestoresGetter,
backupClient arkv1client.BackupsGetter,
restorer restore.Restorer,
objectStoreConfig api.CloudProviderConfig,
bucket string,
backupInformer informers.BackupInformer,
backupLocationInformer informers.BackupStorageLocationInformer,
pvProviderExists bool,
logger logrus.FieldLogger,
logLevel logrus.Level,
pluginRegistry plugin.Registry,
defaultBackupLocation string,
metrics *metrics.ServerMetrics,
) Interface {
c := &restoreController{
namespace: namespace,
restoreClient: restoreClient,
backupClient: backupClient,
restorer: restorer,
objectStoreConfig: objectStoreConfig,
bucket: bucket,
pvProviderExists: pvProviderExists,
backupLister: backupInformer.Lister(),
backupListerSynced: backupInformer.Informer().HasSynced,
restoreLister: restoreInformer.Lister(),
restoreListerSynced: restoreInformer.Informer().HasSynced,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"),
logger: logger,
logLevel: logLevel,
pluginRegistry: pluginRegistry,
metrics: metrics,
namespace: namespace,
restoreClient: restoreClient,
backupClient: backupClient,
restorer: restorer,
pvProviderExists: pvProviderExists,
backupLister: backupInformer.Lister(),
backupListerSynced: backupInformer.Informer().HasSynced,
restoreLister: restoreInformer.Lister(),
restoreListerSynced: restoreInformer.Informer().HasSynced,
backupLocationLister: backupLocationInformer.Lister(),
backupLocationListerSynced: backupLocationInformer.Informer().HasSynced,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"),
logger: logger,
logLevel: logLevel,
pluginRegistry: pluginRegistry,
defaultBackupLocation: defaultBackupLocation,
metrics: metrics,
getBackup: cloudprovider.GetBackup,
downloadBackup: cloudprovider.DownloadBackup,
@@ -193,7 +195,7 @@ func (c *restoreController) Run(ctx context.Context, numWorkers int) error {
defer c.logger.Info("Shutting down RestoreController")
c.logger.Info("Waiting for caches to sync")
if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced) {
if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced, c.backupLocationListerSynced) {
return errors.New("timed out waiting for caches to sync")
}
c.logger.Info("Caches are synced")
@@ -283,28 +285,25 @@ func (c *restoreController) processRestore(key string) error {
pluginManager := c.newPluginManager(logContext, logContext.Level, c.pluginRegistry)
defer pluginManager.CleanupClients()
objectStore, err := getObjectStore(c.objectStoreConfig, pluginManager)
if err != nil {
return errors.Wrap(err, "error initializing object store")
}
actions, err := pluginManager.GetRestoreItemActions()
if err != nil {
return errors.Wrap(err, "error initializing restore item actions")
}
// complete & validate restore
if restore.Status.ValidationErrors = c.completeAndValidate(objectStore, restore); len(restore.Status.ValidationErrors) > 0 {
restore.Status.Phase = api.RestorePhaseFailedValidation
} else {
restore.Status.Phase = api.RestorePhaseInProgress
}
// validate the restore and fetch the backup
info := c.validateAndComplete(restore, pluginManager)
backupScheduleName := restore.Spec.ScheduleName
// Register attempts after validation so we don't have to fetch the backup multiple times
c.metrics.RegisterRestoreAttempt(backupScheduleName)
// update status
if len(restore.Status.ValidationErrors) > 0 {
restore.Status.Phase = api.RestorePhaseFailedValidation
c.metrics.RegisterRestoreValidationFailed(backupScheduleName)
} else {
restore.Status.Phase = api.RestorePhaseInProgress
}
// patch to update status and persist to API
updatedRestore, err := patchRestore(original, restore, c.restoreClient)
if err != nil {
return errors.Wrapf(err, "error updating Restore phase to %s", restore.Status.Phase)
@@ -314,15 +313,16 @@ func (c *restoreController) processRestore(key string) error {
restore = updatedRestore.DeepCopy()
if restore.Status.Phase == api.RestorePhaseFailedValidation {
c.metrics.RegisterRestoreValidationFailed(backupScheduleName)
return nil
}
logContext.Debug("Running restore")
// execution & upload of restore
restoreWarnings, restoreErrors, restoreFailure := c.runRestore(
restore,
actions,
objectStore,
info,
)
restore.Status.Warnings = len(restoreWarnings.Ark) + len(restoreWarnings.Cluster)
@@ -355,7 +355,13 @@ func (c *restoreController) processRestore(key string) error {
return nil
}
func (c *restoreController) completeAndValidate(objectStore cloudprovider.ObjectStore, restore *api.Restore) []string {
type backupInfo struct {
bucketName string
backup *api.Backup
objectStore cloudprovider.ObjectStore
}
func (c *restoreController) validateAndComplete(restore *api.Restore, pluginManager plugin.Manager) backupInfo {
// add non-restorable resources to restore's excluded resources
excludedResources := sets.NewString(restore.Spec.ExcludedResources...)
for _, nonrestorable := range nonRestorableResources {
@@ -363,34 +369,34 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object
restore.Spec.ExcludedResources = append(restore.Spec.ExcludedResources, nonrestorable)
}
}
var validationErrors []string
// validate that included resources don't contain any non-restorable resources
includedResources := sets.NewString(restore.Spec.IncludedResources...)
for _, nonRestorableResource := range nonRestorableResources {
if includedResources.Has(nonRestorableResource) {
validationErrors = append(validationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource))
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource))
}
}
// validate included/excluded resources
for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedResources, restore.Spec.ExcludedResources) {
validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err))
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err))
}
// validate included/excluded namespaces
for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedNamespaces, restore.Spec.ExcludedNamespaces) {
validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err))
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err))
}
// validate that PV provider exists if we're restoring PVs
if boolptr.IsSetToTrue(restore.Spec.RestorePVs) && !c.pvProviderExists {
validationErrors = append(validationErrors, "Server is not configured for PV snapshot restores")
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Server is not configured for PV snapshot restores")
}
// validate that exactly one of BackupName and ScheduleName have been specified
if !backupXorScheduleProvided(restore) {
return append(validationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both")
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both")
return backupInfo{}
}
// if ScheduleName is specified, fill in BackupName with the most recent successful backup from
@@ -402,33 +408,33 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object
backups, err := c.backupLister.Backups(c.namespace).List(selector)
if err != nil {
return append(validationErrors, "Unable to list backups for schedule")
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule")
return backupInfo{}
}
if len(backups) == 0 {
return append(validationErrors, "No backups found for schedule")
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule")
}
if backup := mostRecentCompletedBackup(backups); backup != nil {
restore.Spec.BackupName = backup.Name
} else {
return append(validationErrors, "No completed backups found for schedule")
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule")
return backupInfo{}
}
}
var (
backup *api.Backup
err error
)
if backup, err = c.fetchBackup(objectStore, restore.Spec.BackupName); err != nil {
return append(validationErrors, fmt.Sprintf("Error retrieving backup: %v", err))
info, err := c.fetchBackupInfo(restore.Spec.BackupName, pluginManager)
if err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err))
return backupInfo{}
}
// Fill in the ScheduleName so it's easier to consume for metrics.
if restore.Spec.ScheduleName == "" {
restore.Spec.ScheduleName = backup.GetLabels()["ark-schedule"]
restore.Spec.ScheduleName = info.backup.GetLabels()["ark-schedule"]
}
return validationErrors
return info
}
// backupXorScheduleProvided returns true if exactly one of BackupName and
@@ -462,43 +468,110 @@ func mostRecentCompletedBackup(backups []*api.Backup) *api.Backup {
return nil
}
func (c *restoreController) fetchBackup(objectStore cloudprovider.ObjectStore, name string) (*api.Backup, error) {
backup, err := c.backupLister.Backups(c.namespace).Get(name)
if err == nil {
return backup, nil
}
if !apierrors.IsNotFound(err) {
return nil, errors.WithStack(err)
}
logContext := c.logger.WithField("backupName", name)
logContext.Debug("Backup not found in backupLister, checking object storage directly")
backup, err = c.getBackup(objectStore, c.bucket, name)
// fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't
// find it, it tries to retrieve it from one of the backup storage locations.
func (c *restoreController) fetchBackupInfo(backupName string, pluginManager plugin.Manager) (backupInfo, error) {
var info backupInfo
var err error
info.backup, err = c.backupLister.Backups(c.namespace).Get(backupName)
if err != nil {
return nil, err
if !apierrors.IsNotFound(err) {
return backupInfo{}, errors.WithStack(err)
}
logContext := c.logger.WithField("backupName", backupName)
logContext.Debug("Backup not found in backupLister, checking each backup location directly, starting with default...")
return c.fetchFromBackupStorage(backupName, pluginManager)
}
location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(info.backup.Spec.StorageLocation)
if err != nil {
return backupInfo{}, errors.WithStack(err)
}
info.objectStore, err = getObjectStoreForLocation(location, pluginManager)
if err != nil {
return backupInfo{}, errors.Wrap(err, "error initializing object store")
}
info.bucketName = location.Spec.ObjectStorage.Bucket
return info, nil
}
// fetchFromBackupStorage checks each backup storage location, starting with the default,
// looking for a backup that matches the given backup name.
func (c *restoreController) fetchFromBackupStorage(backupName string, pluginManager plugin.Manager) (backupInfo, error) {
locations, err := c.backupLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything())
if err != nil {
return backupInfo{}, errors.WithStack(err)
}
orderedLocations := orderedBackupLocations(locations, c.defaultBackupLocation)
logContext := c.logger.WithField("backupName", backupName)
for _, location := range orderedLocations {
info, err := c.backupInfoForLocation(location, backupName, pluginManager)
if err != nil {
logContext.WithField("locationName", location.Name).WithError(err).Error("Unable to fetch backup from object storage location")
continue
}
return info, nil
}
return backupInfo{}, errors.New("not able to fetch from backup storage")
}
func orderedBackupLocations(locations []*api.BackupStorageLocation, defaultLocationName string) []*api.BackupStorageLocation {
var result []*api.BackupStorageLocation
for i := range locations {
if locations[i].Name == defaultLocationName {
// put the default location first
result = append(result, locations[i])
// append everything before the default
result = append(result, locations[:i]...)
// append everything after the default
result = append(result, locations[i+1:]...)
return result
}
}
return locations
}
func (c *restoreController) backupInfoForLocation(location *api.BackupStorageLocation, backupName string, pluginManager plugin.Manager) (backupInfo, error) {
objectStore, err := getObjectStoreForLocation(location, pluginManager)
if err != nil {
return backupInfo{}, err
}
backup, err := c.getBackup(objectStore, location.Spec.ObjectStorage.Bucket, backupName)
if err != nil {
return backupInfo{}, err
}
// ResourceVersion needs to be cleared in order to create the object in the API
backup.ResourceVersion = ""
// Clear out the namespace too, just in case
// Clear out the namespace, in case the backup was made in a different cluster, with a different namespace
backup.Namespace = ""
created, createErr := c.backupClient.Backups(c.namespace).Create(backup)
if createErr != nil {
logContext.WithError(errors.WithStack(createErr)).Error("Unable to create API object for Backup")
} else {
backup = created
backupCreated, err := c.backupClient.Backups(c.namespace).Create(backup)
if err != nil {
return backupInfo{}, errors.WithStack(err)
}
return backup, nil
return backupInfo{
bucketName: location.Spec.ObjectStorage.Bucket,
backup: backupCreated,
objectStore: objectStore,
}, nil
}
func (c *restoreController) runRestore(
restore *api.Restore,
actions []restore.ItemAction,
objectStore cloudprovider.ObjectStore,
info backupInfo,
) (restoreWarnings, restoreErrors api.RestoreResult, restoreFailure error) {
logFile, err := ioutil.TempFile("", "")
if err != nil {
@@ -530,14 +603,7 @@ func (c *restoreController) runRestore(
"backup": restore.Spec.BackupName,
})
backup, err := c.fetchBackup(objectStore, restore.Spec.BackupName)
if err != nil {
logContext.WithError(err).Error("Error getting backup")
restoreErrors.Ark = append(restoreErrors.Ark, err.Error())
return
}
backupFile, err := downloadToTempFile(objectStore, c.bucket, restore.Spec.BackupName, c.downloadBackup, c.logger)
backupFile, err := downloadToTempFile(info.objectStore, info.bucketName, restore.Spec.BackupName, c.downloadBackup, c.logger)
if err != nil {
logContext.WithError(err).Error("Error downloading backup")
restoreErrors.Ark = append(restoreErrors.Ark, err.Error())
@@ -558,7 +624,7 @@ func (c *restoreController) runRestore(
// Any return statement above this line means a total restore failure
// Some failures after this line *may* be a total restore failure
logContext.Info("starting restore")
restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, backup, backupFile, actions)
restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, info.backup, backupFile, actions)
logContext.Info("restore completed")
// Try to upload the log file. This is best-effort. If we fail, we'll add to the ark errors.
@@ -571,7 +637,7 @@ func (c *restoreController) runRestore(
return
}
if err := c.uploadRestoreLog(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, logFile); err != nil {
if err := c.uploadRestoreLog(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, logFile); err != nil {
restoreErrors.Ark = append(restoreErrors.Ark, fmt.Sprintf("error uploading log file to object storage: %v", err))
}
@@ -592,7 +658,7 @@ func (c *restoreController) runRestore(
logContext.WithError(errors.WithStack(err)).Error("Error resetting results file offset to 0")
return
}
if err := c.uploadRestoreResults(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, resultsFile); err != nil {
if err := c.uploadRestoreResults(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, resultsFile); err != nil {
logContext.WithError(errors.WithStack(err)).Error("Error uploading results files to object storage")
}

130
pkg/controller/restore_controller_test.go
View File

@@ -19,17 +19,16 @@ package controller
import (
"bytes"
"encoding/json"
"errors"
"io"
"io/ioutil"
"testing"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
core "k8s.io/client-go/testing"
@@ -47,10 +46,11 @@ import (
arktest "github.com/heptio/ark/pkg/util/test"
)
func TestFetchBackup(t *testing.T) {
func TestFetchBackupInfo(t *testing.T) {
tests := []struct {
name string
backupName string
informerLocations []*api.BackupStorageLocation
informerBackups []*api.Backup
backupServiceBackup *api.Backup
backupServiceError error
@@ -58,16 +58,19 @@ func TestFetchBackup(t *testing.T) {
expectedErr bool
}{
{
name: "lister has backup",
backupName: "backup-1",
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup,
name: "lister has backup",
backupName: "backup-1",
informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation},
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
{
name: "backupSvc has backup",
name: "lister does not have a backup, but backupSvc does",
backupName: "backup-1",
backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup,
backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation},
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
{
name: "no backup",
@@ -84,26 +87,43 @@ func TestFetchBackup(t *testing.T) {
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = arktest.NewLogger()
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
)
defer restorer.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
c := NewRestoreController(
api.DefaultNamespace,
sharedInformers.Ark().V1().Restores(),
client.ArkV1(),
client.ArkV1(),
restorer,
api.CloudProviderConfig{},
"bucket",
sharedInformers.Ark().V1().Backups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
false,
logger,
logrus.InfoLevel,
nil, //pluginRegistry
"default",
metrics.NewServerMetrics(),
).(*restoreController)
c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager {
return pluginManager
}
for _, itm := range test.informerBackups {
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm)
if test.backupServiceError == nil {
pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil)
objectStore.On("Init", mock.Anything).Return(nil)
for _, itm := range test.informerLocations {
sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(itm)
}
for _, itm := range test.informerBackups {
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm)
}
}
if test.backupServiceBackup != nil || test.backupServiceError != nil {
@@ -114,10 +134,10 @@ func TestFetchBackup(t *testing.T) {
}
}
backup, err := c.fetchBackup(nil, test.backupName)
info, err := c.fetchBackupInfo(test.backupName, pluginManager)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, backup)
assert.Equal(t, test.expectedRes, info.backup)
}
})
}
@@ -175,13 +195,13 @@ func TestProcessRestoreSkips(t *testing.T) {
client.ArkV1(),
client.ArkV1(),
restorer,
api.CloudProviderConfig{Name: "myCloud"},
"bucket",
sharedInformers.Ark().V1().Backups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
false, // pvProviderExists
logger,
logrus.InfoLevel,
nil, // pluginRegistry
"default",
metrics.NewServerMetrics(),
).(*restoreController)
c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager {
@@ -197,10 +217,12 @@ func TestProcessRestoreSkips(t *testing.T) {
})
}
}
func TestProcessRestore(t *testing.T) {
tests := []struct {
name string
restoreKey string
location *api.BackupStorageLocation
restore *api.Restore
backup *api.Backup
restorerError error
@@ -217,16 +239,18 @@ func TestProcessRestore(t *testing.T) {
}{
{
name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1"},
},
{
name: "restore with resource in both includedResources and excludedResources fails validation",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource"},
@@ -246,11 +270,13 @@ func TestProcessRestore(t *testing.T) {
expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"},
},
{
name: "valid restore with schedule name gets executed",
restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore,
name: "valid restore with schedule name gets executed",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore,
backup: arktest.
NewTestBackup().
WithName("backup-1").
WithStorageLocation("default").
WithLabel("ark-schedule", "sched-1").
WithPhase(api.BackupPhaseCompleted).
Backup,
@@ -263,13 +289,14 @@ func TestProcessRestore(t *testing.T) {
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Restore,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Error retrieving backup: no backup here"},
expectedValidationErrors: []string{"Error retrieving backup: not able to fetch from backup storage"},
backupServiceGetBackupError: errors.New("no backup here"),
},
{
name: "restorer throwing an error causes the restore to fail",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
restorerError: errors.New("blarg"),
expectedErr: false,
expectedPhase: string(api.RestorePhaseInProgress),
@@ -278,16 +305,18 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "valid restore gets executed",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseInProgress),
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Restore,
},
{
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
allowRestoreSnapshots: true,
expectedErr: false,
expectedPhase: string(api.RestorePhaseInProgress),
@@ -295,16 +324,18 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Server is not configured for PV snapshot restores"},
},
{
name: "restoration of nodes is not supported",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{
@@ -314,8 +345,9 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restoration of events is not supported",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{
@@ -325,8 +357,9 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restoration of events.events.k8s.io is not supported",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{
@@ -336,8 +369,9 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restoration of backups.ark.heptio.com is not supported",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.ark.heptio.com", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{
@@ -347,8 +381,9 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restoration of restores.ark.heptio.com is not supported",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.ark.heptio.com", api.RestorePhaseNew).Restore,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{
@@ -358,11 +393,12 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "backup download error results in failed restore",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
expectedPhase: string(api.RestorePhaseInProgress),
expectedFinalPhase: string(api.RestorePhaseFailed),
backupServiceDownloadBackupError: errors.New("Couldn't download backup"),
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
}
@@ -377,6 +413,7 @@ func TestProcessRestore(t *testing.T) {
objectStore = &arktest.ObjectStore{}
)
defer restorer.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
c := NewRestoreController(
@@ -385,23 +422,29 @@ func TestProcessRestore(t *testing.T) {
client.ArkV1(),
client.ArkV1(),
restorer,
api.CloudProviderConfig{Name: "myCloud"},
"bucket",
sharedInformers.Ark().V1().Backups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
test.allowRestoreSnapshots,
logger,
logrus.InfoLevel,
nil, // pluginRegistry
"default",
metrics.NewServerMetrics(),
).(*restoreController)
c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager {
return pluginManager
}
if test.restore != nil {
if test.location != nil {
sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(test.location)
}
if test.backup != nil {
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup)
pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil)
objectStore.On("Init", mock.Anything).Return(nil)
}
if test.restore != nil {
sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(test.restore)
// this is necessary so the Patch() call returns the appropriate object
@@ -590,11 +633,12 @@ func TestProcessRestore(t *testing.T) {
}
}
func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) {
func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) {
var (
client = fake.NewSimpleClientset()
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = arktest.NewLogger()
pluginManager = &pluginmocks.Manager{}
)
c := NewRestoreController(
@@ -603,13 +647,13 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) {
client.ArkV1(),
client.ArkV1(),
nil,
api.CloudProviderConfig{Name: "myCloud"},
"bucket",
sharedInformers.Ark().V1().Backups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
false,
logger,
logrus.DebugLevel,
nil,
"default",
nil,
).(*restoreController)
@@ -632,7 +676,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) {
Backup,
))
errs := c.completeAndValidate(nil, restore)
errs := c.validateAndComplete(restore, pluginManager)
assert.Equal(t, []string{"No backups found for schedule"}, errs)
assert.Empty(t, restore.Spec.BackupName)
@@ -645,7 +689,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) {
Backup,
))
errs = c.completeAndValidate(nil, restore)
errs = c.validateAndComplete(restore, pluginManager)
assert.Equal(t, []string{"No completed backups found for schedule"}, errs)
assert.Empty(t, restore.Spec.BackupName)
@@ -669,7 +713,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) {
Backup,
))
errs = c.completeAndValidate(nil, restore)
errs = c.validateAndComplete(restore, pluginManager)
assert.Nil(t, errs)
assert.Equal(t, "bar", restore.Spec.BackupName)
}

68
pkg/util/test/test_backup_storage_location.go Normal file
View File

@@ -0,0 +1,68 @@
/*
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package test
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/heptio/ark/pkg/apis/ark/v1"
)
type TestBackupStorageLocation struct {
*v1.BackupStorageLocation
}
func NewTestBackupStorageLocation() *TestBackupStorageLocation {
return &TestBackupStorageLocation{
BackupStorageLocation: &v1.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: v1.DefaultNamespace,
},
},
}
}
func (b *TestBackupStorageLocation) WithNamespace(namespace string) *TestBackupStorageLocation {
b.Namespace = namespace
return b
}
func (b *TestBackupStorageLocation) WithName(name string) *TestBackupStorageLocation {
b.Name = name
return b
}
func (b *TestBackupStorageLocation) WithLabel(key, value string) *TestBackupStorageLocation {
if b.Labels == nil {
b.Labels = make(map[string]string)
}
b.Labels[key] = value
return b
}
func (b *TestBackupStorageLocation) WithProvider(name string) *TestBackupStorageLocation {
b.Spec.Provider = name
return b
}
func (b *TestBackupStorageLocation) WithObjectStorage(bucketName string) *TestBackupStorageLocation {
if b.Spec.StorageType.ObjectStorage == nil {
b.Spec.StorageType.ObjectStorage = &v1.ObjectStorageLocation{}
}
b.Spec.ObjectStorage.Bucket = bucketName
return b
}