mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-17 12:46:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add a BackupStore to pkg/persistence that supports prefixes

Browse Source 
Signed-off-by: Steve Kriss <steve@heptio.com>
This commit is contained in:
Steve Kriss
2018-09-06 10:53:58 -06:00
parent af64069d65
commit f0edf7335f
28 changed files with 1391 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/backup_controller.go
+8 -34
View File
@@ -42,7 +42,6 @@ import (
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/backup"
"github.com/heptio/ark/pkg/cloudprovider"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
@@ -74,6 +73,7 @@ type backupController struct {
backupLocationListerSynced cache.InformerSynced
defaultBackupLocation string
metrics *metrics.ServerMetrics
newBackupStore func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
func NewBackupController(
@@ -105,6 +105,8 @@ func NewBackupController(
backupLocationListerSynced: backupLocationInformer.Informer().HasSynced,
defaultBackupLocation: defaultBackupLocation,
metrics: metrics,
newBackupStore: persistence.NewObjectBackupStore,
}
c.syncHandler = c.processBackup
@@ -382,21 +384,21 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation
log.Info("Starting backup")
pluginManager := controller.newPluginManager(log)
defer pluginManager.CleanupClients()
backupFile, err := ioutil.TempFile("", "")
if err != nil {
return errors.Wrap(err, "error creating temp file for backup")
}
defer closeAndRemoveFile(backupFile, log)
pluginManager := controller.newPluginManager(log)
defer pluginManager.CleanupClients()
actions, err := pluginManager.GetBackupItemActions()
if err != nil {
return err
}
objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager)
backupStore, err := controller.newBackupStore(backupLocation, pluginManager, log)
if err != nil {
return err
}
@@ -438,7 +440,7 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation
controller.logger.WithError(err).Error("error closing gzippedLogFile")
}
if err := persistence.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil {
if err := backupStore.PutBackup(backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil {
errs = append(errs, err)
}
@@ -454,34 +456,6 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation
return kerrors.NewAggregate(errs)
}
// TODO(ncdc): move this to a better location that isn't backup specific
func getObjectStoreForLocation(location *api.BackupStorageLocation, manager plugin.Manager) (cloudprovider.ObjectStore, error) {
if location.Spec.Provider == "" {
return nil, errors.New("backup storage location provider name must not be empty")
}
objectStore, err := manager.GetObjectStore(location.Spec.Provider)
if err != nil {
return nil, err
}
// add the bucket name to the config map so that object stores can use
// it when initializing. The AWS object store uses this to determine the
// bucket's region when setting up its client.
if location.Spec.ObjectStorage != nil {
if location.Spec.Config == nil {
location.Spec.Config = make(map[string]string)
}
location.Spec.Config["bucket"] = location.Spec.ObjectStorage.Bucket
}
if err := objectStore.Init(location.Spec.Config); err != nil {
return nil, err
}
return objectStore, nil
}
func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) {
if err := file.Close(); err != nil {
log.WithError(err).WithField("file", file.Name()).Error("error closing file")
pkg/controller/backup_controller_test.go
+14 -15
View File
@@ -19,26 +19,27 @@ package controller
import (
"bytes"
"encoding/json"
"fmt"
"io"
"strings"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/clock"
core "k8s.io/client-go/testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/backup"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/metrics"
"github.com/heptio/ark/pkg/persistence"
persistencemocks "github.com/heptio/ark/pkg/persistence/mocks"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
"github.com/heptio/ark/pkg/util/collections"
@@ -179,12 +180,12 @@ func TestProcessBackup(t *testing.T) {
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = logging.DefaultLogger(logrus.DebugLevel)
clockTime, _ = time.Parse("Mon Jan 2 15:04:05 2006", "Mon Jan 2 15:04:05 2006")
objectStore = &arktest.ObjectStore{}
pluginManager = &pluginmocks.Manager{}
backupStore = &persistencemocks.BackupStore{}
)
defer backupper.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
defer pluginManager.AssertExpectations(t)
defer backupStore.AssertExpectations(t)
c := NewBackupController(
sharedInformers.Ark().V1().Backups(),
@@ -202,6 +203,10 @@ func TestProcessBackup(t *testing.T) {
c.clock = clock.NewFakeClock(clockTime)
c.newBackupStore = func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
return backupStore, nil
}
var expiration, startTime time.Time
if test.backup != nil {
@@ -217,9 +222,6 @@ func TestProcessBackup(t *testing.T) {
}
if test.expectBackup {
pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil)
objectStore.On("Init", mock.Anything).Return(nil)
// set up a Backup object to represent what we expect to be passed to backupper.Backup()
backup := test.backup.DeepCopy()
backup.Spec.IncludedResources = test.expectedIncludes
@@ -278,11 +280,8 @@ func TestProcessBackup(t *testing.T) {
return strings.Contains(json, timeString)
}
objectStore.On("PutObject", "bucket", fmt.Sprintf("%s/%s-logs.gz", test.backup.Name, test.backup.Name), mock.Anything).Return(nil)
objectStore.On("PutObject", "bucket", fmt.Sprintf("%s/ark-backup.json", test.backup.Name), mock.MatchedBy(completionTimestampIsPresent)).Return(nil)
objectStore.On("PutObject", "bucket", fmt.Sprintf("%s/%s.tar.gz", test.backup.Name, test.backup.Name), mock.Anything).Return(nil)
pluginManager.On("CleanupClients")
backupStore.On("PutBackup", test.backup.Name, mock.MatchedBy(completionTimestampIsPresent), mock.Anything, mock.Anything).Return(nil)
pluginManager.On("CleanupClients").Return()
}
// this is necessary so the Patch() call returns the appropriate object
pkg/controller/backup_deletion_controller.go
+4 -4
View File
@@ -58,10 +58,10 @@ type backupDeletionController struct {
resticMgr restic.RepositoryManager
podvolumeBackupLister listers.PodVolumeBackupLister
backupLocationLister listers.BackupStorageLocationLister
deleteBackupDir persistence.DeleteBackupDirFunc
processRequestFunc func(*v1.DeleteBackupRequest) error
clock clock.Clock
newPluginManager func(logrus.FieldLogger) plugin.Manager
newBackupStore func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
// NewBackupDeletionController creates a new backup deletion controller.
@@ -95,7 +95,7 @@ func NewBackupDeletionController(
// use variables to refer to these functions so they can be
// replaced with fakes for testing.
newPluginManager: newPluginManager,
deleteBackupDir: persistence.DeleteBackupDir,
newBackupStore: persistence.NewObjectBackupStore,
clock: &clock.RealClock{},
}
@@ -322,12 +322,12 @@ func (c *backupDeletionController) deleteBackupFromStorage(backup *v1.Backup, lo
return errors.WithStack(err)
}
objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager)
backupStore, err := c.newBackupStore(backupLocation, pluginManager, log)
if err != nil {
return err
}
if err := c.deleteBackupDir(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name); err != nil {
if err := backupStore.DeleteBackup(backup.Name); err != nil {
return errors.Wrap(err, "error deleting backup from backup storage")
}
pkg/controller/backup_deletion_controller_test.go
+19 -21
View File
@@ -21,25 +21,27 @@ import (
"testing"
"time"
"github.com/heptio/ark/pkg/apis/ark/v1"
pkgbackup "github.com/heptio/ark/pkg/backup"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
arktest "github.com/heptio/ark/pkg/util/test"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/sets"
core "k8s.io/client-go/testing"
"github.com/heptio/ark/pkg/apis/ark/v1"
pkgbackup "github.com/heptio/ark/pkg/backup"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/persistence"
persistencemocks "github.com/heptio/ark/pkg/persistence/mocks"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
arktest "github.com/heptio/ark/pkg/util/test"
)
func TestBackupDeletionControllerProcessQueueItem(t *testing.T) {
@@ -112,7 +114,7 @@ type backupDeletionControllerTestData struct {
client *fake.Clientset
sharedInformers informers.SharedInformerFactory
blockStore *arktest.FakeBlockStore
objectStore *arktest.ObjectStore
backupStore *persistencemocks.BackupStore
controller *backupDeletionController
req *v1.DeleteBackupRequest
}
@@ -123,7 +125,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio
sharedInformers = informers.NewSharedInformerFactory(client, 0)
blockStore = &arktest.FakeBlockStore{SnapshotsTaken: sets.NewString()}
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
backupStore = &persistencemocks.BackupStore{}
req = pkgbackup.NewDeleteBackupRequest("foo", "uid")
)
@@ -131,7 +133,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio
client: client,
sharedInformers: sharedInformers,
blockStore: blockStore,
objectStore: objectStore,
backupStore: backupStore,
controller: NewBackupDeletionController(
arktest.NewLogger(),
sharedInformers.Ark().V1().DeleteBackupRequests(),
@@ -150,7 +152,10 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio
req: req,
}
pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil)
data.controller.newBackupStore = func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
return backupStore, nil
}
pluginManager.On("CleanupClients").Return(nil)
req.Namespace = "heptio-ark"
@@ -388,8 +393,6 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) {
}
require.NoError(t, td.sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location))
td.objectStore.On("Init", mock.Anything).Return(nil)
// Clear out req labels to make sure the controller adds them
td.req.Labels = make(map[string]string)
@@ -406,12 +409,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) {
return true, backup, nil
})
td.controller.deleteBackupDir = func(_ logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error {
require.NotNil(t, objectStore)
require.Equal(t, location.Spec.ObjectStorage.Bucket, bucket)
require.Equal(t, td.req.Spec.BackupName, backupName)
return nil
}
td.backupStore.On("DeleteBackup", td.req.Spec.BackupName).Return(nil)
err := td.controller.processRequest(td.req)
require.NoError(t, err)
pkg/controller/backup_sync_controller.go
+7 -8
View File
@@ -29,7 +29,6 @@ import (
"k8s.io/client-go/tools/cache"
arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
@@ -48,7 +47,7 @@ type backupSyncController struct {
namespace string
defaultBackupLocation string
newPluginManager func(logrus.FieldLogger) plugin.Manager
listCloudBackups func(logrus.FieldLogger, cloudprovider.ObjectStore, string) ([]*arkv1api.Backup, error)
newBackupStore func(*arkv1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
func NewBackupSyncController(
@@ -77,7 +76,7 @@ func NewBackupSyncController(
// use variables to refer to these functions so they can be
// replaced with fakes for testing.
newPluginManager: newPluginManager,
listCloudBackups: persistence.ListBackups,
newBackupStore: persistence.NewObjectBackupStore,
}
c.resyncFunc = c.run
@@ -109,19 +108,19 @@ func (c *backupSyncController) run() {
log := c.logger.WithField("backupLocation", location.Name)
log.Info("Syncing backups from backup location")
objectStore, err := getObjectStoreForLocation(location, pluginManager)
backupStore, err := c.newBackupStore(location, pluginManager, log)
if err != nil {
log.WithError(err).Error("Error getting object store for location")
log.WithError(err).Error("Error getting backup store for location")
continue
}
backupsInBackupStore, err := c.listCloudBackups(log, objectStore, location.Spec.ObjectStorage.Bucket)
backupsInBackupStore, err := backupStore.ListBackups()
if err != nil {
log.WithError(err).Error("Error listing backups in object store")
log.WithError(err).Error("Error listing backups in backup store")
continue
}
log.WithField("backupCount", len(backupsInBackupStore)).Info("Got backups from object store")
log.WithField("backupCount", len(backupsInBackupStore)).Info("Got backups from backup store")
cloudBackupNames := sets.NewString()
for _, cloudBackup := range backupsInBackupStore {
pkg/controller/backup_sync_controller_test.go
+15 -16
View File
@@ -20,25 +20,23 @@ import (
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
core "k8s.io/client-go/testing"
arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/persistence"
persistencemocks "github.com/heptio/ark/pkg/persistence/mocks"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
"github.com/heptio/ark/pkg/util/stringslice"
arktest "github.com/heptio/ark/pkg/util/test"
"github.com/stretchr/testify/assert"
)
func defaultLocationsList(namespace string) []*arkv1api.BackupStorageLocation {
@@ -167,7 +165,7 @@ func TestBackupSyncControllerRun(t *testing.T) {
client = fake.NewSimpleClientset()
sharedInformers = informers.NewSharedInformerFactory(client, 0)
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
backupStores = make(map[string]*persistencemocks.BackupStore)
)
c := NewBackupSyncController(
@@ -181,22 +179,23 @@ func TestBackupSyncControllerRun(t *testing.T) {
arktest.NewLogger(),
).(*backupSyncController)
pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil)
pluginManager.On("CleanupClients").Return(nil)
c.newBackupStore = func(loc *arkv1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) {
// this gets populated just below, prior to exercising the method under test
return backupStores[loc.Name], nil
}
objectStore.On("Init", mock.Anything).Return(nil)
pluginManager.On("CleanupClients").Return(nil)
for _, location := range test.locations {
require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location))
backupStores[location.Name] = &persistencemocks.BackupStore{}
}
c.listCloudBackups = func(_ logrus.FieldLogger, _ cloudprovider.ObjectStore, bucket string) ([]*arkv1api.Backup, error) {
backups, ok := test.cloudBackups[bucket]
if !ok {
return nil, errors.New("bucket not found")
}
for _, location := range test.locations {
backupStore, ok := backupStores[location.Name]
require.True(t, ok, "no mock backup store for location %s", location.Name)
return backups, nil
backupStore.On("ListBackups").Return(test.cloudBackups[location.Spec.ObjectStorage.Bucket], nil)
}
for _, existingBackup := range test.existingBackups {
pkg/controller/download_request_controller.go
+10 -11
View File
@@ -47,10 +47,10 @@ type downloadRequestController struct {
downloadRequestLister listers.DownloadRequestLister
restoreLister listers.RestoreLister
clock clock.Clock
createSignedURL persistence.CreateSignedURLFunc
backupLocationLister listers.BackupStorageLocationLister
backupLister listers.BackupLister
newPluginManager func(logrus.FieldLogger) plugin.Manager
newBackupStore func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
// NewDownloadRequestController creates a new DownloadRequestController.
@@ -73,8 +73,8 @@ func NewDownloadRequestController(
// use variables to refer to these functions so they can be
// replaced with fakes for testing.
createSignedURL: persistence.CreateSignedURL,
newPluginManager: newPluginManager,
newBackupStore: persistence.NewObjectBackupStore,
clock: &clock.RealClock{},
}
@@ -146,8 +146,8 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow
update := downloadRequest.DeepCopy()
var (
directory string
err error
backupName string
err error
)
switch downloadRequest.Spec.Target.Kind {
@@ -157,12 +157,12 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow
return errors.Wrap(err, "error getting Restore")
}
directory = restore.Spec.BackupName
backupName = restore.Spec.BackupName
default:
directory = downloadRequest.Spec.Target.Name
backupName = downloadRequest.Spec.Target.Name
}
backup, err := c.backupLister.Backups(downloadRequest.Namespace).Get(directory)
backup, err := c.backupLister.Backups(downloadRequest.Namespace).Get(backupName)
if err != nil {
return errors.WithStack(err)
}
@@ -175,18 +175,17 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow
pluginManager := c.newPluginManager(log)
defer pluginManager.CleanupClients()
objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager)
backupStore, err := c.newBackupStore(backupLocation, pluginManager, log)
if err != nil {
return errors.WithStack(err)
}
update.Status.DownloadURL, err = c.createSignedURL(objectStore, downloadRequest.Spec.Target, backupLocation.Spec.ObjectStorage.Bucket, directory, signedURLTTL)
if err != nil {
if update.Status.DownloadURL, err = backupStore.GetDownloadURL(backupName, downloadRequest.Spec.Target); err != nil {
return err
}
update.Status.Phase = v1.DownloadRequestPhaseProcessed
update.Status.Expiration = metav1.NewTime(c.clock.Now().Add(signedURLTTL))
update.Status.Expiration = metav1.NewTime(c.clock.Now().Add(persistence.DownloadURLTTL))
_, err = patchDownloadRequest(downloadRequest, update, c.downloadRequestClient)
return errors.WithStack(err)
pkg/controller/download_request_controller_test.go
+65 -64
View File
@@ -22,7 +22,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -32,6 +31,8 @@ import (
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/persistence"
persistencemocks "github.com/heptio/ark/pkg/persistence/mocks"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
kubeutil "github.com/heptio/ark/pkg/util/kube"
@@ -42,7 +43,7 @@ type downloadRequestTestHarness struct {
client *fake.Clientset
informerFactory informers.SharedInformerFactory
pluginManager *pluginmocks.Manager
objectStore *arktest.ObjectStore
backupStore *persistencemocks.BackupStore
controller *downloadRequestController
}
@@ -52,7 +53,7 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness {
client = fake.NewSimpleClientset()
informerFactory = informers.NewSharedInformerFactory(client, 0)
pluginManager = new(pluginmocks.Manager)
objectStore = new(arktest.ObjectStore)
backupStore = new(persistencemocks.BackupStore)
controller = NewDownloadRequestController(
client.ArkV1(),
informerFactory.Ark().V1().DownloadRequests(),
@@ -66,17 +67,19 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness {
clockTime, err := time.Parse(time.RFC1123, time.RFC1123)
require.NoError(t, err)
controller.clock = clock.NewFakeClock(clockTime)
controller.newBackupStore = func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
return backupStore, nil
}
pluginManager.On("CleanupClients").Return()
objectStore.On("Init", mock.Anything).Return(nil)
return &downloadRequestTestHarness{
client: client,
informerFactory: informerFactory,
pluginManager: pluginManager,
objectStore: objectStore,
backupStore: backupStore,
controller: controller,
}
}
@@ -118,15 +121,15 @@ func newBackupLocation(name, provider, bucket string) *v1.BackupStorageLocation
func TestProcessDownloadRequest(t *testing.T) {
tests := []struct {
name string
key string
downloadRequest *v1.DownloadRequest
backup *v1.Backup
restore *v1.Restore
backupLocation *v1.BackupStorageLocation
expired bool
expectedErr string
expectedRequestedObject string
name string
key string
downloadRequest *v1.DownloadRequest
backup *v1.Backup
restore *v1.Restore
backupLocation *v1.BackupStorageLocation
expired bool
expectedErr string
expectGetsURL bool
}{
{
name: "empty key returns without error",
@@ -163,64 +166,64 @@ func TestProcessDownloadRequest(t *testing.T) {
expectedErr: "backupstoragelocation.ark.heptio.com \"a-location\" not found",
},
{
name: "backup contents request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/a-backup.tar.gz",
name: "backup contents request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "backup contents request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupContents, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/a-backup.tar.gz",
name: "backup contents request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupContents, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "backup log request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupLog, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/a-backup-logs.gz",
name: "backup log request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupLog, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "backup log request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupLog, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/a-backup-logs.gz",
name: "backup log request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupLog, "a-backup"),
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "restore log request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-logs.gz",
name: "restore log request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "restore log request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-logs.gz",
name: "restore log request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "restore results request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-results.gz",
name: "restore results request with phase '' gets a url",
downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "restore results request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-results.gz",
name: "restore results request with phase 'New' gets a url",
downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"),
restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore,
backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup,
backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"),
expectGetsURL: true,
},
{
name: "request with phase 'Processed' is not deleted if not expired",
@@ -268,12 +271,10 @@ func TestProcessDownloadRequest(t *testing.T) {
if tc.backupLocation != nil {
require.NoError(t, harness.informerFactory.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(tc.backupLocation))
harness.pluginManager.On("GetObjectStore", tc.backupLocation.Spec.Provider).Return(harness.objectStore, nil)
}
if tc.expectedRequestedObject != "" {
harness.objectStore.On("CreateSignedURL", tc.backupLocation.Spec.ObjectStorage.Bucket, tc.expectedRequestedObject, mock.Anything).Return("a-url", nil)
if tc.expectGetsURL {
harness.backupStore.On("GetDownloadURL", tc.backup.Name, tc.downloadRequest.Spec.Target).Return("a-url", nil)
}
// exercise method under test
@@ -291,7 +292,7 @@ func TestProcessDownloadRequest(t *testing.T) {
assert.Nil(t, err)
}
if tc.expectedRequestedObject != "" {
if tc.expectGetsURL {
output, err := harness.client.ArkV1().DownloadRequests(tc.downloadRequest.Namespace).Get(tc.downloadRequest.Name, metav1.GetOptions{})
require.NoError(t, err)
pkg/controller/restore_controller.go
+24 -34
View File
@@ -41,7 +41,6 @@ import (
"k8s.io/client-go/util/workqueue"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
@@ -90,11 +89,8 @@ type restoreController struct {
defaultBackupLocation string
metrics *metrics.ServerMetrics
getBackup persistence.GetBackupFunc
downloadBackup persistence.DownloadBackupFunc
uploadRestoreLog persistence.UploadRestoreLogFunc
uploadRestoreResults persistence.UploadRestoreResultsFunc
newPluginManager func(logger logrus.FieldLogger) plugin.Manager
newPluginManager func(logger logrus.FieldLogger) plugin.Manager
newBackupStore func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
func NewRestoreController(
@@ -132,11 +128,8 @@ func NewRestoreController(
// use variables to refer to these functions so they can be
// replaced with fakes for testing.
newPluginManager: newPluginManager,
getBackup: persistence.GetBackup,
downloadBackup: persistence.DownloadBackup,
uploadRestoreLog: persistence.UploadRestoreLog,
uploadRestoreResults: persistence.UploadRestoreResults,
newPluginManager: newPluginManager,
newBackupStore: persistence.NewObjectBackupStore,
}
c.syncHandler = c.processRestore
@@ -354,9 +347,8 @@ func (c *restoreController) processRestore(key string) error {
}
type backupInfo struct {
bucketName string
backup *api.Backup
objectStore cloudprovider.ObjectStore
backupStore persistence.BackupStore
}
func (c *restoreController) validateAndComplete(restore *api.Restore, pluginManager plugin.Manager) backupInfo {
@@ -469,9 +461,7 @@ func mostRecentCompletedBackup(backups []*api.Backup) *api.Backup {
// fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't
// find it, it tries to retrieve it from one of the backup storage locations.
func (c *restoreController) fetchBackupInfo(backupName string, pluginManager plugin.Manager) (backupInfo, error) {
var info backupInfo
var err error
info.backup, err = c.backupLister.Backups(c.namespace).Get(backupName)
backup, err := c.backupLister.Backups(c.namespace).Get(backupName)
if err != nil {
if !apierrors.IsNotFound(err) {
return backupInfo{}, errors.WithStack(err)
@@ -482,18 +472,20 @@ func (c *restoreController) fetchBackupInfo(backupName string, pluginManager plu
return c.fetchFromBackupStorage(backupName, pluginManager)
}
location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(info.backup.Spec.StorageLocation)
location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(backup.Spec.StorageLocation)
if err != nil {
return backupInfo{}, errors.WithStack(err)
}
info.objectStore, err = getObjectStoreForLocation(location, pluginManager)
backupStore, err := c.newBackupStore(location, pluginManager, c.logger)
if err != nil {
return backupInfo{}, errors.Wrap(err, "error initializing object store")
return backupInfo{}, err
}
info.bucketName = location.Spec.ObjectStorage.Bucket
return info, nil
return backupInfo{
backup: backup,
backupStore: backupStore,
}, nil
}
// fetchFromBackupStorage checks each backup storage location, starting with the default,
@@ -541,12 +533,12 @@ func orderedBackupLocations(locations []*api.BackupStorageLocation, defaultLocat
}
func (c *restoreController) backupInfoForLocation(location *api.BackupStorageLocation, backupName string, pluginManager plugin.Manager) (backupInfo, error) {
objectStore, err := getObjectStoreForLocation(location, pluginManager)
backupStore, err := persistence.NewObjectBackupStore(location, pluginManager, c.logger)
if err != nil {
return backupInfo{}, err
}
backup, err := c.getBackup(objectStore, location.Spec.ObjectStorage.Bucket, backupName)
backup, err := backupStore.GetBackupMetadata(backupName)
if err != nil {
return backupInfo{}, err
}
@@ -562,9 +554,8 @@ func (c *restoreController) backupInfoForLocation(location *api.BackupStorageLoc
}
return backupInfo{
bucketName: location.Spec.ObjectStorage.Bucket,
backup: backupCreated,
objectStore: objectStore,
backupStore: backupStore,
}, nil
}
@@ -603,7 +594,7 @@ func (c *restoreController) runRestore(
"backup": restore.Spec.BackupName,
})
backupFile, err := downloadToTempFile(info.objectStore, info.bucketName, restore.Spec.BackupName, c.downloadBackup, c.logger)
backupFile, err := downloadToTempFile(restore.Spec.BackupName, info.backupStore, c.logger)
if err != nil {
logContext.WithError(err).Error("Error downloading backup")
restoreErrors.Ark = append(restoreErrors.Ark, err.Error())
@@ -637,8 +628,8 @@ func (c *restoreController) runRestore(
return
}
if err := c.uploadRestoreLog(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, logFile); err != nil {
restoreErrors.Ark = append(restoreErrors.Ark, fmt.Sprintf("error uploading log file to object storage: %v", err))
if err := info.backupStore.PutRestoreLog(restore.Spec.BackupName, restore.Name, logFile); err != nil {
restoreErrors.Ark = append(restoreErrors.Ark, fmt.Sprintf("error uploading log file to backup storage: %v", err))
}
m := map[string]api.RestoreResult{
@@ -658,20 +649,19 @@ func (c *restoreController) runRestore(
logContext.WithError(errors.WithStack(err)).Error("Error resetting results file offset to 0")
return
}
if err := c.uploadRestoreResults(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, resultsFile); err != nil {
logContext.WithError(errors.WithStack(err)).Error("Error uploading results files to object storage")
if err := info.backupStore.PutRestoreResults(restore.Spec.BackupName, restore.Name, resultsFile); err != nil {
logContext.WithError(errors.WithStack(err)).Error("Error uploading results file to backup storage")
}
return
}
func downloadToTempFile(
objectStore cloudprovider.ObjectStore,
bucket, backupName string,
downloadBackup persistence.DownloadBackupFunc,
backupName string,
backupStore persistence.BackupStore,
logger logrus.FieldLogger,
) (*os.File, error) {
readCloser, err := downloadBackup(objectStore, bucket, backupName)
readCloser, err := backupStore.GetBackupContents(backupName)
if err != nil {
return nil, err
}
pkg/controller/restore_controller_test.go
+88 -99
View File
@@ -29,16 +29,18 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
core "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/metrics"
"github.com/heptio/ark/pkg/persistence"
persistencemocks "github.com/heptio/ark/pkg/persistence/mocks"
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
"github.com/heptio/ark/pkg/restore"
@@ -48,14 +50,14 @@ import (
func TestFetchBackupInfo(t *testing.T) {
tests := []struct {
name string
backupName string
informerLocations []*api.BackupStorageLocation
informerBackups []*api.Backup
backupServiceBackup *api.Backup
backupServiceError error
expectedRes *api.Backup
expectedErr bool
name string
backupName string
informerLocations []*api.BackupStorageLocation
informerBackups []*api.Backup
backupStoreBackup *api.Backup
backupStoreError error
expectedRes *api.Backup
expectedErr bool
}{
{
name: "lister has backup",
@@ -65,18 +67,18 @@ func TestFetchBackupInfo(t *testing.T) {
expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
{
name: "lister does not have a backup, but backupSvc does",
backupName: "backup-1",
backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation},
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
name: "lister does not have a backup, but backupSvc does",
backupName: "backup-1",
backupStoreBackup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation},
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
{
name: "no backup",
backupName: "backup-1",
backupServiceError: errors.New("no backup here"),
expectedErr: true,
name: "no backup",
backupName: "backup-1",
backupStoreError: errors.New("no backup here"),
expectedErr: true,
},
}
@@ -88,11 +90,11 @@ func TestFetchBackupInfo(t *testing.T) {
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = arktest.NewLogger()
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
backupStore = &persistencemocks.BackupStore{}
)
defer restorer.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
defer backupStore.AssertExpectations(t)
c := NewRestoreController(
api.DefaultNamespace,
@@ -110,10 +112,11 @@ func TestFetchBackupInfo(t *testing.T) {
metrics.NewServerMetrics(),
).(*restoreController)
if test.backupServiceError == nil {
pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil)
objectStore.On("Init", mock.Anything).Return(nil)
c.newBackupStore = func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
return backupStore, nil
}
if test.backupStoreError == nil {
for _, itm := range test.informerLocations {
sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(itm)
}
@@ -123,19 +126,23 @@ func TestFetchBackupInfo(t *testing.T) {
}
}
if test.backupServiceBackup != nil || test.backupServiceError != nil {
c.getBackup = func(_ cloudprovider.ObjectStore, bucket, backup string) (*api.Backup, error) {
require.Equal(t, "bucket", bucket)
require.Equal(t, test.backupName, backup)
return test.backupServiceBackup, test.backupServiceError
}
if test.backupStoreBackup != nil && test.backupStoreError != nil {
panic("developer error - only one of backupStoreBackup, backupStoreError can be non-nil")
}
if test.backupStoreError != nil {
// TODO why do I need .Maybe() here?
backupStore.On("GetBackupMetadata", test.backupName).Return(nil, test.backupStoreError).Maybe()
}
if test.backupStoreBackup != nil {
// TODO why do I need .Maybe() here?
backupStore.On("GetBackupMetadata", test.backupName).Return(test.backupStoreBackup, nil).Maybe()
}
info, err := c.fetchBackupInfo(test.backupName, pluginManager)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, info.backup)
}
require.Equal(t, test.expectedErr, err != nil)
assert.Equal(t, test.expectedRes, info.backup)
})
}
}
@@ -180,11 +187,7 @@ func TestProcessRestoreSkips(t *testing.T) {
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = arktest.NewLogger()
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
)
defer restorer.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
c := NewRestoreController(
api.DefaultNamespace,
@@ -197,7 +200,7 @@ func TestProcessRestoreSkips(t *testing.T) {
false, // pvProviderExists
logger,
logrus.InfoLevel,
func(logrus.FieldLogger) plugin.Manager { return pluginManager },
nil,
"default",
metrics.NewServerMetrics(),
).(*restoreController)
@@ -207,6 +210,7 @@ func TestProcessRestoreSkips(t *testing.T) {
}
err := c.processRestore(test.restoreKey)
assert.Equal(t, test.expectError, err != nil)
})
}
@@ -214,22 +218,22 @@ func TestProcessRestoreSkips(t *testing.T) {
func TestProcessRestore(t *testing.T) {
tests := []struct {
name string
restoreKey string
location *api.BackupStorageLocation
restore *api.Restore
backup *api.Backup
restorerError error
allowRestoreSnapshots bool
expectedErr bool
expectedPhase string
expectedValidationErrors []string
expectedRestoreErrors int
expectedRestorerCall *api.Restore
backupServiceGetBackupError error
uploadLogError error
backupServiceDownloadBackupError error
expectedFinalPhase string
name string
restoreKey string
location *api.BackupStorageLocation
restore *api.Restore
backup *api.Backup
restorerError error
allowRestoreSnapshots bool
expectedErr bool
expectedPhase string
expectedValidationErrors []string
expectedRestoreErrors int
expectedRestorerCall *api.Restore
backupStoreGetBackupMetadataErr error
backupStoreGetBackupContentsErr error
putRestoreLogErr error
expectedFinalPhase string
}{
{
name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation",
@@ -279,12 +283,12 @@ func TestProcessRestore(t *testing.T) {
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).WithSchedule("sched-1").Restore,
},
{
name: "restore with non-existent backup name fails",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Restore,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Error retrieving backup: not able to fetch from backup storage"},
backupServiceGetBackupError: errors.New("no backup here"),
name: "restore with non-existent backup name fails",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Restore,
expectedErr: false,
expectedPhase: string(api.RestorePhaseFailedValidation),
expectedValidationErrors: []string{"Error retrieving backup: not able to fetch from backup storage"},
backupStoreGetBackupMetadataErr: errors.New("no backup here"),
},
{
name: "restorer throwing an error causes the restore to fail",
@@ -386,12 +390,12 @@ func TestProcessRestore(t *testing.T) {
},
},
{
name: "backup download error results in failed restore",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
expectedPhase: string(api.RestorePhaseInProgress),
expectedFinalPhase: string(api.RestorePhaseFailed),
backupServiceDownloadBackupError: errors.New("Couldn't download backup"),
name: "backup download error results in failed restore",
location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation,
restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
expectedPhase: string(api.RestorePhaseInProgress),
expectedFinalPhase: string(api.RestorePhaseFailed),
backupStoreGetBackupContentsErr: errors.New("Couldn't download backup"),
backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup,
},
}
@@ -404,11 +408,11 @@ func TestProcessRestore(t *testing.T) {
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = arktest.NewLogger()
pluginManager = &pluginmocks.Manager{}
objectStore = &arktest.ObjectStore{}
backupStore = &persistencemocks.BackupStore{}
)
defer restorer.AssertExpectations(t)
defer objectStore.AssertExpectations(t)
defer restorer.AssertExpectations(t)
defer backupStore.AssertExpectations(t)
c := NewRestoreController(
api.DefaultNamespace,
@@ -426,13 +430,15 @@ func TestProcessRestore(t *testing.T) {
metrics.NewServerMetrics(),
).(*restoreController)
c.newBackupStore = func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) {
return backupStore, nil
}
if test.location != nil {
sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(test.location)
}
if test.backup != nil {
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup)
pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil)
objectStore.On("Init", mock.Anything).Return(nil)
}
if test.restore != nil {
@@ -481,28 +487,17 @@ func TestProcessRestore(t *testing.T) {
if test.restorerError != nil {
errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}}
}
if test.uploadLogError != nil {
errors.Ark = append(errors.Ark, "error uploading log file to object storage: "+test.uploadLogError.Error())
if test.putRestoreLogErr != nil {
errors.Ark = append(errors.Ark, "error uploading log file to object storage: "+test.putRestoreLogErr.Error())
}
if test.expectedRestorerCall != nil {
c.downloadBackup = func(objectStore cloudprovider.ObjectStore, bucket, backup string) (io.ReadCloser, error) {
require.Equal(t, test.backup.Name, backup)
return ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil
}
backupStore.On("GetBackupContents", test.backup.Name).Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil)
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
c.uploadRestoreLog = func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error {
require.Equal(t, test.backup.Name, backup)
require.Equal(t, test.restore.Name, restore)
return test.uploadLogError
}
backupStore.On("PutRestoreLog", test.backup.Name, test.restore.Name, mock.Anything).Return(test.putRestoreLogErr)
c.uploadRestoreResults = func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error {
require.Equal(t, test.backup.Name, backup)
require.Equal(t, test.restore.Name, restore)
return nil
}
backupStore.On("PutRestoreResults", test.backup.Name, test.restore.Name, mock.Anything).Return(nil)
}
var (
@@ -516,20 +511,14 @@ func TestProcessRestore(t *testing.T) {
}
}
if test.backupServiceGetBackupError != nil {
c.getBackup = func(_ cloudprovider.ObjectStore, bucket, backup string) (*api.Backup, error) {
require.Equal(t, "bucket", bucket)
require.Equal(t, test.restore.Spec.BackupName, backup)
return nil, test.backupServiceGetBackupError
}
if test.backupStoreGetBackupMetadataErr != nil {
// TODO why do I need .Maybe() here?
backupStore.On("GetBackupMetadata", test.restore.Spec.BackupName).Return(nil, test.backupStoreGetBackupMetadataErr).Maybe()
}
if test.backupServiceDownloadBackupError != nil {
c.downloadBackup = func(_ cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error) {
require.Equal(t, "bucket", bucket)
require.Equal(t, test.restore.Spec.BackupName, backupName)
return nil, test.backupServiceDownloadBackupError
}
if test.backupStoreGetBackupContentsErr != nil {
// TODO why do I need .Maybe() here?
backupStore.On("GetBackupContents", test.restore.Spec.BackupName).Return(nil, test.backupStoreGetBackupContentsErr).Maybe()
}
if test.restore != nil {