update sync controller for backup locations

Signed-off-by: Steve Kriss <steve@heptio.com>

pkg/controller/backup_controller.go

@@ -349,6 +349,12 @@ func (controller *backupController) getLocationAndValidate(itm *api.Backup, defa
 		itm.Spec.StorageLocation = defaultBackupLocation
 	}
 
+	// add the storage location as a label for easy filtering later.
+	if itm.Labels == nil {
+		itm.Labels = make(map[string]string)
+	}
+	itm.Labels[api.StorageLocationLabel] = itm.Spec.StorageLocation
+
 	var backupLocation *api.BackupStorageLocation
 	backupLocation, err := controller.backupLocationLister.BackupStorageLocations(itm.Namespace).Get(itm.Spec.StorageLocation)
 	if err != nil {

pkg/controller/backup_controller_test.go

@@ -25,13 +25,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/clock"
 	core "k8s.io/client-go/testing"
 
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
@@ -364,10 +363,14 @@ func TestProcessBackup(t *testing.T) {
 			type SpecPatch struct {
 				StorageLocation string `json:"storageLocation"`
 			}
+			type ObjectMetaPatch struct {
+				Labels map[string]string `json:"labels"`
+			}
 
 			type Patch struct {
-				Status StatusPatch `json:"status"`
-				Spec   SpecPatch   `json:"spec,omitempty"`
+				Status     StatusPatch     `json:"status"`
+				Spec       SpecPatch       `json:"spec,omitempty"`
+				ObjectMeta ObjectMetaPatch `json:"metadata,omitempty"`
 			}
 
 			decode := func(decoder *json.Decoder) (interface{}, error) {
@@ -389,6 +392,11 @@ func TestProcessBackup(t *testing.T) {
 					Spec: SpecPatch{
 						StorageLocation: "default",
 					},
+					ObjectMeta: ObjectMetaPatch{
+						Labels: map[string]string{
+							v1.StorageLocationLabel: "default",
+						},
+					},
 				}
 			} else {
 				expected = Patch{
@@ -397,6 +405,11 @@ func TestProcessBackup(t *testing.T) {
 						Phase:      v1.BackupPhaseInProgress,
 						Expiration: expiration,
 					},
+					ObjectMeta: ObjectMetaPatch{
+						Labels: map[string]string{
+							v1.StorageLocationLabel: test.backup.Spec.StorageLocation,
+						},
+					},
 				}
 			}
 

pkg/controller/backup_sync_controller.go

@@ -17,7 +17,6 @@ limitations under the License.
 package controller
 
 import (
-	"context"
 	"time"
 
 	"github.com/pkg/errors"
@@ -27,133 +26,172 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
 
-	api "github.com/heptio/ark/pkg/apis/ark/v1"
+	arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
 	"github.com/heptio/ark/pkg/cloudprovider"
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/util/kube"
 	"github.com/heptio/ark/pkg/util/stringslice"
 )
 
 type backupSyncController struct {
-	client               arkv1client.BackupsGetter
-	cloudBackupLister    cloudprovider.BackupLister
-	bucket               string
-	syncPeriod           time.Duration
-	namespace            string
-	backupLister         listers.BackupLister
-	backupInformerSynced cache.InformerSynced
-	logger               logrus.FieldLogger
+	*genericController
+
+	client                      arkv1client.BackupsGetter
+	backupLister                listers.BackupLister
+	backupStorageLocationLister listers.BackupStorageLocationLister
+	namespace                   string
+	newPluginManager            func(logrus.FieldLogger) plugin.Manager
+	listCloudBackups            func(logrus.FieldLogger, cloudprovider.ObjectStore, string) ([]*arkv1api.Backup, error)
 }
 
 func NewBackupSyncController(
 	client arkv1client.BackupsGetter,
-	cloudBackupLister cloudprovider.BackupLister,
-	bucket string,
+	backupInformer informers.BackupInformer,
+	backupStorageLocationInformer informers.BackupStorageLocationInformer,
 	syncPeriod time.Duration,
 	namespace string,
-	backupInformer informers.BackupInformer,
+	pluginRegistry plugin.Registry,
 	logger logrus.FieldLogger,
+	logLevel logrus.Level,
 ) Interface {
 	if syncPeriod < time.Minute {
 		logger.Infof("Provided backup sync period %v is too short. Setting to 1 minute", syncPeriod)
 		syncPeriod = time.Minute
 	}
-	return &backupSyncController{
-		client:               client,
-		cloudBackupLister:    cloudBackupLister,
-		bucket:               bucket,
-		syncPeriod:           syncPeriod,
-		namespace:            namespace,
-		backupLister:         backupInformer.Lister(),
-		backupInformerSynced: backupInformer.Informer().HasSynced,
-		logger:               logger,
-	}
-}
 
-// Run is a blocking function that continually runs the object storage -> Ark API
-// sync process according to the controller's syncPeriod. It will return when it
-// receives on the ctx.Done() channel.
-func (c *backupSyncController) Run(ctx context.Context, workers int) error {
-	c.logger.Info("Running backup sync controller")
-	c.logger.Info("Waiting for caches to sync")
-	if !cache.WaitForCacheSync(ctx.Done(), c.backupInformerSynced) {
-		return errors.New("timed out waiting for caches to sync")
+	c := &backupSyncController{
+		genericController:           newGenericController("backup-sync", logger),
+		client:                      client,
+		namespace:                   namespace,
+		backupLister:                backupInformer.Lister(),
+		backupStorageLocationLister: backupStorageLocationInformer.Lister(),
+
+		newPluginManager: func(logger logrus.FieldLogger) plugin.Manager {
+			return plugin.NewManager(logger, logLevel, pluginRegistry)
+		},
+		listCloudBackups: cloudprovider.ListBackups,
 	}
-	c.logger.Info("Caches are synced")
-	wait.Until(c.run, c.syncPeriod, ctx.Done())
-	return nil
+
+	c.resyncFunc = c.run
+	c.resyncPeriod = syncPeriod
+	c.cacheSyncWaiters = []cache.InformerSynced{
+		backupInformer.Informer().HasSynced,
+		backupStorageLocationInformer.Informer().HasSynced,
+	}
+
+	return c
 }
 
 const gcFinalizer = "gc.ark.heptio.com"
 
 func (c *backupSyncController) run() {
-	c.logger.Info("Syncing backups from object storage")
-	backups, err := c.cloudBackupLister.ListBackups(c.bucket)
+	c.logger.Info("Syncing backups from backup storage into cluster")
+
+	locations, err := c.backupStorageLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything())
 	if err != nil {
-		c.logger.WithError(err).Error("error listing backups")
+		c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister")
 		return
 	}
-	c.logger.WithField("backupCount", len(backups)).Info("Got backups from object storage")
 
-	cloudBackupNames := sets.NewString()
-	for _, cloudBackup := range backups {
-		logContext := c.logger.WithField("backup", kube.NamespaceAndName(cloudBackup))
-		logContext.Info("Syncing backup")
+	pluginManager := c.newPluginManager(c.logger)
 
-		cloudBackupNames.Insert(cloudBackup.Name)
+	for _, location := range locations {
+		log := c.logger.WithField("backupLocation", location.Name)
+		log.Info("Syncing backups from backup location")
 
-		// If we're syncing backups made by pre-0.8.0 versions, the server removes all finalizers
-		// faster than the sync finishes. Just process them as we find them.
-		cloudBackup.Finalizers = stringslice.Except(cloudBackup.Finalizers, gcFinalizer)
-
-		cloudBackup.Namespace = c.namespace
-		cloudBackup.ResourceVersion = ""
-
-		// Backup only if backup does not exist in Kubernetes or if we are not able to get the backup for any reason.
-		_, err := c.client.Backups(cloudBackup.Namespace).Get(cloudBackup.Name, metav1.GetOptions{})
+		objectStore, err := getObjectStoreForLocation(location, pluginManager)
 		if err != nil {
+			log.WithError(err).Error("Error getting object store for location")
+			continue
+		}
+
+		backupsInBackupStore, err := c.listCloudBackups(log, objectStore, location.Spec.ObjectStorage.Bucket)
+		if err != nil {
+			log.WithError(err).Error("Error listing backups in object store")
+			continue
+		}
+
+		log.WithField("backupCount", len(backupsInBackupStore)).Info("Got backups from object store")
+
+		cloudBackupNames := sets.NewString()
+		for _, cloudBackup := range backupsInBackupStore {
+			log = log.WithField("backup", kube.NamespaceAndName(cloudBackup))
+			log.Debug("Checking cloud backup to see if it needs to be synced into the cluster")
+
+			cloudBackupNames.Insert(cloudBackup.Name)
+
+			// use the controller's namespace when getting the backup because that's where we
+			// are syncing backups to, regardless of the namespace of the cloud backup.
+			_, err := c.client.Backups(c.namespace).Get(cloudBackup.Name, metav1.GetOptions{})
+			if err == nil {
+				log.Debug("Backup already exists in cluster")
+				continue
+			}
 			if !kuberrs.IsNotFound(err) {
-				logContext.WithError(errors.WithStack(err)).Error("Error getting backup from client, proceeding with backup sync")
+				log.WithError(errors.WithStack(err)).Error("Error getting backup from client, proceeding with sync into cluster")
 			}
 
-			if _, err := c.client.Backups(cloudBackup.Namespace).Create(cloudBackup); err != nil && !kuberrs.IsAlreadyExists(err) {
-				logContext.WithError(errors.WithStack(err)).Error("Error syncing backup from object storage")
+			// remove the pre-v0.8.0 gcFinalizer if it exists
+			// TODO(1.0): remove this
+			cloudBackup.Finalizers = stringslice.Except(cloudBackup.Finalizers, gcFinalizer)
+			cloudBackup.Namespace = c.namespace
+			cloudBackup.ResourceVersion = ""
+
+			// update the StorageLocation field and label since the name of the location
+			// may be different in this cluster than in the cluster that created the
+			// backup.
+			cloudBackup.Spec.StorageLocation = location.Name
+			if cloudBackup.Labels == nil {
+				cloudBackup.Labels = make(map[string]string)
+			}
+			cloudBackup.Labels[arkv1api.StorageLocationLabel] = cloudBackup.Spec.StorageLocation
+
+			_, err = c.client.Backups(cloudBackup.Namespace).Create(cloudBackup)
+			switch {
+			case err != nil && kuberrs.IsAlreadyExists(err):
+				log.Debug("Backup already exists in cluster")
+			case err != nil && !kuberrs.IsAlreadyExists(err):
+				log.WithError(errors.WithStack(err)).Error("Error syncing backup into cluster")
+			default:
+				log.Debug("Synced backup into cluster")
 			}
 		}
-	}
 
-	c.deleteUnused(cloudBackupNames)
-	return
+		c.deleteOrphanedBackups(location.Name, cloudBackupNames, log)
+	}
 }
 
-// deleteUnused deletes backup objects from Kubernetes if they are complete
-// and there is no corresponding backup in the object storage.
-func (c *backupSyncController) deleteUnused(cloudBackupNames sets.String) {
-	// Backups objects in Kubernetes
-	backups, err := c.backupLister.Backups(c.namespace).List(labels.Everything())
+// deleteOrphanedBackups deletes backup objects from Kubernetes that have the specified location
+// and a phase of Completed, but no corresponding backup in object storage.
+func (c *backupSyncController) deleteOrphanedBackups(locationName string, cloudBackupNames sets.String, log logrus.FieldLogger) {
+	locationSelector := labels.Set(map[string]string{
+		arkv1api.StorageLocationLabel: locationName,
+	}).AsSelector()
+
+	backups, err := c.backupLister.Backups(c.namespace).List(locationSelector)
 	if err != nil {
-		c.logger.WithError(errors.WithStack(err)).Error("Error listing backup from Kubernetes")
+		log.WithError(errors.WithStack(err)).Error("Error listing backups from cluster")
+		return
 	}
 	if len(backups) == 0 {
 		return
 	}
 
-	// For each completed backup object in Kubernetes, delete it if it
-	// does not have a corresponding backup in object storage
 	for _, backup := range backups {
-		if backup.Status.Phase == api.BackupPhaseCompleted && !cloudBackupNames.Has(backup.Name) {
-			if err := c.client.Backups(backup.Namespace).Delete(backup.Name, nil); err != nil {
-				c.logger.WithError(errors.WithStack(err)).Error("Error deleting unused backup from Kubernetes")
-			} else {
-				c.logger.Debugf("Deleted backup: %s", backup.Name)
-			}
+		log = log.WithField("backup", backup.Name)
+		if backup.Status.Phase != arkv1api.BackupPhaseCompleted || cloudBackupNames.Has(backup.Name) {
+			continue
+		}
+
+		if err := c.client.Backups(backup.Namespace).Delete(backup.Name, nil); err != nil {
+			log.WithError(errors.WithStack(err)).Error("Error deleting orphaned backup from cluster")
+		} else {
+			log.Debug("Deleted orphaned backup from cluster")
 		}
 	}
-
-	return
 }

pkg/controller/backup_sync_controller_test.go

@@ -20,281 +20,354 @@ import (
 	"testing"
 	"time"
 
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/sirupsen/logrus"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	core "k8s.io/client-go/testing"
 
-	"github.com/heptio/ark/pkg/apis/ark/v1"
-	cloudprovidermocks "github.com/heptio/ark/pkg/cloudprovider/mocks"
+	arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/cloudprovider"
 	"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
+	"github.com/heptio/ark/pkg/plugin"
+	pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
 	"github.com/heptio/ark/pkg/util/stringslice"
 	arktest "github.com/heptio/ark/pkg/util/test"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )
 
+func defaultLocationsList(namespace string) []*arkv1api.BackupStorageLocation {
+	return []*arkv1api.BackupStorageLocation{
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      "location-1",
+			},
+			Spec: arkv1api.BackupStorageLocationSpec{
+				Provider: "objStoreProvider",
+				StorageType: arkv1api.StorageType{
+					ObjectStorage: &arkv1api.ObjectStorageLocation{
+						Bucket: "bucket-1",
+					},
+				},
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      "location-2",
+			},
+			Spec: arkv1api.BackupStorageLocationSpec{
+				Provider: "objStoreProvider",
+				StorageType: arkv1api.StorageType{
+					ObjectStorage: &arkv1api.ObjectStorageLocation{
+						Bucket: "bucket-2",
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestBackupSyncControllerRun(t *testing.T) {
 	tests := []struct {
-		name             string
-		listBackupsError error
-		cloudBackups     []*v1.Backup
-		namespace        string
-		existingBackups  sets.String
+		name            string
+		namespace       string
+		locations       []*arkv1api.BackupStorageLocation
+		cloudBackups    map[string][]*arkv1api.Backup
+		existingBackups []*arkv1api.Backup
 	}{
 		{
 			name: "no cloud backups",
 		},
 		{
-			name:             "backup lister returns error on ListBackups",
-			listBackupsError: errors.New("listBackups"),
-		},
-		{
-			name: "normal case",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:      "normal case",
 			namespace: "ns-1",
+			locations: defaultLocationsList("ns-1"),
+			cloudBackups: map[string][]*arkv1api.Backup{
+				"bucket-1": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
+				},
+				"bucket-2": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
+				},
+			},
 		},
 		{
-			name: "Finalizer gets removed on sync",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers(gcFinalizer).Backup,
-			},
+			name:      "gcFinalizer (only) gets removed on sync",
 			namespace: "ns-1",
+			locations: defaultLocationsList("ns-1"),
+			cloudBackups: map[string][]*arkv1api.Backup{
+				"bucket-1": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers("a-finalizer", gcFinalizer, "some-other-finalizer").Backup,
+				},
+			},
 		},
 		{
-			name: "Only target finalizer is removed",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers(gcFinalizer, "blah").Backup,
-			},
-			namespace: "ns-1",
-		},
-		{
-			name: "backups get created in Ark server's namespace",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-2").WithName("backup-2").Backup,
-			},
+			name:      "all synced backups get created in Ark server's namespace",
 			namespace: "heptio-ark",
+			locations: defaultLocationsList("heptio-ark"),
+			cloudBackups: map[string][]*arkv1api.Backup{
+				"bucket-1": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
+				},
+				"bucket-2": {
+					arktest.NewTestBackup().WithNamespace("ns-2").WithName("backup-3").Backup,
+					arktest.NewTestBackup().WithNamespace("heptio-ark").WithName("backup-4").Backup,
+				},
+			},
 		},
 		{
-			name: "normal case with backups that already exist in Kubernetes",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
+			name:      "new backups get synced when some cloud backups already exist in the cluster",
+			namespace: "ns-1",
+			locations: defaultLocationsList("ns-1"),
+			cloudBackups: map[string][]*arkv1api.Backup{
+				"bucket-1": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
+				},
+				"bucket-2": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-4").Backup,
+				},
+			},
+			existingBackups: []*arkv1api.Backup{
+				// add a label to each existing backup so we can differentiate it from the cloud
+				// backup during verification
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel("i-exist", "true").Backup,
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel("i-exist", "true").Backup,
+			},
+		},
+		{
+			name:      "backup storage location names and labels get updated",
+			namespace: "ns-1",
+			locations: defaultLocationsList("ns-1"),
+			cloudBackups: map[string][]*arkv1api.Backup{
+				"bucket-1": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithStorageLocation("foo").WithLabel(arkv1api.StorageLocationLabel, "foo").Backup,
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
+				},
+				"bucket-2": {
+					arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithStorageLocation("bar").WithLabel(arkv1api.StorageLocationLabel, "bar").Backup,
+				},
 			},
-			existingBackups: sets.NewString("backup-2", "backup-3"),
-			namespace:       "ns-1",
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var (
-				backupLister    = &cloudprovidermocks.BackupLister{}
 				client          = fake.NewSimpleClientset()
 				sharedInformers = informers.NewSharedInformerFactory(client, 0)
-				logger          = arktest.NewLogger()
+				pluginManager   = &pluginmocks.Manager{}
+				objectStore     = &arktest.ObjectStore{}
 			)
 
 			c := NewBackupSyncController(
 				client.ArkV1(),
-				backupLister,
-				"bucket",
+				sharedInformers.Ark().V1().Backups(),
+				sharedInformers.Ark().V1().BackupStorageLocations(),
 				time.Duration(0),
 				test.namespace,
-				sharedInformers.Ark().V1().Backups(),
-				logger,
+				nil, // pluginRegistry
+				arktest.NewLogger(),
+				logrus.DebugLevel,
 			).(*backupSyncController)
 
-			backupLister.On("ListBackups", "bucket").Return(test.cloudBackups, test.listBackupsError)
+			c.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager }
+			pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil)
+			pluginManager.On("CleanupClients").Return(nil)
+			objectStore.On("Init", mock.Anything).Return(nil)
 
-			expectedActions := make([]core.Action, 0)
+			for _, location := range test.locations {
+				require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location))
+			}
 
-			client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) {
-				getAction := action.(core.GetAction)
-				if test.existingBackups.Has(getAction.GetName()) {
-					return true, nil, nil
+			c.listCloudBackups = func(_ logrus.FieldLogger, _ cloudprovider.ObjectStore, bucket string) ([]*arkv1api.Backup, error) {
+				backups, ok := test.cloudBackups[bucket]
+				if !ok {
+					return nil, errors.New("bucket not found")
 				}
-				// We return nil in place of the found backup object because
-				// we exclusively check for the error and don't use the object
-				// returned by the Get / Backups call.
-				return true, nil, apierrors.NewNotFound(v1.SchemeGroupVersion.WithResource("backups").GroupResource(), getAction.GetName())
-			})
+
+				return backups, nil
+			}
+
+			for _, existingBackup := range test.existingBackups {
+				require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(existingBackup))
+
+				_, err := client.ArkV1().Backups(test.namespace).Create(existingBackup)
+				require.NoError(t, err)
+			}
+			client.ClearActions()
 
 			c.run()
 
-			// we only expect creates for items within the target bucket
-			for _, cloudBackup := range test.cloudBackups {
-				// Verify that the run function stripped the GC finalizer
-				assert.False(t, stringslice.Has(cloudBackup.Finalizers, gcFinalizer))
-				assert.Equal(t, test.namespace, cloudBackup.Namespace)
+			for bucket, backups := range test.cloudBackups {
+				for _, cloudBackup := range backups {
+					obj, err := client.ArkV1().Backups(test.namespace).Get(cloudBackup.Name, metav1.GetOptions{})
+					require.NoError(t, err)
 
-				actionGet := core.NewGetAction(
-					v1.SchemeGroupVersion.WithResource("backups"),
-					test.namespace,
-					cloudBackup.Name,
-				)
-				expectedActions = append(expectedActions, actionGet)
+					// did this cloud backup already exist in the cluster?
+					var existing *arkv1api.Backup
+					for _, obj := range test.existingBackups {
+						if obj.Name == cloudBackup.Name {
+							existing = obj
+							break
+						}
+					}
 
-				if test.existingBackups.Has(cloudBackup.Name) {
-					continue
+					if existing != nil {
+						// if this cloud backup already exists in the cluster, make sure that what we get from the
+						// client is the existing backup, not the cloud one.
+						assert.Equal(t, existing, obj)
+					} else {
+						// verify that the GC finalizer is removed
+						assert.Equal(t, stringslice.Except(cloudBackup.Finalizers, gcFinalizer), obj.Finalizers)
+
+						// verify that the storage location field and label are set properly
+						for _, location := range test.locations {
+							if location.Spec.ObjectStorage.Bucket == bucket {
+								assert.Equal(t, location.Name, obj.Spec.StorageLocation)
+								assert.Equal(t, location.Name, obj.Labels[arkv1api.StorageLocationLabel])
+								break
+							}
+						}
+					}
 				}
-				actionCreate := core.NewCreateAction(
-					v1.SchemeGroupVersion.WithResource("backups"),
-					test.namespace,
-					cloudBackup,
-				)
-				expectedActions = append(expectedActions, actionCreate)
 			}
-
-			assert.Equal(t, expectedActions, client.Actions())
 		})
 	}
 }
 
-func TestDeleteUnused(t *testing.T) {
+func TestDeleteOrphanedBackups(t *testing.T) {
 	tests := []struct {
 		name            string
-		cloudBackups    []*v1.Backup
+		cloudBackups    sets.String
 		k8sBackups      []*arktest.TestBackup
 		namespace       string
 		expectedDeletes sets.String
 	}{
 		{
-			name:      "no overlapping backups",
-			namespace: "ns-1",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:         "no overlapping backups",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
 			k8sBackups: []*arktest.TestBackup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupB").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithPhase(v1.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupB").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
 			},
 			expectedDeletes: sets.NewString("backupA", "backupB", "backupC"),
 		},
 		{
-			name:      "some overlapping backups",
-			namespace: "ns-1",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:         "some overlapping backups",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
 			k8sBackups: []*arktest.TestBackup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithPhase(v1.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-C").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
 			},
-			expectedDeletes: sets.NewString("backupC"),
+			expectedDeletes: sets.NewString("backup-C"),
 		},
 		{
-			name:      "all overlapping backups",
-			namespace: "ns-1",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:         "all overlapping backups",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
 			k8sBackups: []*arktest.TestBackup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithPhase(v1.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
 			},
 			expectedDeletes: sets.NewString(),
 		},
 		{
-			name:      "no overlapping backups but including backups that are not complete",
-			namespace: "ns-1",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:         "no overlapping backups but including backups that are not complete",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
 			k8sBackups: []*arktest.TestBackup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithPhase(v1.BackupPhaseCompleted),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("Deleting").WithPhase(v1.BackupPhaseDeleting),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("Failed").WithPhase(v1.BackupPhaseFailed),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("FailedValidation").WithPhase(v1.BackupPhaseFailedValidation),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("InProgress").WithPhase(v1.BackupPhaseInProgress),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("New").WithPhase(v1.BackupPhaseNew),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("Deleting").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseDeleting),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("Failed").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailed),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("FailedValidation").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailedValidation),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("InProgress").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseInProgress),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("New").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseNew),
 			},
 			expectedDeletes: sets.NewString("backupA"),
 		},
 		{
-			name:      "all overlapping backups and all backups that are not complete",
-			namespace: "ns-1",
-			cloudBackups: []*v1.Backup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup,
-			},
+			name:         "all overlapping backups and all backups that are not complete",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
 			k8sBackups: []*arktest.TestBackup{
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseFailed),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseFailedValidation),
-				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithPhase(v1.BackupPhaseInProgress),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailed),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailedValidation),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseInProgress),
 			},
 			expectedDeletes: sets.NewString(),
 		},
+		{
+			name:         "no completed backups in other locations are deleted",
+			namespace:    "ns-1",
+			cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"),
+			k8sBackups: []*arktest.TestBackup{
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-C").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-4").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-5").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted),
+				arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-6").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted),
+			},
+			expectedDeletes: sets.NewString("backup-C"),
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var (
-				backupLister    = &cloudprovidermocks.BackupLister{}
 				client          = fake.NewSimpleClientset()
 				sharedInformers = informers.NewSharedInformerFactory(client, 0)
-				logger          = arktest.NewLogger()
 			)
 
 			c := NewBackupSyncController(
 				client.ArkV1(),
-				backupLister,
-				"bucket",
+				sharedInformers.Ark().V1().Backups(),
+				sharedInformers.Ark().V1().BackupStorageLocations(),
 				time.Duration(0),
 				test.namespace,
-				sharedInformers.Ark().V1().Backups(),
-				logger,
+				nil, // pluginRegistry
+				arktest.NewLogger(),
+				logrus.InfoLevel,
 			).(*backupSyncController)
 
 			expectedDeleteActions := make([]core.Action, 0)
 
-			// setup: insert backups into Kubernetes
 			for _, backup := range test.k8sBackups {
+				// add test backup to informer
+				require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup.Backup), "Error adding backup to informer")
+
+				// add test backup to client
+				_, err := client.Ark().Backups(test.namespace).Create(backup.Backup)
+				require.NoError(t, err, "Error adding backup to clientset")
+
+				// if we expect this backup to be deleted, set up the expected DeleteAction
 				if test.expectedDeletes.Has(backup.Name) {
 					actionDelete := core.NewDeleteAction(
-						v1.SchemeGroupVersion.WithResource("backups"),
+						arkv1api.SchemeGroupVersion.WithResource("backups"),
 						test.namespace,
 						backup.Name,
 					)
 					expectedDeleteActions = append(expectedDeleteActions, actionDelete)
 				}
-
-				// add test backup to informer:
-				err := sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup.Backup)
-				assert.NoError(t, err, "Error adding backup to informer")
-
-				// add test backup to kubernetes:
-				_, err = client.Ark().Backups(test.namespace).Create(backup.Backup)
-				assert.NoError(t, err, "Error deleting from clientset")
 			}
 
-			// get names of client backups
-			testBackupNames := sets.NewString()
-			for _, cloudBackup := range test.cloudBackups {
-				testBackupNames.Insert(cloudBackup.Name)
-			}
-
-			c.deleteUnused(testBackupNames)
+			c.deleteOrphanedBackups("default", test.cloudBackups, arktest.NewLogger())
 
 			numBackups, err := numBackups(t, client, c.namespace)
 			assert.NoError(t, err)