Add a BSL controller to handle validation + update BSL status phase (#2674)

* Add BSL controller Signed-off-by: Carlisia <carlisia@vmware.com> * Add changelog Signed-off-by: Carlisia <carlisia@vmware.com> * Make update Signed-off-by: Carlisia <carlisia@vmware.com> * Update docs Signed-off-by: Carlisia <carlisia@vmware.com> * Add kubebuilder dependency Signed-off-by: Carlisia <carlisia@vmware.com> * Add export Signed-off-by: Carlisia <carlisia@vmware.com> * add kubebuilder binaries into velero builder image Signed-off-by: Ashish Amarnath <ashisham@vmware.com> * Reset velero dockerfile Signed-off-by: Carlisia <carlisia@vmware.com> * Consolidate all logic Signed-off-by: Carlisia <carlisia@vmware.com> * Add copyright header Signed-off-by: Carlisia <carlisia@vmware.com> * Clean up + add "last validated" column Signed-off-by: Carlisia <carlisia@vmware.com> * Better tests Signed-off-by: Carlisia <carlisia@vmware.com> * Add more tests Signed-off-by: Carlisia <carlisia@vmware.com> * Better logging Signed-off-by: Carlisia <carlisia@vmware.com> * Format Signed-off-by: Carlisia <carlisia@vmware.com> * Code reviews Signed-off-by: Carlisia <carlisia@vmware.com> * Address code review Signed-off-by: Carlisia <carlisia@vmware.com> * Remove redundant logic Signed-off-by: Carlisia <carlisia@vmware.com> Co-authored-by: Ashish Amarnath <ashisham@vmware.com>
2026-09-04 15:16:58 +00:00 · 2020-07-14 17:47:00 -04:00
parent 3d3b9e312a
commit dbd0aa4915
19 changed files with 1015 additions and 92 deletions
@@ -40,6 +40,11 @@ type BackupStorageLocationSpec struct {
 	// +optional
 	// +nullable
 	BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"`
+
+	// ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation.
+	// +optional
+	// +nullable
+	ValidationFrequency *metav1.Duration `json:"validationFrequency,omitempty"`
 }

 // BackupStorageLocationStatus defines the observed state of BackupStorageLocation
@@ -54,6 +59,12 @@ type BackupStorageLocationStatus struct {
 	// +nullable
 	LastSyncedTime *metav1.Time `json:"lastSyncedTime,omitempty"`

+	// LastValidationTime is the last time the backup store location was validated
+	// the cluster.
+	// +optional
+	// +nullable
+	LastValidationTime *metav1.Time `json:"lastValidationTime,omitempty"`
+
 	// LastSyncedRevision is the value of the `metadata/revision` file in the backup
 	// storage location the last time the BSL's contents were synced into the cluster.
 	//
@@ -79,6 +90,7 @@ type BackupStorageLocationStatus struct {
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Backup Storage Location status such as Available/Unavailable"
+// +kubebuilder:printcolumn:name="Last Validated",type="date",JSONPath=".status.lastValidationTime",description="LastValidationTime is the last time the backup store location was validated"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"

 // BackupStorageLocation is a location where Velero stores backup objects
@@ -94,6 +106,8 @@ type BackupStorageLocation struct {
 // the k8s:deepcopy marker will no longer be needed and should be removed.
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:object:root=true
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch

 // BackupStorageLocationList contains a list of BackupStorageLocation
 type BackupStorageLocationList struct {
@@ -124,6 +138,7 @@ type ObjectStorageLocation struct {

 // BackupStorageLocationPhase is the lifecycle phase of a Velero BackupStorageLocation.
 // +kubebuilder:validation:Enum=Available;Unavailable
+// +kubebuilder:default=Unavailable
 type BackupStorageLocationPhase string

 const (
@@ -379,6 +379,11 @@ func (in *BackupStorageLocationSpec) DeepCopyInto(out *BackupStorageLocationSpec
 		*out = new(metav1.Duration)
 		**out = **in
 	}
+	if in.ValidationFrequency != nil {
+		in, out := &in.ValidationFrequency, &out.ValidationFrequency
+		*out = new(metav1.Duration)
+		**out = **in
+	}
 	return
 }

@@ -399,6 +404,10 @@ func (in *BackupStorageLocationStatus) DeepCopyInto(out *BackupStorageLocationSt
 		in, out := &in.LastSyncedTime, &out.LastSyncedTime
 		*out = (*in).DeepCopy()
 	}
+	if in.LastValidationTime != nil {
+		in, out := &in.LastValidationTime, &out.LastValidationTime
+		*out = (*in).DeepCopy()
+	}
 	return
 }

@@ -17,6 +17,8 @@ limitations under the License.
 package builder

 import (
+	"time"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -86,3 +88,21 @@ func (b *BackupStorageLocationBuilder) AccessMode(accessMode velerov1api.BackupS
 	b.object.Spec.AccessMode = accessMode
 	return b
 }
+
+// ValidationFrequency sets the BackupStorageLocation's validation frequency.
+func (b *BackupStorageLocationBuilder) ValidationFrequency(frequency time.Duration) *BackupStorageLocationBuilder {
+	b.object.Spec.ValidationFrequency = &metav1.Duration{Duration: frequency}
+	return b
+}
+
+// LastValidationTime sets the BackupStorageLocation's last validated time.
+func (b *BackupStorageLocationBuilder) LastValidationTime(lastValidated time.Time) *BackupStorageLocationBuilder {
+	b.object.Status.LastValidationTime = &metav1.Time{Time: lastValidated}
+	return b
+}
+
+// Phase sets the BackupStorageLocation's status phase.
+func (b *BackupStorageLocationBuilder) Phase(phase velerov1api.BackupStorageLocationPhase) *BackupStorageLocationBuilder {
+	b.object.Status.Phase = phase
+	return b
+}
@@ -58,14 +58,14 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command {
 }

 type CreateOptions struct {
-	Name             string
-	Provider         string
-	Bucket           string
-	Prefix           string
-	BackupSyncPeriod time.Duration
-	Config           flag.Map
-	Labels           flag.Map
-	AccessMode       *flag.Enum
+	Name                                  string
+	Provider                              string
+	Bucket                                string
+	Prefix                                string
+	BackupSyncPeriod, ValidationFrequency time.Duration
+	Config                                flag.Map
+	Labels                                flag.Map
+	AccessMode                            *flag.Enum
 }

 func NewCreateOptions() *CreateOptions {
@@ -83,7 +83,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.Provider, "provider", o.Provider, "name of the backup storage provider (e.g. aws, azure, gcp)")
 	flags.StringVar(&o.Bucket, "bucket", o.Bucket, "name of the object storage bucket where backups should be stored")
 	flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Velero data should be stored within the bucket. Optional.")
-	flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync")
+	flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync. Default: 1 minute.")
+	flags.DurationVar(&o.ValidationFrequency, "validation-frequency", o.ValidationFrequency, "how often to verify if the backup storage location is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.")
 	flags.Var(&o.Config, "config", "configuration key-value pairs")
 	flags.Var(&o.Labels, "labels", "labels to apply to the backup storage location")
 	flags.Var(
@@ -119,12 +120,16 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error {
 }

 func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
-	var backupSyncPeriod *metav1.Duration
+	var backupSyncPeriod, validationFrequency *metav1.Duration

 	if c.Flags().Changed("backup-sync-period") {
 		backupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod}
 	}

+	if c.Flags().Changed("validation-frequency") {
+		validationFrequency = &metav1.Duration{Duration: o.ValidationFrequency}
+	}
+
 	backupStorageLocation := &velerov1api.BackupStorageLocation{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: f.Namespace(),
@@ -139,9 +144,10 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 					Prefix: o.Prefix,
 				},
 			},
-			Config:           o.Config.Data(),
-			AccessMode:       velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
-			BackupSyncPeriod: backupSyncPeriod,
+			Config:              o.Config.Data(),
+			AccessMode:          velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
+			BackupSyncPeriod:    backupSyncPeriod,
+			ValidationFrequency: validationFrequency,
 		},
 	}

@@ -49,7 +49,6 @@ import (
 	snapshotv1beta1informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions"
 	snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1"

-	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/buildinfo"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -72,10 +71,10 @@ import (

 	ctrl "sigs.k8s.io/controller-runtime"

-	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"

 	"github.com/vmware-tanzu/velero/internal/util/managercontroller"
+	"github.com/vmware-tanzu/velero/internal/velero"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 )

@@ -84,6 +83,7 @@ const (
 	defaultMetricsAddress = ":8085"

 	defaultBackupSyncPeriod           = time.Minute
+	defaultStoreValidationFrequency   = time.Minute
 	defaultPodVolumeOperationTimeout  = 240 * time.Minute
 	defaultResourceTerminatingTimeout = 10 * time.Minute

@@ -125,7 +125,7 @@ var disableControllerList = []string{
 type serverConfig struct {
 	pluginDir, metricsAddress, defaultBackupLocation                        string
 	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration
-	defaultBackupTTL                                                        time.Duration
+	defaultBackupTTL, storeValidationFrequency                              time.Duration
 	restoreResourcePriorities                                               []string
 	defaultVolumeSnapshotLocations                                          map[string]string
 	restoreOnly                                                             bool
@@ -154,6 +154,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 			defaultVolumeSnapshotLocations:    make(map[string]string),
 			backupSyncPeriod:                  defaultBackupSyncPeriod,
 			defaultBackupTTL:                  defaultBackupTTL,
+			storeValidationFrequency:          defaultStoreValidationFrequency,
 			podVolumeOperationTimeout:         defaultPodVolumeOperationTimeout,
 			restoreResourcePriorities:         defaultRestorePriorities,
 			clientQPS:                         defaultClientQPS,
@@ -217,6 +218,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	command.Flags().StringSliceVar(&config.disabledControllers, "disable-controllers", config.disabledControllers, fmt.Sprintf("list of controllers to disable on startup. Valid values are %s", strings.Join(disableControllerList, ",")))
 	command.Flags().StringSliceVar(&config.restoreResourcePriorities, "restore-resource-priorities", config.restoreResourcePriorities, "desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources")
 	command.Flags().StringVar(&config.defaultBackupLocation, "default-backup-storage-location", config.defaultBackupLocation, "name of the default backup storage location")
+	command.Flags().DurationVar(&config.storeValidationFrequency, "store-validation-frequency", config.storeValidationFrequency, "how often to verify if the storage is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.")
 	command.Flags().Var(&volumeSnapshotLocations, "default-volume-snapshot-locations", "list of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...)")
 	command.Flags().Float32Var(&config.clientQPS, "client-qps", config.clientQPS, "maximum number of requests per second by the server to the Kubernetes API once the burst limit has been reached")
 	command.Flags().IntVar(&config.clientBurst, "client-burst", config.clientBurst, "maximum number of requests by the server to the Kubernetes API in a short period of time")
@@ -296,7 +298,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 	}

 	var csiSnapClient *snapshotv1beta1client.Clientset
-	if features.IsEnabled(api.CSIFeatureFlag) {
+	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		csiSnapClient, err = snapshotv1beta1client.NewForConfig(clientConfig)
 		if err != nil {
 			cancelFunc()
@@ -359,21 +361,6 @@ func (s *server) run() error {
 		return err
 	}

-	if err := s.validateBackupStorageLocations(); err != nil {
-		return err
-	}
-
-	// Fetching from the server directly since at this point
-	// the cache has not yet started
-	bsl := &velerov1api.BackupStorageLocation{}
-	if err := s.mgr.GetAPIReader().Get(context.Background(), kbclient.ObjectKey{
-		Namespace: s.namespace,
-		Name:      s.config.defaultBackupLocation,
-	}, bsl); err != nil {
-		s.logger.WithError(errors.WithStack(err)).
-			Warnf("A backup storage location named %s has been specified for the server to use by default, but no corresponding backup storage location exists. Backups with a location not matching the default will need to explicitly specify an existing location", s.config.defaultBackupLocation)
-	}
-
 	if err := s.initRestic(); err != nil {
 		return err
 	}
@@ -427,14 +414,14 @@ func (s *server) veleroResourcesExist() error {

 	var veleroGroupVersion *metav1.APIResourceList
 	for _, gv := range s.discoveryHelper.Resources() {
-		if gv.GroupVersion == api.SchemeGroupVersion.String() {
+		if gv.GroupVersion == velerov1api.SchemeGroupVersion.String() {
 			veleroGroupVersion = gv
 			break
 		}
 	}

 	if veleroGroupVersion == nil {
-		return errors.Errorf("Velero API group %s not found. Apply examples/common/00-prereqs.yaml to create it.", api.SchemeGroupVersion)
+		return errors.Errorf("Velero API group %s not found. Apply examples/common/00-prereqs.yaml to create it.", velerov1api.SchemeGroupVersion)
 	}

 	foundResources := sets.NewString()
@@ -443,13 +430,13 @@ func (s *server) veleroResourcesExist() error {
 	}

 	var errs []error
-	for kind := range api.CustomResources() {
+	for kind := range velerov1api.CustomResources() {
 		if foundResources.Has(kind) {
 			s.logger.WithField("kind", kind).Debug("Found custom resource")
 			continue
 		}

-		errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, api.SchemeGroupVersion))
+		errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, velerov1api.SchemeGroupVersion))
 	}

 	if len(errs) > 0 {
@@ -461,43 +448,6 @@ func (s *server) veleroResourcesExist() error {
 	return nil
 }

-// validateBackupStorageLocations checks to ensure all backup storage locations exist
-// and have a compatible layout, and returns an error if not.
-func (s *server) validateBackupStorageLocations() error {
-	s.logger.Info("Checking that all backup storage locations are valid")
-
-	pluginManager := clientmgmt.NewManager(s.logger, s.logLevel, s.pluginRegistry)
-	defer pluginManager.CleanupClients()
-
-	// Fetching from the server directly since at this point
-	// the cache has not yet started
-	locations := &velerov1api.BackupStorageLocationList{}
-	if err := s.mgr.GetAPIReader().List(context.Background(), locations, &kbclient.ListOptions{
-		Namespace: s.namespace,
-	}); err != nil {
-		return errors.WithStack(err)
-	}
-
-	var invalid []string
-	for _, location := range locations.Items {
-		backupStore, err := persistence.NewObjectBackupStore(&location, pluginManager, s.logger)
-		if err != nil {
-			invalid = append(invalid, errors.Wrapf(err, "error getting backup store for location %q", location.Name).Error())
-			continue
-		}
-
-		if err := backupStore.IsValid(); err != nil {
-			invalid = append(invalid, errors.Wrapf(err, "backup store for location %q is invalid", location.Name).Error())
-		}
-	}
-
-	if len(invalid) > 0 {
-		return errors.Errorf("some backup storage locations are invalid: %s", strings.Join(invalid, "; "))
-	}
-
-	return nil
-}
-
 // - Custom Resource Definitions come before Custom Resource so that they can be
 //   restored with their corresponding CRD.
 // - Namespaces go second because all namespaced resources depend on them.
@@ -595,12 +545,12 @@ func (s *server) getCSISnapshotListers() (snapshotv1beta1listers.VolumeSnapshotL

 	// If CSI is enabled, check for the CSI groups and generate the listers
 	// If CSI isn't enabled, return empty listers.
-	if features.IsEnabled(api.CSIFeatureFlag) {
+	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		_, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String())
 		switch {
 		case apierrors.IsNotFound(err):
 			// CSI is enabled, but the required CRDs aren't installed, so halt.
-			s.logger.Fatalf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", api.CSIFeatureFlag, snapshotv1beta1api.SchemeGroupVersion.String())
+			s.logger.Fatalf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", velerov1api.CSIFeatureFlag, snapshotv1beta1api.SchemeGroupVersion.String())
 		case err == nil:
 			// CSI is enabled, and the resources were found.
 			// Instantiate the listers fully
@@ -901,6 +851,22 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 		s.logger.WithField("informer", informer).Info("Informer cache synced")
 	}

+	storageLocationInfo := velero.StorageLocation{
+		Client:                          s.mgr.GetClient(),
+		Ctx:                             s.ctx,
+		DefaultStorageLocation:          s.config.defaultBackupLocation,
+		DefaultStoreValidationFrequency: s.config.storeValidationFrequency,
+		NewPluginManager:                newPluginManager,
+		NewBackupStore:                  persistence.NewObjectBackupStore,
+	}
+	if err := (&controller.BackupStorageLocationReconciler{
+		Scheme:          s.mgr.GetScheme(),
+		StorageLocation: storageLocationInfo,
+		Log:             s.logger,
+	}).SetupWithManager(s.mgr); err != nil {
+		s.logger.Fatal(err, "unable to create controller", "controller", "BackupStorageLocation")
+	}
+
 	// TODO(2.0): presuming all controllers and resources are converted to runtime-controller
 	// by v2.0, the block from this line and including the `s.mgr.Start() will be
 	// deprecated, since the manager auto-starts all the caches. Until then, we need to start the
@@ -944,7 +910,7 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme
 	// This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer
 	w := &CSIInformerFactoryWrapper{}

-	if features.IsEnabled(api.CSIFeatureFlag) {
+	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		w.factory = snapshotv1beta1informers.NewSharedInformerFactoryWithOptions(c, 0)
 	}
 	return w
@@ -952,14 +918,14 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme

 // Start proxies the Start call to the CSI SharedInformerFactory.
 func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) {
-	if features.IsEnabled(api.CSIFeatureFlag) {
+	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		w.factory.Start(stopCh)
 	}
 }

 // WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory.
 func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
-	if features.IsEnabled(api.CSIFeatureFlag) {
+	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		return w.factory.WaitForCacheSync(stopCh)
 	}
 	return nil
@@ -31,6 +31,7 @@ var (
 		{Name: "Provider"},
 		{Name: "Bucket/Prefix"},
 		{Name: "Phase"},
+		{Name: "Last Validated"},
 		{Name: "Access Mode"},
 	}
 )
@@ -64,11 +65,18 @@ func printBackupStorageLocation(location *velerov1api.BackupStorageLocation) []m
 		status = "Unknown"
 	}

+	lastValidated := location.Status.LastValidationTime
+	LastValidatedStr := "Unknown"
+	if lastValidated != nil {
+		LastValidatedStr = lastValidated.String()
+	}
+
 	row.Cells = append(row.Cells,
 		location.Name,
 		location.Spec.Provider,
 		bucketAndPrefix,
 		status,
+		LastValidatedStr,
 		accessMode,
 	)

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"

+	"github.com/vmware-tanzu/velero/internal/velero"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/features"
 	velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1"
@@ -122,16 +123,14 @@ func orderedBackupLocations(locationList *velerov1api.BackupStorageLocationList,
 func (c *backupSyncController) run() {
 	c.logger.Debug("Checking for existing backup storage locations to sync into cluster")

-	locationList := &velerov1api.BackupStorageLocationList{}
-	if err := c.kbClient.List(context.Background(), locationList, &client.ListOptions{
-		Namespace: c.namespace,
-	}); err != nil {
-		c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister")
+	locationList, err := velero.ListBackupStorageLocations(c.kbClient, context.Background(), c.namespace)
+	if err != nil {
+		c.logger.WithError(err).Error("No backup storage locations found, at least one is required")
 		return
 	}

 	// sync the default location first, if it exists
-	locations := orderedBackupLocations(locationList, c.defaultBackupLocation)
+	locations := orderedBackupLocations(&locationList, c.defaultBackupLocation)

 	pluginManager := c.newPluginManager(c.logger)
 	defer pluginManager.CleanupClients()
@@ -0,0 +1,142 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/vmware-tanzu/velero/internal/velero"
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+// BackupStorageLocationReconciler reconciles a BackupStorageLocation object
+type BackupStorageLocationReconciler struct {
+	Scheme          *runtime.Scheme
+	StorageLocation velero.StorageLocation
+
+	Log logrus.FieldLogger
+}
+
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch
+func (r *BackupStorageLocationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithField("controller", "backupstoragelocation")
+
+	log.Info("Checking for existing backup locations ready to be verified; there needs to be at least 1 backup location available")
+
+	locationList, err := velero.ListBackupStorageLocations(r.StorageLocation.Client, r.StorageLocation.Ctx, req.Namespace)
+	if err != nil {
+		log.WithError(err).Error("No backup storage locations found, at least one is required")
+	}
+
+	var defaultFound bool
+	var unavailableErrors []string
+	var anyVerified bool
+	for i := range locationList.Items {
+		location := &locationList.Items[i]
+		log := r.Log.WithField("controller", "backupstoragelocation").WithField("backupstoragelocation", location.Name)
+
+		if location.Name == r.StorageLocation.DefaultStorageLocation {
+			defaultFound = true
+		}
+
+		if !r.StorageLocation.IsReadyToValidate(location, log) {
+			continue
+		}
+
+		anyVerified = true
+
+		log.Debug("Verifying backup storage location")
+
+		if err := r.StorageLocation.IsValid(location, log); err != nil {
+			log.Debug("Backup location verified, not valid")
+			unavailableErrors = append(unavailableErrors, errors.Wrapf(err, "Backup location %q is unavailable", location.Name).Error())
+
+			if location.Name == r.StorageLocation.DefaultStorageLocation {
+				log.Warnf("The specified default backup location named %q is unavailable; for convenience, be sure to configure it properly or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation)
+			}
+
+			if err2 := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseUnavailable); err2 != nil {
+				log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseUnavailable)
+				continue
+			}
+		} else {
+			log.Debug("Backup location verified and it is valid")
+			if err := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseAvailable); err != nil {
+				log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseAvailable)
+				continue
+			}
+		}
+	}
+
+	if !anyVerified {
+		log.Info("No backup locations were ready to be verified")
+	}
+
+	r.logReconciledPhase(defaultFound, locationList, unavailableErrors)
+
+	return ctrl.Result{Requeue: true}, nil
+}
+
+func (r *BackupStorageLocationReconciler) logReconciledPhase(defaultFound bool, locationList velerov1api.BackupStorageLocationList, errs []string) {
+	var availableBSLs []*velerov1api.BackupStorageLocation
+	var unAvailableBSLs []*velerov1api.BackupStorageLocation
+	var unknownBSLs []*velerov1api.BackupStorageLocation
+	log := r.Log.WithField("controller", "backupstoragelocation")
+
+	for i, location := range locationList.Items {
+		phase := location.Status.Phase
+		switch phase {
+		case velerov1api.BackupStorageLocationPhaseAvailable:
+			availableBSLs = append(availableBSLs, &locationList.Items[i])
+		case velerov1api.BackupStorageLocationPhaseUnavailable:
+			unAvailableBSLs = append(unAvailableBSLs, &locationList.Items[i])
+		default:
+			unknownBSLs = append(unknownBSLs, &locationList.Items[i])
+		}
+	}
+
+	numAvailable := len(availableBSLs)
+	numUnavailable := len(unAvailableBSLs)
+	numUnknown := len(unknownBSLs)
+
+	if numUnavailable+numUnknown == len(locationList.Items) { // no available BSL
+		if len(errs) > 0 {
+			log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; "))
+		} else {
+			log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v)", numAvailable, numUnavailable, numUnknown)
+		}
+	} else if numUnavailable > 0 { // some but not all BSL unavailable
+		log.Warnf("Invalid backup locations detected: available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; "))
+	}
+
+	if !defaultFound {
+		log.Warnf("The specified default backup location named %q was not found; for convenience, be sure to create one or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation)
+	}
+}
+
+func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&velerov1api.BackupStorageLocation{}).
+		Complete(r)
+}
@@ -0,0 +1,185 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/vmware-tanzu/velero/internal/velero"
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	"github.com/vmware-tanzu/velero/pkg/persistence"
+	persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks"
+	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
+	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+)
+
+var _ = Describe("Backup Storage Location Reconciler", func() {
+	BeforeEach(func() {})
+	AfterEach(func() {})
+
+	It("Should successfully patch a backup storage location object status phase according to whether its storage is valid or not", func() {
+		tests := []struct {
+			backupLocation *velerov1api.BackupStorageLocation
+			isValidError   error
+			expectedPhase  velerov1api.BackupStorageLocationPhase
+		}{
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).Result(),
+				isValidError:   nil,
+				expectedPhase:  velerov1api.BackupStorageLocationPhaseAvailable,
+			},
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).Result(),
+				isValidError:   errors.New("an error"),
+				expectedPhase:  velerov1api.BackupStorageLocationPhaseUnavailable,
+			},
+		}
+
+		// Setup
+		var (
+			pluginManager = &pluginmocks.Manager{}
+			backupStores  = make(map[string]*persistencemocks.BackupStore)
+		)
+		pluginManager.On("CleanupClients").Return(nil)
+
+		locations := new(velerov1api.BackupStorageLocationList)
+		for i, test := range tests {
+			location := test.backupLocation
+			locations.Items = append(locations.Items, *location)
+			backupStores[location.Name] = &persistencemocks.BackupStore{}
+			backupStore := backupStores[location.Name]
+			backupStore.On("IsValid").Return(tests[i].isValidError)
+		}
+
+		// Setup reconciler
+		Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed())
+		storageLocationInfo := velero.StorageLocation{
+			Client:                          fake.NewFakeClientWithScheme(scheme.Scheme, locations),
+			DefaultStorageLocation:          "default",
+			DefaultStoreValidationFrequency: 0,
+			NewPluginManager:                func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
+			NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) {
+				return backupStores[loc.Name], nil
+			},
+		}
+
+		r := &BackupStorageLocationReconciler{
+			StorageLocation: storageLocationInfo,
+			Log:             velerotest.NewLogger(),
+		}
+
+		actualResult, err := r.Reconcile(ctrl.Request{
+			NamespacedName: types.NamespacedName{Namespace: "ns-1"},
+		})
+
+		Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true}))
+		Expect(err).To(BeNil())
+
+		// Assertions
+		for i, location := range locations.Items {
+			key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace}
+			instance := &velerov1api.BackupStorageLocation{}
+			err := r.StorageLocation.Client.Get(ctx, key, instance)
+			Expect(err).To(BeNil())
+			Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase))
+		}
+	})
+
+	It("Should not patch a backup storage location object status phase if the location's validation frequency is specifically set to zero", func() {
+		tests := []struct {
+			backupLocation *velerov1api.BackupStorageLocation
+			isValidError   error
+			expectedPhase  velerov1api.BackupStorageLocationPhase
+			wantErr        bool
+		}{
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(0).Result(),
+				isValidError:   nil,
+				expectedPhase:  "",
+				wantErr:        false,
+			},
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(0).Result(),
+				isValidError:   nil,
+				expectedPhase:  "",
+				wantErr:        false,
+			},
+		}
+
+		// Setup
+		var (
+			pluginManager = &pluginmocks.Manager{}
+			backupStores  = make(map[string]*persistencemocks.BackupStore)
+		)
+		pluginManager.On("CleanupClients").Return(nil)
+
+		locations := new(velerov1api.BackupStorageLocationList)
+		for i, test := range tests {
+			location := test.backupLocation
+			locations.Items = append(locations.Items, *location)
+			backupStores[location.Name] = &persistencemocks.BackupStore{}
+			backupStores[location.Name].On("IsValid").Return(tests[i].isValidError)
+		}
+
+		// Setup reconciler
+		Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed())
+		storageLocationInfo := velero.StorageLocation{
+			Client:                          fake.NewFakeClientWithScheme(scheme.Scheme, locations),
+			DefaultStorageLocation:          "default",
+			DefaultStoreValidationFrequency: 0,
+			NewPluginManager:                func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
+			NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) {
+				// this gets populated just below, prior to exercising the method under test
+				return backupStores[loc.Name], nil
+			},
+		}
+
+		r := &BackupStorageLocationReconciler{
+			StorageLocation: storageLocationInfo,
+			Log:             velerotest.NewLogger(),
+		}
+
+		actualResult, err := r.Reconcile(ctrl.Request{
+			NamespacedName: types.NamespacedName{Namespace: "ns-1"},
+		})
+
+		Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true}))
+		Expect(err).To(BeNil())
+
+		// Assertions
+		for i, location := range locations.Items {
+			key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace}
+			instance := &velerov1api.BackupStorageLocation{}
+			err := r.StorageLocation.Client.Get(ctx, key, instance)
+			Expect(err).To(BeNil())
+			Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase))
+		}
+	})
+})
@@ -17,19 +17,116 @@ limitations under the License.
 package controller

 import (
+	"context"
+	"path/filepath"
 	"testing"
-
-	"github.com/stretchr/testify/require"
+	"time"

 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/stretchr/testify/require"

 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"

 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	// +kubebuilder:scaffold:imports
 )

+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+const (
+	timeout = time.Second * 30
+)
+
+var (
+	env     *envtest.Environment
+	testEnv *testEnvironment
+	ctx     = context.Background()
+)
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Controller Suite")
+}
+
+var _ = BeforeSuite(func(done Done) {
+	By("bootstrapping test environment")
+	testEnv = newTestEnvironment()
+
+	By("starting the manager")
+	go func() {
+		defer GinkgoRecover()
+		Expect(testEnv.startManager()).To(Succeed())
+	}()
+
+	close(done)
+}, 60)
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.stop()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+// testEnvironment encapsulates a Kubernetes local test environment.
+type testEnvironment struct {
+	manager.Manager
+	client.Client
+	Config *rest.Config
+
+	doneMgr chan struct{}
+}
+
+// newTestEnvironment creates a new environment spinning up a local api-server.
+//
+// This function should be called only once for each package you're running tests within,
+// usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block.
+func newTestEnvironment() *testEnvironment {
+	err := velerov1api.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	env = &envtest.Environment{
+		CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
+	}
+
+	if _, err := env.Start(); err != nil {
+		panic(err)
+	}
+
+	mgr, err := manager.New(env.Config, manager.Options{
+		Scheme: scheme.Scheme,
+	})
+	if err != nil {
+		klog.Fatalf("Failed to start testenv manager: %v", err)
+	}
+
+	return &testEnvironment{
+		Manager: mgr,
+		Client:  mgr.GetClient(),
+		Config:  mgr.GetConfig(),
+		doneMgr: make(chan struct{}),
+	}
+}
+
+func (t *testEnvironment) startManager() error {
+	return t.Manager.Start(t.doneMgr)
+}
+
+func (t *testEnvironment) stop() error {
+	t.doneMgr <- struct{}{}
+	return env.Stop()
+}
+
 func newFakeClient(t *testing.T, initObjs ...runtime.Object) client.Client {
 	err := velerov1api.AddToScheme(scheme.Scheme)
 	require.NoError(t, err)