Add a BSL controller to handle validation + update BSL status phase (#2674)

* Add BSL controller Signed-off-by: Carlisia <carlisia@vmware.com> * Add changelog Signed-off-by: Carlisia <carlisia@vmware.com> * Make update Signed-off-by: Carlisia <carlisia@vmware.com> * Update docs Signed-off-by: Carlisia <carlisia@vmware.com> * Add kubebuilder dependency Signed-off-by: Carlisia <carlisia@vmware.com> * Add export Signed-off-by: Carlisia <carlisia@vmware.com> * add kubebuilder binaries into velero builder image Signed-off-by: Ashish Amarnath <ashisham@vmware.com> * Reset velero dockerfile Signed-off-by: Carlisia <carlisia@vmware.com> * Consolidate all logic Signed-off-by: Carlisia <carlisia@vmware.com> * Add copyright header Signed-off-by: Carlisia <carlisia@vmware.com> * Clean up + add "last validated" column Signed-off-by: Carlisia <carlisia@vmware.com> * Better tests Signed-off-by: Carlisia <carlisia@vmware.com> * Add more tests Signed-off-by: Carlisia <carlisia@vmware.com> * Better logging Signed-off-by: Carlisia <carlisia@vmware.com> * Format Signed-off-by: Carlisia <carlisia@vmware.com> * Code reviews Signed-off-by: Carlisia <carlisia@vmware.com> * Address code review Signed-off-by: Carlisia <carlisia@vmware.com> * Remove redundant logic Signed-off-by: Carlisia <carlisia@vmware.com> Co-authored-by: Ashish Amarnath <ashisham@vmware.com>
2026-08-17 04:36:05 +00:00 · 2020-07-14 17:47:00 -04:00
parent 3d3b9e312a
commit dbd0aa4915
19 changed files with 1015 additions and 92 deletions
@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"

+	"github.com/vmware-tanzu/velero/internal/velero"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/features"
 	velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1"
@@ -122,16 +123,14 @@ func orderedBackupLocations(locationList *velerov1api.BackupStorageLocationList,
 func (c *backupSyncController) run() {
 	c.logger.Debug("Checking for existing backup storage locations to sync into cluster")

-	locationList := &velerov1api.BackupStorageLocationList{}
-	if err := c.kbClient.List(context.Background(), locationList, &client.ListOptions{
-		Namespace: c.namespace,
-	}); err != nil {
-		c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister")
+	locationList, err := velero.ListBackupStorageLocations(c.kbClient, context.Background(), c.namespace)
+	if err != nil {
+		c.logger.WithError(err).Error("No backup storage locations found, at least one is required")
 		return
 	}

 	// sync the default location first, if it exists
-	locations := orderedBackupLocations(locationList, c.defaultBackupLocation)
+	locations := orderedBackupLocations(&locationList, c.defaultBackupLocation)

 	pluginManager := c.newPluginManager(c.logger)
 	defer pluginManager.CleanupClients()
@@ -0,0 +1,142 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/vmware-tanzu/velero/internal/velero"
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+// BackupStorageLocationReconciler reconciles a BackupStorageLocation object
+type BackupStorageLocationReconciler struct {
+	Scheme          *runtime.Scheme
+	StorageLocation velero.StorageLocation
+
+	Log logrus.FieldLogger
+}
+
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch
+func (r *BackupStorageLocationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithField("controller", "backupstoragelocation")
+
+	log.Info("Checking for existing backup locations ready to be verified; there needs to be at least 1 backup location available")
+
+	locationList, err := velero.ListBackupStorageLocations(r.StorageLocation.Client, r.StorageLocation.Ctx, req.Namespace)
+	if err != nil {
+		log.WithError(err).Error("No backup storage locations found, at least one is required")
+	}
+
+	var defaultFound bool
+	var unavailableErrors []string
+	var anyVerified bool
+	for i := range locationList.Items {
+		location := &locationList.Items[i]
+		log := r.Log.WithField("controller", "backupstoragelocation").WithField("backupstoragelocation", location.Name)
+
+		if location.Name == r.StorageLocation.DefaultStorageLocation {
+			defaultFound = true
+		}
+
+		if !r.StorageLocation.IsReadyToValidate(location, log) {
+			continue
+		}
+
+		anyVerified = true
+
+		log.Debug("Verifying backup storage location")
+
+		if err := r.StorageLocation.IsValid(location, log); err != nil {
+			log.Debug("Backup location verified, not valid")
+			unavailableErrors = append(unavailableErrors, errors.Wrapf(err, "Backup location %q is unavailable", location.Name).Error())
+
+			if location.Name == r.StorageLocation.DefaultStorageLocation {
+				log.Warnf("The specified default backup location named %q is unavailable; for convenience, be sure to configure it properly or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation)
+			}
+
+			if err2 := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseUnavailable); err2 != nil {
+				log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseUnavailable)
+				continue
+			}
+		} else {
+			log.Debug("Backup location verified and it is valid")
+			if err := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseAvailable); err != nil {
+				log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseAvailable)
+				continue
+			}
+		}
+	}
+
+	if !anyVerified {
+		log.Info("No backup locations were ready to be verified")
+	}
+
+	r.logReconciledPhase(defaultFound, locationList, unavailableErrors)
+
+	return ctrl.Result{Requeue: true}, nil
+}
+
+func (r *BackupStorageLocationReconciler) logReconciledPhase(defaultFound bool, locationList velerov1api.BackupStorageLocationList, errs []string) {
+	var availableBSLs []*velerov1api.BackupStorageLocation
+	var unAvailableBSLs []*velerov1api.BackupStorageLocation
+	var unknownBSLs []*velerov1api.BackupStorageLocation
+	log := r.Log.WithField("controller", "backupstoragelocation")
+
+	for i, location := range locationList.Items {
+		phase := location.Status.Phase
+		switch phase {
+		case velerov1api.BackupStorageLocationPhaseAvailable:
+			availableBSLs = append(availableBSLs, &locationList.Items[i])
+		case velerov1api.BackupStorageLocationPhaseUnavailable:
+			unAvailableBSLs = append(unAvailableBSLs, &locationList.Items[i])
+		default:
+			unknownBSLs = append(unknownBSLs, &locationList.Items[i])
+		}
+	}
+
+	numAvailable := len(availableBSLs)
+	numUnavailable := len(unAvailableBSLs)
+	numUnknown := len(unknownBSLs)
+
+	if numUnavailable+numUnknown == len(locationList.Items) { // no available BSL
+		if len(errs) > 0 {
+			log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; "))
+		} else {
+			log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v)", numAvailable, numUnavailable, numUnknown)
+		}
+	} else if numUnavailable > 0 { // some but not all BSL unavailable
+		log.Warnf("Invalid backup locations detected: available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; "))
+	}
+
+	if !defaultFound {
+		log.Warnf("The specified default backup location named %q was not found; for convenience, be sure to create one or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation)
+	}
+}
+
+func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&velerov1api.BackupStorageLocation{}).
+		Complete(r)
+}
@@ -0,0 +1,185 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/vmware-tanzu/velero/internal/velero"
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	"github.com/vmware-tanzu/velero/pkg/persistence"
+	persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks"
+	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
+	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
+	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+)
+
+var _ = Describe("Backup Storage Location Reconciler", func() {
+	BeforeEach(func() {})
+	AfterEach(func() {})
+
+	It("Should successfully patch a backup storage location object status phase according to whether its storage is valid or not", func() {
+		tests := []struct {
+			backupLocation *velerov1api.BackupStorageLocation
+			isValidError   error
+			expectedPhase  velerov1api.BackupStorageLocationPhase
+		}{
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).Result(),
+				isValidError:   nil,
+				expectedPhase:  velerov1api.BackupStorageLocationPhaseAvailable,
+			},
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).Result(),
+				isValidError:   errors.New("an error"),
+				expectedPhase:  velerov1api.BackupStorageLocationPhaseUnavailable,
+			},
+		}
+
+		// Setup
+		var (
+			pluginManager = &pluginmocks.Manager{}
+			backupStores  = make(map[string]*persistencemocks.BackupStore)
+		)
+		pluginManager.On("CleanupClients").Return(nil)
+
+		locations := new(velerov1api.BackupStorageLocationList)
+		for i, test := range tests {
+			location := test.backupLocation
+			locations.Items = append(locations.Items, *location)
+			backupStores[location.Name] = &persistencemocks.BackupStore{}
+			backupStore := backupStores[location.Name]
+			backupStore.On("IsValid").Return(tests[i].isValidError)
+		}
+
+		// Setup reconciler
+		Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed())
+		storageLocationInfo := velero.StorageLocation{
+			Client:                          fake.NewFakeClientWithScheme(scheme.Scheme, locations),
+			DefaultStorageLocation:          "default",
+			DefaultStoreValidationFrequency: 0,
+			NewPluginManager:                func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
+			NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) {
+				return backupStores[loc.Name], nil
+			},
+		}
+
+		r := &BackupStorageLocationReconciler{
+			StorageLocation: storageLocationInfo,
+			Log:             velerotest.NewLogger(),
+		}
+
+		actualResult, err := r.Reconcile(ctrl.Request{
+			NamespacedName: types.NamespacedName{Namespace: "ns-1"},
+		})
+
+		Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true}))
+		Expect(err).To(BeNil())
+
+		// Assertions
+		for i, location := range locations.Items {
+			key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace}
+			instance := &velerov1api.BackupStorageLocation{}
+			err := r.StorageLocation.Client.Get(ctx, key, instance)
+			Expect(err).To(BeNil())
+			Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase))
+		}
+	})
+
+	It("Should not patch a backup storage location object status phase if the location's validation frequency is specifically set to zero", func() {
+		tests := []struct {
+			backupLocation *velerov1api.BackupStorageLocation
+			isValidError   error
+			expectedPhase  velerov1api.BackupStorageLocationPhase
+			wantErr        bool
+		}{
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(0).Result(),
+				isValidError:   nil,
+				expectedPhase:  "",
+				wantErr:        false,
+			},
+			{
+				backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(0).Result(),
+				isValidError:   nil,
+				expectedPhase:  "",
+				wantErr:        false,
+			},
+		}
+
+		// Setup
+		var (
+			pluginManager = &pluginmocks.Manager{}
+			backupStores  = make(map[string]*persistencemocks.BackupStore)
+		)
+		pluginManager.On("CleanupClients").Return(nil)
+
+		locations := new(velerov1api.BackupStorageLocationList)
+		for i, test := range tests {
+			location := test.backupLocation
+			locations.Items = append(locations.Items, *location)
+			backupStores[location.Name] = &persistencemocks.BackupStore{}
+			backupStores[location.Name].On("IsValid").Return(tests[i].isValidError)
+		}
+
+		// Setup reconciler
+		Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed())
+		storageLocationInfo := velero.StorageLocation{
+			Client:                          fake.NewFakeClientWithScheme(scheme.Scheme, locations),
+			DefaultStorageLocation:          "default",
+			DefaultStoreValidationFrequency: 0,
+			NewPluginManager:                func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
+			NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) {
+				// this gets populated just below, prior to exercising the method under test
+				return backupStores[loc.Name], nil
+			},
+		}
+
+		r := &BackupStorageLocationReconciler{
+			StorageLocation: storageLocationInfo,
+			Log:             velerotest.NewLogger(),
+		}
+
+		actualResult, err := r.Reconcile(ctrl.Request{
+			NamespacedName: types.NamespacedName{Namespace: "ns-1"},
+		})
+
+		Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true}))
+		Expect(err).To(BeNil())
+
+		// Assertions
+		for i, location := range locations.Items {
+			key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace}
+			instance := &velerov1api.BackupStorageLocation{}
+			err := r.StorageLocation.Client.Get(ctx, key, instance)
+			Expect(err).To(BeNil())
+			Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase))
+		}
+	})
+})
@@ -17,19 +17,116 @@ limitations under the License.
 package controller

 import (
+	"context"
+	"path/filepath"
 	"testing"
-
-	"github.com/stretchr/testify/require"
+	"time"

 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/stretchr/testify/require"

 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"

 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	// +kubebuilder:scaffold:imports
 )

+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+const (
+	timeout = time.Second * 30
+)
+
+var (
+	env     *envtest.Environment
+	testEnv *testEnvironment
+	ctx     = context.Background()
+)
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Controller Suite")
+}
+
+var _ = BeforeSuite(func(done Done) {
+	By("bootstrapping test environment")
+	testEnv = newTestEnvironment()
+
+	By("starting the manager")
+	go func() {
+		defer GinkgoRecover()
+		Expect(testEnv.startManager()).To(Succeed())
+	}()
+
+	close(done)
+}, 60)
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.stop()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+// testEnvironment encapsulates a Kubernetes local test environment.
+type testEnvironment struct {
+	manager.Manager
+	client.Client
+	Config *rest.Config
+
+	doneMgr chan struct{}
+}
+
+// newTestEnvironment creates a new environment spinning up a local api-server.
+//
+// This function should be called only once for each package you're running tests within,
+// usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block.
+func newTestEnvironment() *testEnvironment {
+	err := velerov1api.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	env = &envtest.Environment{
+		CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
+	}
+
+	if _, err := env.Start(); err != nil {
+		panic(err)
+	}
+
+	mgr, err := manager.New(env.Config, manager.Options{
+		Scheme: scheme.Scheme,
+	})
+	if err != nil {
+		klog.Fatalf("Failed to start testenv manager: %v", err)
+	}
+
+	return &testEnvironment{
+		Manager: mgr,
+		Client:  mgr.GetClient(),
+		Config:  mgr.GetConfig(),
+		doneMgr: make(chan struct{}),
+	}
+}
+
+func (t *testEnvironment) startManager() error {
+	return t.Manager.Start(t.doneMgr)
+}
+
+func (t *testEnvironment) stop() error {
+	t.doneMgr <- struct{}{}
+	return env.Stop()
+}
+
 func newFakeClient(t *testing.T, initObjs ...runtime.Object) client.Client {
 	err := velerov1api.AddToScheme(scheme.Scheme)
 	require.NoError(t, err)