Support setting a custom CA bundle to use with a BackupStorageLocation (#2353)

* Support setting a custom CA certificate for a BSL

Signed-off-by: Sam Lucidi <slucidi@redhat.com>

* update CRDS

Signed-off-by: Sam Lucidi <slucidi@redhat.com>

* Add changelog for #2353

Signed-off-by: Sam Lucidi <slucidi@redhat.com>

* Clean up temp file from TestTempCACertFile

Signed-off-by: Sam Lucidi <slucidi@redhat.com>

changelogs/unreleased/2353-mansam

@@ -0,0 +1 @@
+support setting a custom CA certificate on a BSL to use when verifying TLS connections

pkg/apis/velero/v1/backup_storage_location.go

@@ -64,6 +64,10 @@ type ObjectStorageLocation struct {
 	// Prefix is the path inside a bucket to use for Velero storage. Optional.
 	// +optional
 	Prefix string `json:"prefix,omitempty"`
+
+	// CACert defines a CA bundle to use when verifying TLS connections to the provider.
+	// +optional
+	CACert []byte `json:"caCert,omitempty"`
 }
 
 // BackupStorageLocationSpec defines the specification for a Velero BackupStorageLocation.

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -623,6 +623,11 @@ func (in *ExecHook) DeepCopy() *ExecHook {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectStorageLocation) DeepCopyInto(out *ObjectStorageLocation) {
 	*out = *in
+	if in.CACert != nil {
+		in, out := &in.CACert, &out.CACert
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -1330,7 +1335,7 @@ func (in *StorageType) DeepCopyInto(out *StorageType) {
 	if in.ObjectStorage != nil {
 		in, out := &in.ObjectStorage, &out.ObjectStorage
 		*out = new(ObjectStorageLocation)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }

pkg/controller/pod_volume_backup_controller.go

@@ -212,21 +212,37 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 	log.WithField("path", path).Debugf("Found path matching glob")
 
 	// temp creds
-	file, err := restic.TempCredentialsFile(c.secretLister, req.Namespace, req.Spec.Pod.Namespace, c.fileSystem)
+	credentialsFile, err := restic.TempCredentialsFile(c.secretLister, req.Namespace, req.Spec.Pod.Namespace, c.fileSystem)
 	if err != nil {
 		log.WithError(err).Error("Error creating temp restic credentials file")
 		return c.fail(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log)
 	}
 	// ignore error since there's nothing we can do and it's a temp file.
-	defer os.Remove(file)
+	defer os.Remove(credentialsFile)
 
 	resticCmd := restic.BackupCommand(
 		req.Spec.RepoIdentifier,
-		file,
+		credentialsFile,
 		path,
 		req.Spec.Tags,
 	)
 
+	// if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic
+	caCert, err := restic.GetCACert(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation)
+	if err != nil {
+		log.WithError(err).Error("Error getting caCert")
+	}
+	var caCertFile string
+	if caCert != nil {
+		caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem)
+		if err != nil {
+			log.WithError(err).Error("Error creating temp cacert file")
+		}
+		// ignore error since there's nothing we can do and it's a temp file.
+		defer os.Remove(caCertFile)
+	}
+	resticCmd.CACertFile = caCertFile
+
 	// Running restic command might need additional provider specific environment variables. Based on the provider, we
 	// set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations)
 	var env []string
@@ -272,7 +288,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 
 	var snapshotID string
 	if !emptySnapshot {
-		snapshotID, err = restic.GetSnapshotID(req.Spec.RepoIdentifier, file, req.Spec.Tags, env)
+		snapshotID, err = restic.GetSnapshotID(req.Spec.RepoIdentifier, credentialsFile, req.Spec.Tags, env, caCertFile)
 		if err != nil {
 			log.WithError(err).Error("Error getting SnapshotID")
 			return c.fail(req, errors.Wrap(err, "error getting snapshot id").Error(), log)

pkg/controller/pod_volume_restore_controller.go

@@ -293,8 +293,23 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe
 	// ignore error since there's nothing we can do and it's a temp file.
 	defer os.Remove(credsFile)
 
+	// if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic
+	caCert, err := restic.GetCACert(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation)
+	if err != nil {
+		log.WithError(err).Error("Error getting caCert")
+	}
+	var caCertFile string
+	if caCert != nil {
+		caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem)
+		if err != nil {
+			log.WithError(err).Error("Error creating temp cacert file")
+		}
+		// ignore error since there's nothing we can do and it's a temp file.
+		defer os.Remove(caCertFile)
+	}
+
 	// execute the restore process
-	if err := c.restorePodVolume(req, credsFile, volumeDir, log); err != nil {
+	if err := c.restorePodVolume(req, credsFile, caCertFile, volumeDir, log); err != nil {
 		log.WithError(err).Error("Error restoring volume")
 		return c.failRestore(req, errors.Wrap(err, "error restoring volume").Error(), log)
 	}
@@ -313,7 +328,7 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe
 	return nil
 }
 
-func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolumeRestore, credsFile, volumeDir string, log logrus.FieldLogger) error {
+func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolumeRestore, credsFile, caCertFile, volumeDir string, log logrus.FieldLogger) error {
 	// Get the full path of the new volume's directory as mounted in the daemonset pod, which
 	// will look like: /host_pods/<new-pod-uid>/volumes/<volume-plugin-name>/<volume-dir>
 	volumePath, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir))
@@ -327,6 +342,7 @@ func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolume
 		req.Spec.SnapshotID,
 		volumePath,
 	)
+	resticCmd.CACertFile = caCertFile
 
 	// Running restic command might need additional provider specific environment variables. Based on the provider, we
 	// set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations)

pkg/generated/crds/manifests/velero.io_backupstoragelocations.yaml

@@ -61,6 +61,11 @@ spec:
                 bucket:
                   description: Bucket is the bucket to use for object storage.
                   type: string
+                caCert:
+                  description: CACert defines a CA bundle to use when verifying TLS
+                    connections to the provider.
+                  format: byte
+                  type: string
                 prefix:
                   description: Prefix is the path inside a bucket to use for Velero
                     storage. Optional.

pkg/persistence/object_store.go

@@ -114,6 +114,7 @@ func NewObjectBackupStore(location *velerov1api.BackupStorageLocation, objectSto
 		}
 		location.Spec.Config["bucket"] = bucket
 		location.Spec.Config["prefix"] = prefix
+		location.Spec.Config["caCert"] = string(location.Spec.ObjectStorage.CACert)
 	}
 
 	objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider)

pkg/plugin/framework/validation.go

@@ -27,7 +27,7 @@ import (
 func ValidateObjectStoreConfigKeys(config map[string]string, validKeys ...string) error {
 	// `bucket` and `prefix` are automatically added to all object
 	// store config by velero, so add them as valid keys.
-	return validateConfigKeys(config, append(validKeys, "bucket", "prefix")...)
+	return validateConfigKeys(config, append(validKeys, "bucket", "prefix", "caCert")...)
 }
 
 // ValidateVolumeSnapshotterConfigKeys ensures that a volume snapshotter's

pkg/plugin/generated/RestoreItemAction.pb.go

@@ -24,10 +24,12 @@ type RestoreItemActionExecuteRequest struct {
 	ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"`
 }
 
-func (m *RestoreItemActionExecuteRequest) Reset()                    { *m = RestoreItemActionExecuteRequest{} }
-func (m *RestoreItemActionExecuteRequest) String() string            { return proto.CompactTextString(m) }
-func (*RestoreItemActionExecuteRequest) ProtoMessage()               {}
-func (*RestoreItemActionExecuteRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} }
+func (m *RestoreItemActionExecuteRequest) Reset()         { *m = RestoreItemActionExecuteRequest{} }
+func (m *RestoreItemActionExecuteRequest) String() string { return proto.CompactTextString(m) }
+func (*RestoreItemActionExecuteRequest) ProtoMessage()    {}
+func (*RestoreItemActionExecuteRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor3, []int{0}
+}
 
 func (m *RestoreItemActionExecuteRequest) GetPlugin() string {
 	if m != nil {

pkg/restic/command.go

@@ -29,6 +29,7 @@ type Command struct {
 	Command        string
 	RepoIdentifier string
 	PasswordFile   string
+	CACertFile     string
 	Dir            string
 	Args           []string
 	ExtraFlags     []string
@@ -51,6 +52,9 @@ func (c *Command) StringSlice() []string {
 	if c.PasswordFile != "" {
 		res = append(res, passwordFlag(c.PasswordFile))
 	}
+	if c.CACertFile != "" {
+		res = append(res, cacertFlag(c.CACertFile))
+	}
 
 	// If VELERO_SCRATCH_DIR is defined, put the restic cache within it. If not,
 	// allow restic to choose the location. This makes running either in-cluster
@@ -94,3 +98,7 @@ func passwordFlag(file string) string {
 func cacheDirFlag(dir string) string {
 	return fmt.Sprintf("--cache-dir=%s", dir)
 }
+
+func cacertFlag(path string) string {
+	return fmt.Sprintf("--cacert=%s", path)
+}

pkg/restic/common.go

@@ -204,6 +204,44 @@ func TempCredentialsFile(secretLister corev1listers.SecretLister, veleroNamespac
 	return name, nil
 }
 
+// TempCACertFile creates a temp file containing a CA bundle
+// and returns its path. The caller should generally call os.Remove()
+// to remove the file when done with it.
+func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, error) {
+	file, err := fs.TempFile("", fmt.Sprintf("cacert-%s", bsl))
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	if _, err := file.Write(caCert); err != nil {
+		// nothing we can do about an error closing the file here, and we're
+		// already returning an error about the write failing.
+		file.Close()
+		return "", errors.WithStack(err)
+	}
+
+	name := file.Name()
+
+	if err := file.Close(); err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	return name, nil
+}
+
+func GetCACert(backupLocationLister velerov1listers.BackupStorageLocationLister, namespace, bsl string) ([]byte, error) {
+	location, err := backupLocationLister.BackupStorageLocations(namespace).Get(bsl)
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting backup storage location")
+	}
+
+	if location.Spec.ObjectStorage != nil {
+		return location.Spec.ObjectStorage.CACert, nil
+	}
+
+	return nil, nil
+}
+
 // NewPodVolumeBackupListOptions creates a ListOptions with a label selector configured to
 // find PodVolumeBackups for the backup identified by name.
 func NewPodVolumeBackupListOptions(name string) metav1.ListOptions {

pkg/restic/common_test.go

@@ -17,9 +17,12 @@ limitations under the License.
 package restic
 
 import (
+	"os"
 	"sort"
 	"testing"
 
+	velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
@@ -375,3 +378,44 @@ func TestTempCredentialsFile(t *testing.T) {
 
 	assert.Equal(t, "passw0rd", string(contents))
 }
+
+func TestTempCACertFile(t *testing.T) {
+	var (
+		bslInformer = cache.NewSharedIndexInformer(nil, new(velerov1api.BackupStorageLocation), 0, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+		bslLister   = velerov1listers.NewBackupStorageLocationLister(bslInformer.GetIndexer())
+		fs          = velerotest.NewFakeFileSystem()
+		bsl         = &velerov1api.BackupStorageLocation{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "velero",
+				Name:      "default",
+			},
+			Spec: velerov1api.BackupStorageLocationSpec{
+				StorageType: velerov1api.StorageType{
+					ObjectStorage: &velerov1api.ObjectStorageLocation{CACert: []byte("cacert")},
+				},
+			},
+		}
+	)
+
+	// bsl not in lister: expect an error
+	caCert, err := GetCACert(bslLister, "velero", "default")
+	assert.Error(t, err)
+
+	// now add bsl to lister
+	require.NoError(t, bslInformer.GetStore().Add(bsl))
+
+	// bsl in lister: expect temp file to be created with cacert value
+	caCert, err = GetCACert(bslLister, "velero", "default")
+	require.NoError(t, err)
+
+	fileName, err := TempCACertFile(caCert, "default", fs)
+	require.NoError(t, err)
+
+	contents, err := fs.ReadFile(fileName)
+	require.NoError(t, err)
+
+	assert.Equal(t, "cacert", string(contents))
+
+	os.Remove(fileName)
+}

pkg/restic/exec_commands.go

@@ -47,11 +47,12 @@ type backupStatusLine struct {
 // GetSnapshotID runs a 'restic snapshots' command to get the ID of the snapshot
 // in the specified repo matching the set of provided tags, or an error if a
 // unique snapshot cannot be identified.
-func GetSnapshotID(repoIdentifier, passwordFile string, tags map[string]string, env []string) (string, error) {
+func GetSnapshotID(repoIdentifier, passwordFile string, tags map[string]string, env []string, caCertFile string) (string, error) {
 	cmd := GetSnapshotCommand(repoIdentifier, passwordFile, tags)
 	if len(env) > 0 {
 		cmd.Env = env
 	}
+	cmd.CACertFile = caCertFile
 
 	stdout, stderr, err := exec.RunCommand(cmd.Cmd())
 	if err != nil {

pkg/restic/repository_manager.go

@@ -244,6 +244,22 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error {
 
 	cmd.PasswordFile = file
 
+	// if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic
+	caCert, err := GetCACert(rm.backupLocationLister, rm.namespace, backupLocation)
+	if err != nil {
+		return err
+	}
+	var caCertFile string
+	if caCert != nil {
+		caCertFile, err = TempCACertFile(caCert, backupLocation, rm.fileSystem)
+		if err != nil {
+			return err
+		}
+		// ignore error since there's nothing we can do and it's a temp file.
+		defer os.Remove(caCertFile)
+	}
+	cmd.CACertFile = caCertFile
+
 	if strings.HasPrefix(cmd.RepoIdentifier, "azure") {
 		if !cache.WaitForCacheSync(rm.ctx.Done(), rm.backupLocationInformerSynced) {
 			return errors.New("timed out waiting for cache to sync")