From 9ffffda11e90241b6b74e9d81b578e631b435a99 Mon Sep 17 00:00:00 2001 From: Bridget McErlean Date: Thu, 11 Mar 2021 13:10:51 -0500 Subject: [PATCH] Use Credential from BSL for restic commands (#3489) * Use Credential from BSL for restic commands This change introduces support for restic to make use of per-BSL credentials. It makes use of the `credentials.FileStore` introduced in PR #3442 to write the BSL credentials to disk. To support per-BSL credentials for restic, the environment for the restic commands needs to be modified for each provider to ensure that the credentials are provided via the correct provider specific environment variables. This change introduces a new function `restic.CmdEnv` to check the BSL provider and create the correct mapping of environment variables for each provider. Previously, AWS and GCP could rely on the environment variables in the Velero deployments to obtain the credentials file, but now these environment variables need to be set with the path to the serialized credentials file if a credential is set on the BSL. For Azure, the credentials file in the environment was loaded and parsed to set the environment variables for restic. Now, we check if the BSL has a credential, and if it does, load and parse that file instead. This change also introduces a few other small improvements. Now that we are fetching the BSL to check for the `Credential` field, we can use the BSL directly to get the `CACert` which means that we can remove the `GetCACert` function. Also, now that we have a way to serialize secrets to disk, we can use the `credentials.FileStore` to get a temp file for the restic repo password and remove the `restic.TempCredentialsFile` function. Signed-off-by: Bridget McErlean * Add documentation for per-BSL credentials Signed-off-by: Bridget McErlean * Address review feedback Signed-off-by: Bridget McErlean * Address review comments Signed-off-by: Bridget McErlean --- changelogs/unreleased/3489-zubron | 1 + pkg/cmd/cli/restic/server.go | 19 +++ pkg/cmd/server/server.go | 28 ++-- .../pod_volume_backup_controller.go | 44 +++--- .../pod_volume_restore_controller.go | 78 +++++----- pkg/restic/aws.go | 11 +- pkg/restic/aws_test.go | 65 +++++++++ pkg/restic/azure.go | 33 +++-- pkg/restic/azure_test.go | 88 ++++++++++++ pkg/restic/common.go | 136 ++++++------------ pkg/restic/common_test.go | 59 +------- pkg/restic/config.go | 15 +- pkg/restic/exec_commands.go | 17 +-- pkg/restic/gcp.go | 34 +++++ pkg/restic/gcp_test.go | 56 ++++++++ pkg/restic/repository_keys.go | 24 +++- pkg/restic/repository_keys_test.go | 30 ++++ pkg/restic/repository_manager.go | 82 ++++++----- .../main/api-types/backupstoragelocation.md | 6 + site/content/docs/main/locations.md | 82 ++++++++++- site/content/docs/main/troubleshooting.md | 43 +++++- 21 files changed, 647 insertions(+), 304 deletions(-) create mode 100644 changelogs/unreleased/3489-zubron create mode 100644 pkg/restic/aws_test.go create mode 100644 pkg/restic/azure_test.go create mode 100644 pkg/restic/gcp.go create mode 100644 pkg/restic/gcp_test.go create mode 100644 pkg/restic/repository_keys_test.go diff --git a/changelogs/unreleased/3489-zubron b/changelogs/unreleased/3489-zubron new file mode 100644 index 000000000..3b0e5d936 --- /dev/null +++ b/changelogs/unreleased/3489-zubron @@ -0,0 +1 @@ +Add support for restic to use per-BSL credentials. Velero will now serialize the secret referenced by the `Credential` field in the BSL and use this path when setting provider specific environment variables for restic commands. \ No newline at end of file diff --git a/pkg/cmd/cli/restic/server.go b/pkg/cmd/cli/restic/server.go index c3cd57c0c..4de389bf9 100644 --- a/pkg/cmd/cli/restic/server.go +++ b/pkg/cmd/cli/restic/server.go @@ -42,6 +42,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" + "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -63,6 +64,10 @@ var ( const ( // the port where prometheus metrics are exposed defaultMetricsAddress = ":8085" + + // defaultCredentialsDirectory is the path on disk where credential + // files will be written to + defaultCredentialsDirectory = "/tmp/credentials" ) func NewServerCommand(f client.Factory) *cobra.Command { @@ -108,6 +113,7 @@ type resticServer struct { mgr manager.Manager metrics *metrics.ServerMetrics metricsAddress string + namespace string } func newResticServer(logger logrus.FieldLogger, factory client.Factory, metricAddress string) (*resticServer, error) { @@ -164,6 +170,7 @@ func newResticServer(logger logrus.FieldLogger, factory client.Factory, metricAd fileSystem: filesystem.NewFileSystem(), mgr: mgr, metricsAddress: metricAddress, + namespace: factory.Namespace(), } if err := s.validatePodVolumesHostPath(); err != nil { @@ -190,6 +197,16 @@ func (s *resticServer) run() { s.logger.Info("Starting controllers") + credentialFileStore, err := credentials.NewNamespacedFileStore( + s.mgr.GetClient(), + s.namespace, + defaultCredentialsDirectory, + filesystem.NewFileSystem(), + ) + if err != nil { + s.logger.Fatalf("Failed to create credentials file store: %v", err) + } + backupController := controller.NewPodVolumeBackupController( s.logger, s.veleroInformerFactory.Velero().V1().PodVolumeBackups(), @@ -200,6 +217,7 @@ func (s *resticServer) run() { s.metrics, s.mgr.GetClient(), os.Getenv("NODE_NAME"), + credentialFileStore, ) restoreController := controller.NewPodVolumeRestoreController( @@ -211,6 +229,7 @@ func (s *resticServer) run() { s.kubeInformerFactory.Core().V1().PersistentVolumes(), s.mgr.GetClient(), os.Getenv("NODE_NAME"), + credentialFileStore, ) go s.veleroInformerFactory.Start(s.ctx.Done()) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index b6eb0b9a8..7d1aec490 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -99,6 +99,8 @@ const ( // the default TTL for a backup defaultBackupTTL = 30 * 24 * time.Hour + // defaultCredentialsDirectory is the path on disk where credential + // files will be written to defaultCredentialsDirectory = "/tmp/credentials" ) @@ -233,6 +235,7 @@ type server struct { metrics *metrics.ServerMetrics config serverConfig mgr manager.Manager + credentialFileStore credentials.FileStore } func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*server, error) { @@ -299,6 +302,17 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s return nil, err } + credentialFileStore, err := credentials.NewNamespacedFileStore( + mgr.GetClient(), + f.Namespace(), + defaultCredentialsDirectory, + filesystem.NewFileSystem(), + ) + if err != nil { + cancelFunc() + return nil, err + } + s := &server{ namespace: f.Namespace(), metricsAddress: config.metricsAddress, @@ -317,6 +331,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s pluginRegistry: pluginRegistry, config: config, mgr: mgr, + credentialFileStore: credentialFileStore, } return s, nil @@ -497,6 +512,7 @@ func (s *server) initRestic() error { s.mgr.GetClient(), s.kubeClient.CoreV1(), s.kubeClient.CoreV1(), + s.credentialFileStore, s.logger, ) if err != nil { @@ -557,17 +573,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string return clientmgmt.NewManager(logger, s.logLevel, s.pluginRegistry) } - // Create the credentials store which will fetch secrets from the Velero - // namespace and store them on the file system - credentialFileStore, err := credentials.NewNamespacedFileStore( - s.mgr.GetClient(), - s.namespace, - defaultCredentialsDirectory, - filesystem.NewFileSystem(), - ) - cmd.CheckError(err) - - backupStoreGetter := persistence.NewObjectBackupStoreGetter(credentialFileStore) + backupStoreGetter := persistence.NewObjectBackupStoreGetter(s.credentialFileStore) csiVSLister, csiVSCLister := s.getCSISnapshotListers() diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 1a14d61c9..ef9c8e4c1 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -37,6 +37,7 @@ import ( corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" @@ -60,6 +61,7 @@ type podVolumeBackupController struct { kbClient client.Client nodeName string metrics *metrics.ServerMetrics + credentialsFileStore credentials.FileStore processBackupFunc func(*velerov1api.PodVolumeBackup) error fileSystem filesystem.Interface @@ -77,6 +79,7 @@ func NewPodVolumeBackupController( metrics *metrics.ServerMetrics, kbClient client.Client, nodeName string, + credentialsFileStore credentials.FileStore, ) Interface { c := &podVolumeBackupController{ genericController: newGenericController(PodVolumeBackup, logger), @@ -88,6 +91,7 @@ func NewPodVolumeBackupController( kbClient: kbClient, nodeName: nodeName, metrics: metrics, + credentialsFileStore: credentialsFileStore, fileSystem: filesystem.NewFileSystem(), clock: &clock.RealClock{}, @@ -221,7 +225,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack log.WithField("path", path).Debugf("Found path matching glob") // temp creds - credentialsFile, err := restic.TempCredentialsFile(c.kbClient, req.Namespace, c.fileSystem) + credentialsFile, err := c.credentialsFileStore.Path(restic.RepoKeySelector()) if err != nil { log.WithError(err).Error("Error creating temp restic credentials file") return c.fail(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log) @@ -236,15 +240,18 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack req.Spec.Tags, ) - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := restic.GetCACert(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) - if err != nil { - log.WithError(err).Error("Error getting caCert") + backupLocation := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: req.Namespace, + Name: req.Spec.BackupStorageLocation, + }, backupLocation); err != nil { + return c.fail(req, errors.Wrap(err, "error getting backup storage location").Error(), log) } + // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic var caCertFile string - if caCert != nil { - caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem) + if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { + caCertFile, err = restic.TempCACertFile(backupLocation.Spec.ObjectStorage.CACert, req.Spec.BackupStorageLocation, c.fileSystem) if err != nil { log.WithError(err).Error("Error creating temp cacert file") } @@ -253,20 +260,11 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack } resticCmd.CACertFile = caCertFile - // Running restic command might need additional provider specific environment variables. Based on the provider, we - // set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations) - var env []string - if strings.HasPrefix(req.Spec.RepoIdentifier, "azure") { - if env, err = restic.AzureCmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation); err != nil { - return c.fail(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) - } - resticCmd.Env = env - } else if strings.HasPrefix(req.Spec.RepoIdentifier, "s3") { - if env, err = restic.S3CmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation); err != nil { - return c.fail(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) - } - resticCmd.Env = env + env, err := restic.CmdEnv(backupLocation, c.credentialsFileStore) + if err != nil { + return c.fail(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } + resticCmd.Env = env // If this is a PVC, look for the most recent completed pod volume backup for it and get // its restic snapshot ID to use as the value of the `--parent` flag. Without this, @@ -298,7 +296,11 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack var snapshotID string if !emptySnapshot { - snapshotID, err = restic.GetSnapshotID(req.Spec.RepoIdentifier, credentialsFile, req.Spec.Tags, env, caCertFile) + cmd := restic.GetSnapshotCommand(req.Spec.RepoIdentifier, credentialsFile, req.Spec.Tags) + cmd.Env = env + cmd.CACertFile = caCertFile + + snapshotID, err = restic.GetSnapshotID(cmd) if err != nil { log.WithError(err).Error("Error getting SnapshotID") return c.fail(req, errors.Wrap(err, "error getting snapshot id").Error(), log) diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index eafe9847a..3b13cb8a8 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" jsonpatch "github.com/evanphx/json-patch" "github.com/pkg/errors" @@ -40,6 +39,7 @@ import ( k8scache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" @@ -61,6 +61,7 @@ type podVolumeRestoreController struct { backupLocationInformer k8scache.Informer kbClient client.Client nodeName string + credentialsFileStore credentials.FileStore processRestoreFunc func(*velerov1api.PodVolumeRestore) error fileSystem filesystem.Interface @@ -77,6 +78,7 @@ func NewPodVolumeRestoreController( pvInformer corev1informers.PersistentVolumeInformer, kbClient client.Client, nodeName string, + credentialsFileStore credentials.FileStore, ) Interface { c := &podVolumeRestoreController{ genericController: newGenericController(PodVolumeRestore, logger), @@ -87,6 +89,7 @@ func NewPodVolumeRestoreController( pvLister: pvInformer.Lister(), kbClient: kbClient, nodeName: nodeName, + credentialsFileStore: credentialsFileStore, fileSystem: filesystem.NewFileSystem(), clock: &clock.RealClock{}, @@ -300,32 +303,8 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe return c.failRestore(req, errors.Wrap(err, "error getting volume directory name").Error(), log) } - credsFile, err := restic.TempCredentialsFile(c.kbClient, req.Namespace, c.fileSystem) - if err != nil { - log.WithError(err).Error("Error creating temp restic credentials file") - return c.failRestore(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log) - } - // ignore error since there's nothing we can do and it's a temp file. - defer os.Remove(credsFile) - - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := restic.GetCACert(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) - if err != nil { - log.WithError(err).Error("Error getting caCert") - } - - var caCertFile string - if caCert != nil { - caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem) - if err != nil { - log.WithError(err).Error("Error creating temp cacert file") - } - // ignore error since there's nothing we can do and it's a temp file. - defer os.Remove(caCertFile) - } - // execute the restore process - if err := c.restorePodVolume(req, credsFile, caCertFile, volumeDir, log); err != nil { + if err := c.restorePodVolume(req, volumeDir, log); err != nil { log.WithError(err).Error("Error restoring volume") return c.failRestore(req, errors.Wrap(err, "error restoring volume").Error(), log) } @@ -344,7 +323,7 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe return nil } -func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolumeRestore, credsFile, caCertFile, volumeDir string, log logrus.FieldLogger) error { +func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolumeRestore, volumeDir string, log logrus.FieldLogger) error { // Get the full path of the new volume's directory as mounted in the daemonset pod, which // will look like: /host_pods//volumes// volumePath, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir)) @@ -352,29 +331,46 @@ func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolume return errors.Wrap(err, "error identifying path of volume") } + credsFile, err := c.credentialsFileStore.Path(restic.RepoKeySelector()) + if err != nil { + log.WithError(err).Error("Error creating temp restic credentials file") + return c.failRestore(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log) + } + // ignore error since there's nothing we can do and it's a temp file. + defer os.Remove(credsFile) + resticCmd := restic.RestoreCommand( req.Spec.RepoIdentifier, credsFile, req.Spec.SnapshotID, volumePath, ) + + backupLocation := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: req.Namespace, + Name: req.Spec.BackupStorageLocation, + }, backupLocation); err != nil { + return c.failRestore(req, errors.Wrap(err, "error getting backup storage location").Error(), log) + } + + // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic + var caCertFile string + if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { + caCertFile, err = restic.TempCACertFile(backupLocation.Spec.ObjectStorage.CACert, req.Spec.BackupStorageLocation, c.fileSystem) + if err != nil { + log.WithError(err).Error("Error creating temp cacert file") + } + // ignore error since there's nothing we can do and it's a temp file. + defer os.Remove(caCertFile) + } resticCmd.CACertFile = caCertFile - // Running restic command might need additional provider specific environment variables. Based on the provider, we - // set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations) - if strings.HasPrefix(req.Spec.RepoIdentifier, "azure") { - env, err := restic.AzureCmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) - if err != nil { - return c.failRestore(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) - } - resticCmd.Env = env - } else if strings.HasPrefix(req.Spec.RepoIdentifier, "s3") { - env, err := restic.S3CmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) - if err != nil { - return c.failRestore(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) - } - resticCmd.Env = env + env, err := restic.CmdEnv(backupLocation, c.credentialsFileStore) + if err != nil { + return c.failRestore(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } + resticCmd.Env = env var stdout, stderr string diff --git a/pkg/restic/aws.go b/pkg/restic/aws.go index 78550a3c6..d97c5f0b7 100644 --- a/pkg/restic/aws.go +++ b/pkg/restic/aws.go @@ -1,5 +1,5 @@ /* -Copyright 2019 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,8 +18,9 @@ package restic const ( // AWS specific environment variable - awsProfileEnvVar = "AWS_PROFILE" - awsProfileKey = "profile" + awsProfileEnvVar = "AWS_PROFILE" + awsProfileKey = "profile" + awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE" ) // getS3ResticEnvVars gets the environment variables that restic @@ -28,6 +29,10 @@ const ( func getS3ResticEnvVars(config map[string]string) (map[string]string, error) { result := make(map[string]string) + if credentialsFile, ok := config[credentialsFileKey]; ok { + result[awsCredentialsFileEnvVar] = credentialsFile + } + if profile, ok := config[awsProfileKey]; ok { result[awsProfileEnvVar] = profile } diff --git a/pkg/restic/aws_test.go b/pkg/restic/aws_test.go new file mode 100644 index 000000000..51f3ceb99 --- /dev/null +++ b/pkg/restic/aws_test.go @@ -0,0 +1,65 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restic + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetS3ResticEnvVars(t *testing.T) { + testCases := []struct { + name string + config map[string]string + expected map[string]string + }{ + { + name: "when config is empty, no env vars are returned", + config: map[string]string{}, + expected: map[string]string{}, + }, + { + name: "when config contains profile key, profile env var is set with profile value", + config: map[string]string{ + "profile": "profile-value", + }, + expected: map[string]string{ + "AWS_PROFILE": "profile-value", + }, + }, + { + name: "when config contains credentials file key, credentials file env var is set with credentials file value", + config: map[string]string{ + "credentialsFile": "/tmp/credentials/path/to/secret", + }, + expected: map[string]string{ + "AWS_SHARED_CREDENTIALS_FILE": "/tmp/credentials/path/to/secret", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual, err := getS3ResticEnvVars(tc.config) + + require.NoError(t, err) + + require.Equal(t, tc.expected, actual) + }) + } +} diff --git a/pkg/restic/azure.go b/pkg/restic/azure.go index 24ff6af90..20324b8e3 100644 --- a/pkg/restic/azure.go +++ b/pkg/restic/azure.go @@ -1,5 +1,5 @@ /* -Copyright 2017, 2019, 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -50,8 +50,9 @@ func getSubscriptionID(config map[string]string) string { } func getStorageAccountKey(config map[string]string) (string, *azure.Environment, error) { - // load environment vars from $AZURE_CREDENTIALS_FILE, if it exists - if err := loadEnv(); err != nil { + credentialsFile := selectCredentialsFile(config) + + if err := loadCredentialsIntoEnv(credentialsFile); err != nil { return "", nil, err } @@ -149,14 +150,30 @@ func getAzureResticEnvVars(config map[string]string) (map[string]string, error) }, nil } -func loadEnv() error { - envFile := os.Getenv("AZURE_CREDENTIALS_FILE") - if envFile == "" { +// credentialsFileFromEnv retrieves the Azure credentials file from the environment. +func credentialsFileFromEnv() string { + return os.Getenv("AZURE_CREDENTIALS_FILE") +} + +// selectCredentialsFile selects the Azure credentials file to use, retrieving it +// from the given config or falling back to retrieving it from the environment. +func selectCredentialsFile(config map[string]string) string { + if credentialsFile, ok := config[credentialsFileKey]; ok { + return credentialsFile + } + + return credentialsFileFromEnv() +} + +// loadCredentialsIntoEnv loads the variables in the given credentials +// file into the current environment. +func loadCredentialsIntoEnv(credentialsFile string) error { + if credentialsFile == "" { return nil } - if err := godotenv.Overload(envFile); err != nil { - return errors.Wrapf(err, "error loading environment from AZURE_CREDENTIALS_FILE (%s)", envFile) + if err := godotenv.Overload(credentialsFile); err != nil { + return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) } return nil diff --git a/pkg/restic/azure_test.go b/pkg/restic/azure_test.go new file mode 100644 index 000000000..acb2f2506 --- /dev/null +++ b/pkg/restic/azure_test.go @@ -0,0 +1,88 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restic + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// setAzureEnvironment sets the Azure credentials environment variable to the +// given value and returns a function to restore it to its previous value +func setAzureEnvironment(t *testing.T, value string) func() { + envVar := "AZURE_CREDENTIALS_FILE" + var cleanup func() + + if original, exists := os.LookupEnv(envVar); exists { + cleanup = func() { + require.NoError(t, os.Setenv(envVar, original), "failed to reset %s environment variable", envVar) + } + } else { + cleanup = func() { + require.NoError(t, os.Unsetenv(envVar), "failed to reset %s environment variable", envVar) + } + } + + require.NoError(t, os.Setenv(envVar, value), "failed to set %s environment variable", envVar) + + return cleanup +} + +func TestSelectCredentialsFile(t *testing.T) { + testCases := []struct { + name string + config map[string]string + environment string + expected string + }{ + { + name: "when config is empty and environment variable is not set, no file is selected", + expected: "", + }, + { + name: "when config contains credentials file and environment variable is not set, file from config is selected", + config: map[string]string{ + "credentialsFile": "/tmp/credentials/path/to/secret", + }, + expected: "/tmp/credentials/path/to/secret", + }, + { + name: "when config is empty and environment variable is set, file from environment is selected", + environment: "/credentials/file/from/env", + expected: "/credentials/file/from/env", + }, + { + name: "when config contains credentials file and environment variable is set, file from config is selected", + config: map[string]string{ + "credentialsFile": "/tmp/credentials/path/to/secret", + }, + environment: "/credentials/file/from/env", + expected: "/tmp/credentials/path/to/secret", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cleanup := setAzureEnvironment(t, tc.environment) + defer cleanup() + + selectedFile := selectCredentialsFile(tc.config) + require.Equal(t, tc.expected, selectedFile) + }) + } +} diff --git a/pkg/restic/common.go b/pkg/restic/common.go index f01ddbb05..5798f0218 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -17,25 +17,21 @@ limitations under the License. package restic import ( - "context" "fmt" "os" "strings" "time" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/builder" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/util/filesystem" - "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( @@ -66,6 +62,10 @@ const ( // should be excluded from restic backup. VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes" + // credentialsFileKey is the key within a BSL config that is checked to see if + // the BSL is using its own credentials, rather than those in the environment + credentialsFileKey = "credentialsFile" + // Deprecated. // // TODO(2.0): remove @@ -238,42 +238,6 @@ func GetSnapshotsInBackup(backup *velerov1api.Backup, podVolumeBackupLister vele return res, nil } -// TempCredentialsFile creates a temp file containing the restic -// encryption key and returns its path. The caller should generally -// call os.Remove() to remove the file when done with it. -func TempCredentialsFile(client kbclient.Client, veleroNamespace string, fs filesystem.Interface) (string, error) { - // For now, all restic repos share the same key so we don't need the repoName to fetch it. - // When we move to full-backup encryption, we'll likely have a separate key per restic repo - // (all within the Velero server's namespace) so repoKeySelector will need to select the key - // for that repo. - repoKeySelector := builder.ForSecretKeySelector(CredentialsSecretName, CredentialsKey).Result() - - repoKey, err := kube.GetSecretKey(client, veleroNamespace, repoKeySelector) - if err != nil { - return "", err - } - - file, err := fs.TempFile("", fmt.Sprintf("%s-%s", CredentialsSecretName, CredentialsKey)) - if err != nil { - return "", errors.WithStack(err) - } - - if _, err := file.Write(repoKey); err != nil { - // nothing we can do about an error closing the file here, and we're - // already returning an error about the write failing. - _ = file.Close() - return "", errors.WithStack(err) - } - - name := file.Name() - - if err := file.Close(); err != nil { - return "", errors.WithStack(err) - } - - return name, nil -} - // TempCACertFile creates a temp file containing a CA bundle // and returns its path. The caller should generally call os.Remove() // to remove the file when done with it. @@ -299,22 +263,6 @@ func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, return name, nil } -func GetCACert(client kbclient.Client, namespace, backupLocation string) ([]byte, error) { - location := &velerov1api.BackupStorageLocation{} - if err := client.Get(context.Background(), kbclient.ObjectKey{ - Namespace: namespace, - Name: backupLocation, - }, location); err != nil { - return nil, err - } - - if location.Spec.ObjectStorage == nil { - return nil, nil - } - - return location.Spec.ObjectStorage.CACert, nil -} - // NewPodVolumeRestoreListOptions creates a ListOptions with a label selector configured to // find PodVolumeRestores for the restore identified by name. func NewPodVolumeRestoreListOptions(name string) metav1.ListOptions { @@ -323,52 +271,48 @@ func NewPodVolumeRestoreListOptions(name string) metav1.ListOptions { } } -// AzureCmdEnv returns a list of environment variables (in the format var=val) that -// should be used when running a restic command for an Azure backend. This list is -// the current environment, plus the Azure-specific variables restic needs, namely -// a storage account name and key. -func AzureCmdEnv(client kbclient.Client, namespace, backupLocation string) ([]string, error) { - loc := &velerov1api.BackupStorageLocation{} - if err := client.Get(context.Background(), kbclient.ObjectKey{ - Namespace: namespace, - Name: backupLocation, - }, loc); err != nil { - return nil, err - } - - azureVars, err := getAzureResticEnvVars(loc.Spec.Config) - if err != nil { - return nil, errors.Wrap(err, "error getting azure restic env vars") - } - +// CmdEnv returns a list of environment variables (in the format var=val) that +// should be used when running a restic command for a particular backend provider. +// This list is the current environment, plus any provider-specific variables restic needs. +func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { env := os.Environ() - for k, v := range azureVars { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } + customEnv := map[string]string{} + var err error - return env, nil -} + config := backupLocation.Spec.Config + if config == nil { + config = map[string]string{} + } -// S3CmdEnv returns a list of environment variables (in the format var=val) that -// should be used when running a restic command for an S3 backend. This list is -// the current environment, plus the AWS-specific variables restic needs, namely -// a credential profile. -func S3CmdEnv(client kbclient.Client, namespace, backupLocation string) ([]string, error) { - loc := &velerov1api.BackupStorageLocation{} - if err := client.Get(context.Background(), kbclient.ObjectKey{ - Namespace: namespace, - Name: backupLocation, - }, loc); err != nil { - return nil, err + if backupLocation.Spec.Credential != nil { + credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential) + if err != nil { + return []string{}, errors.WithStack(err) + } + config[credentialsFileKey] = credsFile } - awsVars, err := getS3ResticEnvVars(loc.Spec.Config) - if err != nil { - return nil, errors.Wrap(err, "error getting aws restic env vars") + backendType := getBackendType(backupLocation.Spec.Provider) + + switch backendType { + case AWSBackend: + customEnv, err = getS3ResticEnvVars(config) + if err != nil { + return []string{}, err + } + case AzureBackend: + customEnv, err = getAzureResticEnvVars(config) + if err != nil { + return []string{}, err + } + case GCPBackend: + customEnv, err = getGCPResticEnvVars(config) + if err != nil { + return []string{}, err + } } - env := os.Environ() - for k, v := range awsVars { + for k, v := range customEnv { env = append(env, fmt.Sprintf("%s=%s", k, v)) } diff --git a/pkg/restic/common_test.go b/pkg/restic/common_test.go index fe88e3d20..d08d1f076 100644 --- a/pkg/restic/common_test.go +++ b/pkg/restic/common_test.go @@ -17,7 +17,6 @@ limitations under the License. package restic import ( - "context" "os" "sort" "testing" @@ -358,69 +357,19 @@ func TestGetSnapshotsInBackup(t *testing.T) { } } -func TestTempCredentialsFile(t *testing.T) { - var ( - fakeClient = velerotest.NewFakeControllerRuntimeClient(t) - fs = velerotest.NewFakeFileSystem() - secret = &corev1api.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "velero", - Name: CredentialsSecretName, - }, - Data: map[string][]byte{ - CredentialsKey: []byte("passw0rd"), - }, - } - ) - - // secret not in server: expect an error - fileName, err := TempCredentialsFile(fakeClient, "velero", fs) - assert.Error(t, err) - - // now add secret - require.NoError(t, fakeClient.Create(context.Background(), secret)) - - // secret in server: expect temp file to be created with password - fileName, err = TempCredentialsFile(fakeClient, "velero", fs) - require.NoError(t, err) - - contents, err := fs.ReadFile(fileName) - require.NoError(t, err) - - assert.Equal(t, "passw0rd", string(contents)) -} - func TestTempCACertFile(t *testing.T) { var ( - fs = velerotest.NewFakeFileSystem() - bsl = &velerov1api.BackupStorageLocation{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Namespace: "velero", - Name: "default", - }, - Spec: velerov1api.BackupStorageLocationSpec{ - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{CACert: []byte("cacert")}, - }, - }, - } + fs = velerotest.NewFakeFileSystem() + caCertData = []byte("cacert") ) - fakeClient := velerotest.NewFakeControllerRuntimeClient(t) - fakeClient.Create(context.Background(), bsl) - - // expect temp file to be created with cacert value - caCert, err := GetCACert(fakeClient, bsl.Namespace, bsl.Name) - require.NoError(t, err) - - fileName, err := TempCACertFile(caCert, "default", fs) + fileName, err := TempCACertFile(caCertData, "default", fs) require.NoError(t, err) contents, err := fs.ReadFile(fileName) require.NoError(t, err) - assert.Equal(t, "cacert", string(contents)) + assert.Equal(t, string(caCertData), string(contents)) os.Remove(fileName) } diff --git a/pkg/restic/config.go b/pkg/restic/config.go index 771c5448d..452adbc0c 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -55,16 +55,13 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) prefix = layout.GetResticDir() } - var provider = location.Spec.Provider - if !strings.Contains(provider, "/") { - provider = "velero.io/" + provider - } + backendType := getBackendType(location.Spec.Provider) if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" { return repoPrefix, nil } - switch BackendType(provider) { + switch backendType { case AWSBackend: var url string switch { @@ -91,6 +88,14 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config") } +func getBackendType(provider string) BackendType { + if !strings.Contains(provider, "/") { + provider = "velero.io/" + provider + } + + return BackendType(provider) +} + // GetRepoIdentifier returns the string to be used as the value of the --repo flag in // restic commands for the given repository. func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) { diff --git a/pkg/restic/exec_commands.go b/pkg/restic/exec_commands.go index 2d0850139..63e3e1361 100644 --- a/pkg/restic/exec_commands.go +++ b/pkg/restic/exec_commands.go @@ -1,5 +1,5 @@ /* -Copyright 2018 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -44,17 +44,10 @@ type backupStatusLine struct { TotalBytesProcessed int64 `json:"total_bytes_processed"` } -// GetSnapshotID runs a 'restic snapshots' command to get the ID of the snapshot -// in the specified repo matching the set of provided tags, or an error if a -// unique snapshot cannot be identified. -func GetSnapshotID(repoIdentifier, passwordFile string, tags map[string]string, env []string, caCertFile string) (string, error) { - cmd := GetSnapshotCommand(repoIdentifier, passwordFile, tags) - if len(env) > 0 { - cmd.Env = env - } - cmd.CACertFile = caCertFile - - stdout, stderr, err := exec.RunCommand(cmd.Cmd()) +// GetSnapshotID runs provided 'restic snapshots' command to get the ID of a snapshot +// and an error if a unique snapshot cannot be identified. +func GetSnapshotID(snapshotIdCmd *Command) (string, error) { + stdout, stderr, err := exec.RunCommand(snapshotIdCmd.Cmd()) if err != nil { return "", errors.Wrapf(err, "error running command, stderr=%s", stderr) } diff --git a/pkg/restic/gcp.go b/pkg/restic/gcp.go new file mode 100644 index 000000000..96d1edfe6 --- /dev/null +++ b/pkg/restic/gcp.go @@ -0,0 +1,34 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restic + +const ( + // GCP specific environment variable + gcpCredentialsFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS" +) + +// getGCPResticEnvVars gets the environment variables that restic relies +// on based on info in the provided object storage location config map. +func getGCPResticEnvVars(config map[string]string) (map[string]string, error) { + result := make(map[string]string) + + if credentialsFile, ok := config[credentialsFileKey]; ok { + result[gcpCredentialsFileEnvVar] = credentialsFile + } + + return result, nil +} diff --git a/pkg/restic/gcp_test.go b/pkg/restic/gcp_test.go new file mode 100644 index 000000000..37f2bf2c7 --- /dev/null +++ b/pkg/restic/gcp_test.go @@ -0,0 +1,56 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restic + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetGCPResticEnvVars(t *testing.T) { + testCases := []struct { + name string + config map[string]string + expected map[string]string + }{ + { + name: "when config is empty, no env vars are returned", + config: map[string]string{}, + expected: map[string]string{}, + }, + { + name: "when config contains credentials file key, credentials file env var is set with credentials file value", + config: map[string]string{ + "credentialsFile": "/tmp/credentials/path/to/secret", + }, + expected: map[string]string{ + "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/credentials/path/to/secret", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual, err := getGCPResticEnvVars(tc.config) + + require.NoError(t, err) + + require.Equal(t, tc.expected, actual) + }) + } +} diff --git a/pkg/restic/repository_keys.go b/pkg/restic/repository_keys.go index 48f3e381e..28c190f70 100644 --- a/pkg/restic/repository_keys.go +++ b/pkg/restic/repository_keys.go @@ -24,17 +24,19 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/vmware-tanzu/velero/pkg/builder" ) const ( - CredentialsSecretName = "velero-restic-credentials" - CredentialsKey = "repository-password" + credentialsSecretName = "velero-restic-credentials" + credentialsKey = "repository-password" encryptionKey = "static-passw0rd" ) func EnsureCommonRepositoryKey(secretClient corev1client.SecretsGetter, namespace string) error { - _, err := secretClient.Secrets(namespace).Get(context.TODO(), CredentialsSecretName, metav1.GetOptions{}) + _, err := secretClient.Secrets(namespace).Get(context.TODO(), credentialsSecretName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { return errors.WithStack(err) } @@ -47,17 +49,27 @@ func EnsureCommonRepositoryKey(secretClient corev1client.SecretsGetter, namespac secret := &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: CredentialsSecretName, + Name: credentialsSecretName, }, Type: corev1api.SecretTypeOpaque, Data: map[string][]byte{ - CredentialsKey: []byte(encryptionKey), + credentialsKey: []byte(encryptionKey), }, } if _, err = secretClient.Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { - return errors.Wrapf(err, "error creating %s secret", CredentialsSecretName) + return errors.Wrapf(err, "error creating %s secret", credentialsSecretName) } return nil } + +// RepoKeySelector returns the SecretKeySelector which can be used to fetch +// the restic repository key. +func RepoKeySelector() *corev1api.SecretKeySelector { + // For now, all restic repos share the same key so we don't need the repoName to fetch it. + // When we move to full-backup encryption, we'll likely have a separate key per restic repo + // (all within the Velero server's namespace) so RepoKeySelector will need to select the key + // for that repo. + return builder.ForSecretKeySelector(credentialsSecretName, credentialsKey).Result() +} diff --git a/pkg/restic/repository_keys_test.go b/pkg/restic/repository_keys_test.go new file mode 100644 index 000000000..6af6641ce --- /dev/null +++ b/pkg/restic/repository_keys_test.go @@ -0,0 +1,30 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restic + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRepoKeySelector(t *testing.T) { + selector := RepoKeySelector() + + require.Equal(t, credentialsSecretName, selector.Name) + require.Equal(t, credentialsKey, selector.Key) +} diff --git a/pkg/restic/repository_manager.go b/pkg/restic/repository_manager.go index 68766c356..b4282111e 100644 --- a/pkg/restic/repository_manager.go +++ b/pkg/restic/repository_manager.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "os" - "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -30,6 +29,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -80,18 +80,19 @@ type RestorerFactory interface { } type repositoryManager struct { - namespace string - veleroClient clientset.Interface - repoLister velerov1listers.ResticRepositoryLister - repoInformerSynced cache.InformerSynced - kbClient kbclient.Client - log logrus.FieldLogger - repoLocker *repoLocker - repoEnsurer *repositoryEnsurer - fileSystem filesystem.Interface - ctx context.Context - pvcClient corev1client.PersistentVolumeClaimsGetter - pvClient corev1client.PersistentVolumesGetter + namespace string + veleroClient clientset.Interface + repoLister velerov1listers.ResticRepositoryLister + repoInformerSynced cache.InformerSynced + kbClient kbclient.Client + log logrus.FieldLogger + repoLocker *repoLocker + repoEnsurer *repositoryEnsurer + fileSystem filesystem.Interface + ctx context.Context + pvcClient corev1client.PersistentVolumeClaimsGetter + pvClient corev1client.PersistentVolumesGetter + credentialsFileStore credentials.FileStore } // NewRepositoryManager constructs a RepositoryManager. @@ -104,18 +105,20 @@ func NewRepositoryManager( kbClient kbclient.Client, pvcClient corev1client.PersistentVolumeClaimsGetter, pvClient corev1client.PersistentVolumesGetter, + credentialFileStore credentials.FileStore, log logrus.FieldLogger, ) (RepositoryManager, error) { rm := &repositoryManager{ - namespace: namespace, - veleroClient: veleroClient, - repoLister: repoInformer.Lister(), - repoInformerSynced: repoInformer.Informer().HasSynced, - kbClient: kbClient, - pvcClient: pvcClient, - pvClient: pvClient, - log: log, - ctx: ctx, + namespace: namespace, + veleroClient: veleroClient, + repoLister: repoInformer.Lister(), + repoInformerSynced: repoInformer.Informer().HasSynced, + kbClient: kbClient, + pvcClient: pvcClient, + pvClient: pvClient, + credentialsFileStore: credentialFileStore, + log: log, + ctx: ctx, repoLocker: newRepoLocker(), repoEnsurer: newRepositoryEnsurer(repoInformer, repoClient, log), @@ -227,7 +230,7 @@ func (rm *repositoryManager) Forget(ctx context.Context, snapshot SnapshotIdenti } func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { - file, err := TempCredentialsFile(rm.kbClient, rm.namespace, rm.fileSystem) + file, err := rm.credentialsFileStore.Path(RepoKeySelector()) if err != nil { return err } @@ -236,36 +239,31 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { cmd.PasswordFile = file - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := GetCACert(rm.kbClient, rm.namespace, backupLocation) - if err != nil { - return err + loc := &velerov1api.BackupStorageLocation{} + if err := rm.kbClient.Get(context.Background(), kbclient.ObjectKey{ + Namespace: rm.namespace, + Name: backupLocation, + }, loc); err != nil { + return errors.Wrap(err, "error getting backup storage location") } + // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic var caCertFile string - if caCert != nil { - caCertFile, err = TempCACertFile(caCert, backupLocation, rm.fileSystem) + if loc.Spec.ObjectStorage != nil && loc.Spec.ObjectStorage.CACert != nil { + caCertFile, err = TempCACertFile(loc.Spec.ObjectStorage.CACert, backupLocation, rm.fileSystem) if err != nil { - return err + return errors.Wrap(err, "error creating temp cacert file") } // ignore error since there's nothing we can do and it's a temp file. defer os.Remove(caCertFile) } cmd.CACertFile = caCertFile - if strings.HasPrefix(cmd.RepoIdentifier, "azure") { - env, err := AzureCmdEnv(rm.kbClient, rm.namespace, backupLocation) - if err != nil { - return err - } - cmd.Env = env - } else if strings.HasPrefix(cmd.RepoIdentifier, "s3") { - env, err := S3CmdEnv(rm.kbClient, rm.namespace, backupLocation) - if err != nil { - return err - } - cmd.Env = env + env, err := CmdEnv(loc, rm.credentialsFileStore) + if err != nil { + return err } + cmd.Env = env stdout, stderr, err := veleroexec.RunCommand(cmd.Cmd()) rm.log.WithFields(logrus.Fields{ diff --git a/site/content/docs/main/api-types/backupstoragelocation.md b/site/content/docs/main/api-types/backupstoragelocation.md index 9a8df1535..b6c58ece7 100644 --- a/site/content/docs/main/api-types/backupstoragelocation.md +++ b/site/content/docs/main/api-types/backupstoragelocation.md @@ -22,6 +22,9 @@ spec: provider: aws objectStorage: bucket: myBucket + credential: + name: secret-name + key: key-in-secret config: region: us-west-2 profile: "default" @@ -45,4 +48,7 @@ The configurable parameters are as follows: | `accessMode` | String | `ReadWrite` | How Velero can access the backup storage location. Valid values are `ReadWrite`, `ReadOnly`. | | `backupSyncPeriod` | metav1.Duration | Optional Field | How frequently Velero should synchronize backups in object storage. Default is Velero's server backup sync period. Set this to `0s` to disable sync. | | `validationFrequency` | metav1.Duration | Optional Field | How frequently Velero should validate the object storage . Default is Velero's server validation frequency. Set this to `0s` to disable validation. Default 1 minute. | +| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. | +| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. | +| `credential/key` | String | Optional Field | The key to use within the secret. | {{< /table >}} diff --git a/site/content/docs/main/locations.md b/site/content/docs/main/locations.md index 917d6d96c..74dce9927 100644 --- a/site/content/docs/main/locations.md +++ b/site/content/docs/main/locations.md @@ -16,12 +16,18 @@ The user can pre-configure one or more possible `BackupStorageLocations` and one This configuration design enables a number of different use cases, including: - Take snapshots of more than one kind of persistent volume in a single Velero backup. For example, in a cluster with both EBS volumes and Portworx volumes -- Have some Velero backups go to a bucket in an eastern USA region, and others go to a bucket in a western USA region +- Have some Velero backups go to a bucket in an eastern USA region, and others go to a bucket in a western USA region, or to a different storage provider - For volume providers that support it, like Portworx, you can have some snapshots stored locally on the cluster and have others stored in the cloud ## Limitations / Caveats -- Velero only supports a single set of credentials *per provider*. It's not yet possible to use different credentials for different locations, if they're for the same provider. +- Velero supports multiple credentials for `BackupStorageLocations`, allowing you to specify the credentials to use with any `BackupStorageLocation`. + However, use of this feature requires support within the plugin for the object storage provider you wish to use. + All [plugins maintained by the Velero team][5] support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this feature is supported. + +- Velero only supports a single set of credentials for `VolumeSnapshotLocations`. + Velero will always use the credentials provided at install time (stored in the `cloud-credentials` secret) for volume snapshots. - Volume snapshots are still limited by where your provider allows you to create snapshots. For example, AWS and Azure do not allow you to create a volume snapshot in a different region than where the volume is. If you try to take a Velero backup using a volume snapshot location with a different region than where your cluster's volumes are, the backup will fail. @@ -66,6 +72,10 @@ velero backup create full-cluster-backup ### Have some Velero backups go to a bucket in an eastern USA region (default), and others go to a bucket in a western USA region +In this example, two `BackupStorageLocations` will be created within the same account but in different regions. +They will both use the credentials provided at install time and stored in the `cloud-credentials` secret. +If you need to configure unique credentials for each `BackupStorageLocation`, please refer to the [later example][8]. + During server configuration: ```shell @@ -166,6 +176,68 @@ During backup creation: velero backup create full-cluster-backup ``` +### Create a storage location that uses unique credentials + +It is possible to create additional `BackupStorageLocations` that use their own credentials. +This enables you to save backups to another storage provider or to another account with the storage provider you are already using. + +If you create additional `BackupStorageLocations` without specifying the credentials to use, Velero will use the credentials provided at install time and stored in the `cloud-credentials` secret. +Please see the [earlier example][9] for details on how to create multiple `BackupStorageLocations` that use the same credentials. + +#### Prerequisites +- This feature requires support from the [object storage provider plugin][5] you wish to use. + All plugins maintained by the Velero team support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this is supported. +- The [plugin for the object storage provider][5] you wish to use must be [installed][6]. +- You must create a file with the object storage credentials. Follow the instructions provided by your object storage provider plugin to create this file. + +Once you have installed the necessary plugin and created the credentials file, create a [Kubernetes Secret][6] in the Velero namespace that contains these credentials: + +```shell +kubectl create secret generic -n velero credentials --from-file=bsl= +``` + +This will create a secret named `credentials` with a single key (`bsl`) which contains the contents of your credentials file. +Next, create a `BackupStorageLocation` that uses this Secret by passing the Secret name and key in the `--credential` flag. +When interacting with this `BackupStroageLocation` in the future, Velero will fetch the data from the key within the Secret you provide. + +For example, a new `BackupStorageLocation` with a Secret would be configured as follows: + +```bash +velero backup-location create \ + --provider \ + --bucket \ + --config region= \ + --credential== +``` + +The `BackupStorageLocation` is ready to use when it has the phase `Available`. +You can check the status with the following command: + +```bash +velero backup-location get +``` + +To use this new `BackupStorageLocation` when performing a backup, use the flag `--storage-location ` when running `velero backup create`. +You may also set this new `BackupStorageLocation` as the default with the command `velero backup-location set --default `. + +### Modify the credentials used by an existing storage location + +By default, `BackupStorageLocations` will use the credentials provided at install time and stored in the `cloud-credentials` secret in the Velero namespace. +You can modify these existing credentials by [editing the `cloud-credentials` secret][10], however, these changes will apply to all locations using this secret. +This may be the desired outcome, for example, in the case where you wish to rotate the credentials used for a particular account. + +You can also opt to modify an existing `BackupStorageLocation` such that it uses its own credentials by using the `backup-location set` command. + +If you have a credentials file that you wish to use for a `BackupStorageLocation`, follow the instructions above to create the Secret with that file in the Velero namespace. + +Once you have created the Secret, or have an existing Secret which contains the credentials you wish to use for your `BackupStorageLocation`, set the credential to use as follows: + +```bash +velero backup-location set \ + --credential== +``` + ## Additional Use Cases 1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details. @@ -178,3 +250,9 @@ velero backup create full-cluster-backup [2]: api-types/volumesnapshotlocation.md [3]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/volumesnapshotlocation.md [4]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md +[5]: /plugins +[6]: overview-plugins.md +[7]: https://kubernetes.io/docs/concepts/configuration/secret/ +[8]: #create-a-storage-location-that-uses-unique-credentials +[9]: #have-some-velero-backups-go-to-a-bucket-in-an-eastern-usa-region-default-and-others-go-to-a-bucket-in-a-western-usa-region +[10]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret diff --git a/site/content/docs/main/troubleshooting.md b/site/content/docs/main/troubleshooting.md index 1328e151d..7b035006f 100644 --- a/site/content/docs/main/troubleshooting.md +++ b/site/content/docs/main/troubleshooting.md @@ -106,11 +106,17 @@ Now, visiting http://localhost:8085/metrics on a browser should show the metrics ## Is Velero using the correct cloud credentials? -Cloud provider credentials are given to Velero to store and retrieve backups from the object store and to perform volume snapshotting operations. These credentials are passed to Velero at install time either using: +Cloud provider credentials are given to Velero to store and retrieve backups from the object store and to perform volume snapshotting operations. + +These credentials are either passed to Velero at install time using: 1. `--secret-file` flag to the `velero install` command. OR 1. `--set-file credentials.secretContents.cloud` flag to the `helm install` command. -The supplied credentials are stored in the cluster as a Kubernetes secret named `cloud-credentials` in the same namespace in which Velero is installed. +Or, they are specified when creating a `BackupStorageLocation` using the `--credential` flag. + +### Troubleshooting credentials provided during install + +If using the credentials provided at install time, they are stored in the cluster as a Kubernetes secret named `cloud-credentials` in the same namespace in which Velero is installed. Follow the below troubleshooting steps to confirm that Velero is using the correct credentials: 1. Confirm that the `cloud-credentials` secret exists and has the correct content. @@ -168,6 +174,36 @@ Follow the below troubleshooting steps to confirm that Velero is using the corre ``` +### Troubleshooting `BackupStorageLocation` credentials + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation`][10]: +1. Confirm that the object storage provider plugin being used supports multiple credentials. + + If the logs from the Velero deployment contain the error message `"config has invalid keys credentialsFile"`, the version of your object storage plugin does not yet support multiple credentials. + + The object storage plugins [maintained by the Velero team][11] support this feature, so please update your plugin to the latest version if you see the above error message. + + If you are using a plugin from a different provider, please contact them for further advice. + +1. Confirm that the secret and key referenced by the `BackupStorageLocation` exists in the Velero namespace and has the correct content: + ```bash + # Determine which secret and key the BackupStorageLocation is using + BSL_SECRET=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.name}) + BSL_SECRET_KEY=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.key}) + + # Confirm that the secret exists + kubectl -n velero get secret $BSL_SECRET + + # Print the content of the secret and ensure it is correct + kubectl -n velero get secret $BSL_SECRET -ojsonpath={.data.$BSL_SECRET_KEY} | base64 --decode + ``` + If the secret can't be found, the secret does not exist within the Velero namespace and must be created. + + If no output is produced when printing the contents of the secret, the key within the secret may not exist or may have no content. + Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET`. + If it does not exist, follow the instructions for [editing a Kubernetes secret][12] to add the base64 encoded credentials data. + + [1]: debugging-restores.md [2]: debugging-install.md [3]: restic.md @@ -177,4 +213,7 @@ Follow the below troubleshooting steps to confirm that Velero is using the corre [7]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L44 [8]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L49-L52 [9]: https://kubectl.docs.kubernetes.io/pages/container_debugging/port_forward_to_pods.html +[10]: locations.md +[11]: /plugins +[12]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret [25]: https://kubernetes.slack.com/messages/velero