diff --git a/changelogs/unreleased/1895-boxcee b/changelogs/unreleased/1895-boxcee new file mode 100644 index 000000000..cc25a7e38 --- /dev/null +++ b/changelogs/unreleased/1895-boxcee @@ -0,0 +1 @@ +Azure: add support for cross-subscription backups diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 95f5e68da..27fc8f8ef 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -35,6 +35,7 @@ import ( const ( storageAccountConfigKey = "storageAccount" + subscriptionIdConfigKey = "subscriptionId" ) type containerGetter interface { @@ -146,22 +147,28 @@ func getStorageAccountKey(config map[string]string) (string, error) { return "", errors.Wrap(err, "unable to get all required environment variables") } - // 2. we need config["resourceGroup"], config["storageAccount"] + // 2. check whether a different subscription ID was set for backups in config["subscriptionId"] + subscriptionId := envVars[subscriptionIDEnvVar] + if val := config[subscriptionIdConfigKey]; val != "" { + subscriptionId = val + } + + // 3. we need config["resourceGroup"], config["storageAccount"] if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { return "", errors.Wrap(err, "unable to get all required config values") } - // 3. get SPT + // 4. get SPT spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) if err != nil { return "", errors.Wrap(err, "error getting service principal token") } - // 4. get storageAccountsClient - storageAccountsClient := storagemgmt.NewAccountsClient(envVars[subscriptionIDEnvVar]) + // 5. get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClient(subscriptionId) storageAccountsClient.Authorizer = autorest.NewBearerAuthorizer(spt) - // 5. get storage key + // 6. get storage key res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey]) if err != nil { return "", errors.WithStack(err) @@ -194,7 +201,11 @@ func mapLookup(data map[string]string) func(string) string { } func (o *ObjectStore) Init(config map[string]string) error { - if err := cloudprovider.ValidateObjectStoreConfigKeys(config, resourceGroupConfigKey, storageAccountConfigKey); err != nil { + if err := cloudprovider.ValidateObjectStoreConfigKeys(config, + resourceGroupConfigKey, + storageAccountConfigKey, + subscriptionIdConfigKey, + ); err != nil { return err } diff --git a/pkg/cloudprovider/azure/volume_snapshotter.go b/pkg/cloudprovider/azure/volume_snapshotter.go index d819f892b..10f426f72 100644 --- a/pkg/cloudprovider/azure/volume_snapshotter.go +++ b/pkg/cloudprovider/azure/volume_snapshotter.go @@ -51,7 +51,8 @@ type VolumeSnapshotter struct { log logrus.FieldLogger disks *disk.DisksClient snaps *disk.SnapshotsClient - subscription string + disksSubscription string + snapsSubscription string disksResourceGroup string snapsResourceGroup string apiTimeout time.Duration @@ -72,7 +73,7 @@ func NewVolumeSnapshotter(logger logrus.FieldLogger) *VolumeSnapshotter { } func (b *VolumeSnapshotter) Init(config map[string]string) error { - if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey); err != nil { + if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey, subscriptionIdConfigKey); err != nil { return err } @@ -87,7 +88,17 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { return errors.Wrap(err, "unable to get all required environment variables") } - // 2. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it + // 2. set a different subscriptionId for snapshots if specified + snapshotsSubscriptionId := envVars[subscriptionIDEnvVar] + if val := config[subscriptionIdConfigKey]; val != "" { + // if subscription was set in config, it is required to also set the resource group + if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey); err != nil { + return errors.Wrap(err, "resourceGroup not specified, but is a requirement when backing up to a different subscription") + } + snapshotsSubscriptionId = val + } + + // 3. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it var apiTimeout time.Duration if val := config[apiTimeoutConfigKey]; val == "" { apiTimeout = 2 * time.Minute @@ -98,15 +109,15 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { } } - // 3. get SPT + // 4. get SPT spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) if err != nil { return errors.Wrap(err, "error getting service principal token") } - // 4. set up clients + // 5. set up clients disksClient := disk.NewDisksClient(envVars[subscriptionIDEnvVar]) - snapsClient := disk.NewSnapshotsClient(envVars[subscriptionIDEnvVar]) + snapsClient := disk.NewSnapshotsClient(snapshotsSubscriptionId) disksClient.PollingDelay = 5 * time.Second snapsClient.PollingDelay = 5 * time.Second @@ -117,7 +128,8 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { b.disks = &disksClient b.snaps = &snapsClient - b.subscription = envVars[subscriptionIDEnvVar] + b.disksSubscription = envVars[subscriptionIDEnvVar] + b.snapsSubscription = snapshotsSubscriptionId b.disksResourceGroup = envVars[resourceGroupEnvVar] b.snapsResourceGroup = config[resourceGroupConfigKey] @@ -205,7 +217,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s return "", errors.WithStack(err) } - fullDiskName := getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID) + fullDiskName := getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID) // snapshot names must be <= 80 characters long var snapshotName string suffix := "-" + uuid.NewV4().String() @@ -242,7 +254,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s return "", errors.WithStack(err) } - return getComputeResourceName(b.subscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil + return getComputeResourceName(b.snapsSubscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil } func getSnapshotTags(veleroTags map[string]string, diskTags map[string]*string) map[string]*string { @@ -374,7 +386,7 @@ func (b *VolumeSnapshotter) SetVolumeID(unstructuredPV runtime.Unstructured, vol } pv.Spec.AzureDisk.DiskName = volumeID - pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID) + pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID) res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) if err != nil { diff --git a/pkg/cloudprovider/azure/volume_snapshotter_test.go b/pkg/cloudprovider/azure/volume_snapshotter_test.go index 1df1ba7d8..f01085040 100644 --- a/pkg/cloudprovider/azure/volume_snapshotter_test.go +++ b/pkg/cloudprovider/azure/volume_snapshotter_test.go @@ -57,7 +57,7 @@ func TestGetVolumeID(t *testing.T) { func TestSetVolumeID(t *testing.T) { b := &VolumeSnapshotter{ disksResourceGroup: "rg", - subscription: "sub", + disksSubscription: "sub", } pv := &unstructured.Unstructured{ diff --git a/site/docs/master/api-types/backupstoragelocation.md b/site/docs/master/api-types/backupstoragelocation.md index 7c1b16f1f..4e907ca20 100644 --- a/site/docs/master/api-types/backupstoragelocation.md +++ b/site/docs/master/api-types/backupstoragelocation.md @@ -64,6 +64,7 @@ The configurable parameters are as follows: | --- | --- | --- | --- | | `resourceGroup` | string | Required Field | Name of the resource group containing the storage account for this backup storage location. | | `storageAccount` | string | Required Field | Name of the storage account for this backup storage location. | +| `subscriptionId` | string | Optional Field | ID of the subscription for this backup storage location. | #### GCP diff --git a/site/docs/master/api-types/volumesnapshotlocation.md b/site/docs/master/api-types/volumesnapshotlocation.md index 8e6ad15c2..f6fcd0d4b 100644 --- a/site/docs/master/api-types/volumesnapshotlocation.md +++ b/site/docs/master/api-types/volumesnapshotlocation.md @@ -51,6 +51,7 @@ The configurable parameters are as follows: | --- | --- | --- | --- | | `apiTimeout` | metav1.Duration | 2m0s | How long to wait for an Azure API request to complete before timeout. | | `resourceGroup` | string | Optional | The name of the resource group where volume snapshots should be stored, if different from the cluster's resource group. | +| `subscriptionId` | string | Optional | The ID of the subscription where volume snapshots should be stored, if different from the cluster's subscription. Requires `resourceGroup`to be set. #### GCP diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 02f56381c..97082f66f 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -38,6 +38,29 @@ of the Velero repository is under active development and is not guaranteed to be 1. Move the `velero` binary from the Velero directory to somewhere in your PATH. +## (Optional) Change to the Azure subscription you want to create your backups in + +By default, Velero will store backups in the same Subscription as your VMs and disks and will +not allow you to restore backups to a Resource Group in a different Subscription. To enable backups/restore +across Subscriptions you will need to specify the Subscription ID to backup to. + +Use `az` to switch to the Subscription the backups should be created in. + +First, find the Subscription ID by name. + +```bash +AZURE_BACKUP_SUBSCRIPTION_NAME= +AZURE_BACKUP_SUBSCRIPTION_ID=$(az account list --query="[?name=='$AZURE_BACKUP_SUBSCRIPTION_NAME'].id | [0]" -o tsv) +``` + +Second, change the Subscription. + +```bash +az account set -s $AZURE_BACKUP_SUBSCRIPTION_ID +``` + +Execute the next step – creating an storage account and blob container – using the active Subscription. + ## Create Azure storage account and blob container Velero requires a storage account and blob container in which to store backups. @@ -82,6 +105,9 @@ az storage container create -n $BLOB_CONTAINER --public-access off --account-nam ## Get resource group for persistent volume snapshots +_(Optional) If you decided to backup to a different Subscription, make sure you change back to the Subscription +of your cluster's resources before continuing._ + 1. Set the name of the Resource Group that contains your Kubernetes cluster's virtual machines/disks. **WARNING**: If you're using [AKS][22], `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created @@ -115,9 +141,13 @@ To integrate Velero with Azure, you must create a Velero-specific [service princ If you'll be using Velero to backup multiple clusters with multiple blob containers, it may be desirable to create a unique username per cluster rather than the default `velero`. Create service principal and let the CLI generate a password for you. Make sure to capture the password. + + _(Optional) If you are using a different Subscription for backups and cluster resources, make sure to specify both subscriptions + in the `az` command using `--scopes`._ ```bash - AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv` + AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv \ + [--scopes /subscriptions/$AZURE_BACKUP_SUBSCRIPTION_ID /subscriptions/$AZURE_SUBSCRIPTION_ID]` ``` After creating the service principal, obtain the client id. @@ -147,8 +177,8 @@ velero install \ --provider azure \ --bucket $BLOB_CONTAINER \ --secret-file ./credentials-velero \ - --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \ - --snapshot-location-config apiTimeout= + --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ + --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. diff --git a/site/docs/master/locations.md b/site/docs/master/locations.md index 7c7a0b9c9..8c55e5a36 100644 --- a/site/docs/master/locations.md +++ b/site/docs/master/locations.md @@ -154,7 +154,7 @@ velero backup create full-cluster-backup 1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details. -1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount` and/or `resourceGroup`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details. +1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details.