Add support for GKE Workload Identity (#1810)

* Allow the velero server to be created on GCP even without a provided service account key in order to support workload identity and default compute engine credentials. Add option for adding service account annotations.

Signed-off-by: Joshua Wong <joshua99wong@gmail.com>

Gopkg.lock

@@ -584,7 +584,7 @@
   revision = "26559e0f760e39c24d730d3224364aef164ee23f"
 
 [[projects]]
-  digest = "1:352a8b8a41fd11320b8b2327a4c2c8f6967578a2d54886b64748c0a46d4a8b5e"
+  digest = "1:98ef43d760c0123cf289071e1043176178c60205902a60beba99bb9f988fdcf9"
   name = "google.golang.org/api"
   packages = [
     "compute/v1",
@@ -592,6 +592,7 @@
     "googleapi",
     "googleapi/internal/uritemplates",
     "googleapi/transport",
+    "iamcredentials/v1",
     "internal",
     "iterator",
     "option",
@@ -1102,6 +1103,7 @@
     "golang.org/x/oauth2/google",
     "google.golang.org/api/compute/v1",
     "google.golang.org/api/googleapi",
+    "google.golang.org/api/iamcredentials/v1",
     "google.golang.org/api/iterator",
     "google.golang.org/api/option",
     "google.golang.org/grpc",

pkg/cloudprovider/gcp/object_store.go

@@ -18,15 +18,15 @@ package gcp
 
 import (
 	"context"
+	"encoding/base64"
 	"io"
-	"io/ioutil"
-	"os"
 	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2/google"
+	"google.golang.org/api/iamcredentials/v1"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 
@@ -34,8 +34,9 @@ import (
 )
 
 const (
-	credentialsEnvVar   = "GOOGLE_APPLICATION_CREDENTIALS"
-	kmsKeyNameConfigKey = "kmsKeyName"
+	credentialsEnvVar    = "GOOGLE_APPLICATION_CREDENTIALS"
+	kmsKeyNameConfigKey  = "kmsKeyName"
+	serviceAccountConfig = "serviceAccount"
 )
 
 // bucketWriter wraps the GCP SDK functions for accessing object store so they can be faked for testing.
@@ -67,6 +68,7 @@ type ObjectStore struct {
 	googleAccessID string
 	privateKey     []byte
 	bucketWriter   bucketWriter
+	iamSvc         *iamcredentials.Service
 }
 
 func NewObjectStore(logger logrus.FieldLogger) *ObjectStore {
@@ -74,35 +76,30 @@ func NewObjectStore(logger logrus.FieldLogger) *ObjectStore {
 }
 
 func (o *ObjectStore) Init(config map[string]string) error {
-	if err := cloudprovider.ValidateObjectStoreConfigKeys(config, kmsKeyNameConfigKey); err != nil {
+	if err := cloudprovider.ValidateObjectStoreConfigKeys(config, kmsKeyNameConfigKey, serviceAccountConfig); err != nil {
 		return err
 	}
+	// Find default token source to extract the GoogleAccessID
+	ctx := context.Background()
+	creds, err := google.FindDefaultCredentials(ctx)
 
-	credentialsFile := os.Getenv(credentialsEnvVar)
-	if credentialsFile == "" {
-		return errors.Errorf("%s is undefined", credentialsEnvVar)
-	}
-
-	// Get the email and private key from the credentials file so we can pre-sign download URLs
-	creds, err := ioutil.ReadFile(credentialsFile)
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	jwtConfig, err := google.JWTConfigFromJSON(creds)
+
+	if creds.JSON != nil {
+		// Using Credentials File
+		err = o.initFromKeyFile(creds)
+	} else {
+		// Using compute engine credentials. Use this if workload identity is enabled.
+		err = o.initFromComputeEngine(config)
+	}
+
 	if err != nil {
-		return errors.Wrap(err, "error parsing credentials file; should be JSON")
-	}
-	if jwtConfig.Email == "" {
-		return errors.Errorf("credentials file pointed to by %s does not contain an email", credentialsEnvVar)
-	}
-	if len(jwtConfig.PrivateKey) == 0 {
-		return errors.Errorf("credentials file pointed to by %s does not contain a private key", credentialsEnvVar)
+		return errors.WithStack(err)
 	}
 
-	o.googleAccessID = jwtConfig.Email
-	o.privateKey = jwtConfig.PrivateKey
-
-	client, err := storage.NewClient(context.Background(), option.WithScopes(storage.ScopeReadWrite))
+	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeReadWrite))
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -112,10 +109,37 @@ func (o *ObjectStore) Init(config map[string]string) error {
 		client:     o.client,
 		kmsKeyName: config[kmsKeyNameConfigKey],
 	}
-
 	return nil
 }
 
+func (o *ObjectStore) initFromKeyFile(creds *google.Credentials) error {
+	jwtConfig, err := google.JWTConfigFromJSON(creds.JSON)
+	if err != nil {
+		return errors.Wrap(err, "error parsing credentials file; should be JSON")
+	}
+	if jwtConfig.Email == "" {
+		return errors.Errorf("credentials file pointed to by %s does not contain an email", "GOOGLE_APPLICATION_CREDENTIALS")
+	}
+	if len(jwtConfig.PrivateKey) == 0 {
+		return errors.Errorf("credentials file pointed to by %s does not contain a private key", "GOOGLE_APPLICATION_CREDENTIALS")
+	}
+
+	o.googleAccessID = jwtConfig.Email
+	o.privateKey = jwtConfig.PrivateKey
+	return nil
+}
+
+func (o *ObjectStore) initFromComputeEngine(config map[string]string) error {
+	var err error
+	var ok bool
+	o.googleAccessID, ok = config["serviceAccount"]
+	if !ok {
+		return errors.Errorf("serviceAccount is expected to be provided as an item in BackupStorageLocation's config")
+	}
+	o.iamSvc, err = iamcredentials.NewService(context.Background())
+	return err
+}
+
 func (o *ObjectStore) PutObject(bucket, key string, body io.Reader) error {
 	w := o.bucketWriter.getWriteCloser(bucket, key)
 
@@ -204,11 +228,34 @@ func (o *ObjectStore) DeleteObject(bucket, key string) error {
 	return errors.Wrapf(o.client.Bucket(bucket).Object(key).Delete(context.Background()), "error deleting object %s", key)
 }
 
+/*
+ * Use the iamSignBlob api call to sign the url if there is no credentials file to get the key from.
+ * https://cloud.google.com/iam/credentials/reference/rest/v1/projects.serviceAccounts/signBlob
+ */
+func (o *ObjectStore) SignBytes(bytes []byte) ([]byte, error) {
+	name := "projects/-/serviceAccounts/" + o.googleAccessID
+	resp, err := o.iamSvc.Projects.ServiceAccounts.SignBlob(name, &iamcredentials.SignBlobRequest{
+		Payload: base64.StdEncoding.EncodeToString(bytes),
+	}).Context(context.Background()).Do()
+
+	if err != nil {
+		return nil, err
+	}
+	return base64.StdEncoding.DecodeString(resp.SignedBlob)
+}
+
 func (o *ObjectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
-	return storage.SignedURL(bucket, key, &storage.SignedURLOptions{
+	options := storage.SignedURLOptions{
 		GoogleAccessID: o.googleAccessID,
-		PrivateKey:     o.privateKey,
 		Method:         "GET",
 		Expires:        time.Now().Add(ttl),
-	})
+	}
+
+	if o.privateKey == nil {
+		options.SignBytes = o.SignBytes
+	} else {
+		options.PrivateKey = o.privateKey
+	}
+
+	return storage.SignedURL(bucket, key, &options)
 }

pkg/cloudprovider/gcp/volume_snapshotter.go

@@ -18,9 +18,7 @@ package gcp
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -60,13 +58,14 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 		return err
 	}
 
+	/* Works with both credential files and the default compute engine service account */
+	creds, err := google.FindDefaultCredentials(oauth2.NoContext, compute.ComputeScope)
+	if err != nil {
+		return errors.WithStack(err)
+	}
 	b.snapshotLocation = config[snapshotLocationKey]
 
-	project, err := extractProjectFromCreds()
-	if err != nil {
-		return err
-	}
-	b.volumeProject = project
+	b.volumeProject = creds.ProjectID
 
 	// get snapshot project from 'project' config key if specified,
 	// otherwise from the credentials file
@@ -74,11 +73,7 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 	if b.snapshotProject == "" {
 		b.snapshotProject = b.volumeProject
 	}
-
-	client, err := google.DefaultClient(oauth2.NoContext, compute.ComputeScope)
-	if err != nil {
-		return errors.WithStack(err)
-	}
+	client := oauth2.NewClient(oauth2.NoContext, creds.TokenSource)
 
 	gce, err := compute.New(client)
 	if err != nil {
@@ -90,28 +85,6 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 	return nil
 }
 
-func extractProjectFromCreds() (string, error) {
-	credsBytes, err := ioutil.ReadFile(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	type credentials struct {
-		ProjectID string `json:"project_id"`
-	}
-
-	var creds credentials
-	if err := json.Unmarshal(credsBytes, &creds); err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	if creds.ProjectID == "" {
-		return "", errors.New("cannot fetch project_id from GCP credentials file")
-	}
-
-	return creds.ProjectID, nil
-}
-
 // isMultiZone returns true if the failure-domain tag contains
 // double underscore, which is the separator used
 // by GKE when a storage class spans multiple availablity

pkg/cmd/cli/install/install.go

@@ -45,6 +45,7 @@ type InstallOptions struct {
 	Prefix                            string
 	ProviderName                      string
 	PodAnnotations                    flag.Map
+	ServiceAccountAnnotations         flag.Map
 	VeleroPodCPURequest               string
 	VeleroPodMemRequest               string
 	VeleroPodCPULimit                 string
@@ -74,6 +75,7 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.Image, "image", o.Image, "image to use for the Velero and restic server pods. Optional.")
 	flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Velero data should be stored within the bucket. Optional.")
 	flags.Var(&o.PodAnnotations, "pod-annotations", "annotations to add to the Velero and restic pods. Optional. Format is key1=value1,key2=value2")
+	flags.Var(&o.ServiceAccountAnnotations, "sa-annotations", "annotations to add to the Velero ServiceAccount. Add iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_NAME].iam.gserviceaccount.com for workload identity. Optional. Format is key1=value1,key2=value2")
 	flags.StringVar(&o.VeleroPodCPURequest, "velero-pod-cpu-request", o.VeleroPodCPURequest, `CPU request for Velero pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.VeleroPodMemRequest, "velero-pod-mem-request", o.VeleroPodMemRequest, `memory request for Velero pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.VeleroPodCPULimit, "velero-pod-cpu-limit", o.VeleroPodCPULimit, `CPU limit for Velero pod. A value of "0" is treated as unbounded. Optional.`)
@@ -95,19 +97,20 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 // NewInstallOptions instantiates a new, default InstallOptions struct.
 func NewInstallOptions() *InstallOptions {
 	return &InstallOptions{
-		Namespace:            velerov1api.DefaultNamespace,
-		Image:                install.DefaultImage,
-		BackupStorageConfig:  flag.NewMap(),
-		VolumeSnapshotConfig: flag.NewMap(),
-		PodAnnotations:       flag.NewMap(),
-		VeleroPodCPURequest:  install.DefaultVeleroPodCPURequest,
-		VeleroPodMemRequest:  install.DefaultVeleroPodMemRequest,
-		VeleroPodCPULimit:    install.DefaultVeleroPodCPULimit,
-		VeleroPodMemLimit:    install.DefaultVeleroPodMemLimit,
-		ResticPodCPURequest:  install.DefaultResticPodCPURequest,
-		ResticPodMemRequest:  install.DefaultResticPodMemRequest,
-		ResticPodCPULimit:    install.DefaultResticPodCPULimit,
-		ResticPodMemLimit:    install.DefaultResticPodMemLimit,
+		Namespace:                 velerov1api.DefaultNamespace,
+		Image:                     install.DefaultImage,
+		BackupStorageConfig:       flag.NewMap(),
+		VolumeSnapshotConfig:      flag.NewMap(),
+		PodAnnotations:            flag.NewMap(),
+		ServiceAccountAnnotations: flag.NewMap(),
+		VeleroPodCPURequest:       install.DefaultVeleroPodCPURequest,
+		VeleroPodMemRequest:       install.DefaultVeleroPodMemRequest,
+		VeleroPodCPULimit:         install.DefaultVeleroPodCPULimit,
+		VeleroPodMemLimit:         install.DefaultVeleroPodMemLimit,
+		ResticPodCPURequest:       install.DefaultResticPodCPURequest,
+		ResticPodMemRequest:       install.DefaultResticPodMemRequest,
+		ResticPodCPULimit:         install.DefaultResticPodCPULimit,
+		ResticPodMemLimit:         install.DefaultResticPodMemLimit,
 		// Default to creating a VSL unless we're told otherwise
 		UseVolumeSnapshots: true,
 	}
@@ -142,6 +145,7 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 		Bucket:                            o.BucketName,
 		Prefix:                            o.Prefix,
 		PodAnnotations:                    o.PodAnnotations.Data(),
+		ServiceAccountAnnotations:         o.ServiceAccountAnnotations.Data(),
 		VeleroPodResources:                veleroPodResources,
 		ResticPodResources:                resticPodResources,
 		SecretData:                        secretData,

pkg/install/deployment.go

@@ -69,6 +69,7 @@ func WithEnvFromSecretKey(varName, secret, key string) podTemplateOption {
 func WithSecret(secretPresent bool) podTemplateOption {
 	return func(c *podTemplateConfig) {
 		c.withSecret = secretPresent
+
 	}
 }
 

pkg/install/resources.go

@@ -91,9 +91,11 @@ func objectMeta(namespace, name string) metav1.ObjectMeta {
 	}
 }
 
-func ServiceAccount(namespace string) *corev1.ServiceAccount {
+func ServiceAccount(namespace string, annotations map[string]string) *corev1.ServiceAccount {
+	objMeta := objectMeta(namespace, "velero")
+	objMeta.Annotations = annotations
 	return &corev1.ServiceAccount{
-		ObjectMeta: objectMeta(namespace, "velero"),
+		ObjectMeta: objMeta,
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ServiceAccount",
 			APIVersion: corev1.SchemeGroupVersion.String(),
@@ -185,6 +187,7 @@ func Secret(namespace string, data []byte) *corev1.Secret {
 
 func appendUnstructured(list *unstructured.UnstructuredList, obj runtime.Object) error {
 	u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj)
+
 	// Remove the status field so we're not sending blank data to the server.
 	// On CRDs, having an empty status is actually a validation error.
 	delete(u, "status")
@@ -202,6 +205,7 @@ type VeleroOptions struct {
 	Bucket                            string
 	Prefix                            string
 	PodAnnotations                    map[string]string
+	ServiceAccountAnnotations         map[string]string
 	VeleroPodResources                corev1.ResourceRequirements
 	ResticPodResources                corev1.ResourceRequirements
 	SecretData                        []byte
@@ -231,7 +235,7 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) {
 	crb := ClusterRoleBinding(o.Namespace)
 	appendUnstructured(resources, crb)
 
-	sa := ServiceAccount(o.Namespace)
+	sa := ServiceAccount(o.Namespace, o.ServiceAccountAnnotations)
 	appendUnstructured(resources, sa)
 
 	if o.SecretData != nil {

pkg/install/resources_test.go

@@ -45,6 +45,7 @@ func TestResources(t *testing.T) {
 	assert.Equal(t, "", crb.ObjectMeta.Namespace)
 	assert.Equal(t, "velero", crb.Subjects[0].Namespace)
 
-	sa := ServiceAccount("velero")
+	sa := ServiceAccount("velero", map[string]string{"abcd": "cbd"})
 	assert.Equal(t, "velero", sa.ObjectMeta.Namespace)
+	assert.Equal(t, "cbd", sa.ObjectMeta.Annotations["abcd"])
 }

site/docs/master/api-types/backupstoragelocation.md

@@ -71,6 +71,7 @@ The configurable parameters are as follows:
 | Key | Type | Default | Meaning |
 | --- | --- | --- | --- |
 | `kmsKeyName` | string | Empty | Name of the Cloud KMS key to use to encrypt backups stored in this location, in the form `projects/P/locations/L/keyRings/R/cryptoKeys/K`. See [customer-managed Cloud KMS keys](https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys) for details.  |
+| `serviceAccount` | string | Empty | Name of the GCP service account to use for this backup storage location. Specify the service account here if you want to use workload identity instead of providing the key file.
 
 [0]: #aws
 [1]: #gcp

site/docs/master/gcp-config.md

@@ -136,6 +136,29 @@ Additionally, you can specify `--use-restic` to enable restic support, and `--wa
 
 For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation.
 
+## Using Workload Identity (Optional)
+
+If you are running Velero on a GKE cluster with workload identity enabled, you may want to bind Velero's Kubernetes service account to a GCP service account with the appropriate permissions instead of providing the key file during installation.
+
+To do this, you must grant the GCP service account(the one you created in Step 3) the 'iam.serviceAccounts.signBlob' role. This is so that Velero's Kubernetes service account can create signed urls for the GCP bucket.
+
+Next, add an IAM policy binding to grant Velero's Kubernetes service account access to your created GCP service account.
+
+```bash
+gcloud iam service-accounts add-iam-policy-binding \
+    --role roles/iam.workloadIdentityUser \ 
+    --member serviceAccount:[PROJECT_ID].svc.id.goog[velero/velero] \
+    [GSA_NAME]@[PROJECT_ID].iam.gserviceaccount.com
+```
+
+For more information on configuring workload identity on GKE, look at the [official GCP documentation][24] for more details.
+
+Finally, you must add a service account annotation to the Kubernetes service account so that it will know which GCP service account to use. You can do this during installation with `--sa-annotations`. Furthermore, you must also use the flag `--no-secret` so that Velero will know not to look for a key file. You must also add the GCP service account name in `--backup-location-config`.
+
+```bash
+velero install --provider gcp --no-secret --sa-annotations iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_ID].iam.gserviceaccount.com --backup-location-config serviceAccount=[GSA_NAME]@[PROJECT_ID].iam.gserviceaccount.com
+```
+
 [0]: namespace.md
 [7]: api-types/backupstoragelocation.md#gcp
 [8]: api-types/volumesnapshotlocation.md#gcp
@@ -144,3 +167,4 @@ For more complex installation needs, use either the Helm chart, or add `--dry-ru
 [20]: faq.md
 [22]: https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control#iam-rolebinding-bootstrap
 [23]: install-overview.md#velero-resource-requirements
+[24]: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity