Azure: add support for loading env vars from a file, $AZURE_CREDENTIALS_FILE (#1364)

* azure: load env vars from AZURE_CREDENTIALS_FILE if it exists

Signed-off-by: Steve Kriss <krisss@vmware.com>

Gopkg.lock

@@ -285,6 +285,14 @@
   pruneopts = "NUT"
   revision = "0b12d6b5"
 
+[[projects]]
+  digest = "1:da62aa6632d04e080b8a8b85a59ed9ed1550842a0099a55f3ae3a20d02a3745a"
+  name = "github.com/joho/godotenv"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "23d116af351c84513e1946b527c88823e476be13"
+  version = "v1.3.0"
+
 [[projects]]
   digest = "1:8e36686e8b139f8fe240c1d5cf3a145bc675c22ff8e707857cdd3ae17b00d728"
   name = "github.com/json-iterator/go"
@@ -1020,6 +1028,7 @@
     "github.com/golang/protobuf/proto",
     "github.com/hashicorp/go-hclog",
     "github.com/hashicorp/go-plugin",
+    "github.com/joho/godotenv",
     "github.com/pkg/errors",
     "github.com/prometheus/client_golang/prometheus",
     "github.com/prometheus/client_golang/prometheus/promhttp",

Gopkg.toml

@@ -125,6 +125,10 @@
   name = "google.golang.org/grpc"
   version = "~v1.19.0"
 
+[[constraint]]
+  name = "github.com/joho/godotenv"
+  version = "~v1.3.0"
+
 [[override]]
   name = "golang.org/x/sys"
-  branch = "master"
+  branch = "master"

changelogs/unreleased/1364-skriss

@@ -0,0 +1,7 @@
+Azure: allow credentials to be provided in a .env file (path specified by $AZURE_CREDENTIALS_FILE), formatted like:
+    
+    AZURE_TENANT_ID=${AZURE_TENANT_ID}
+    AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}
+    AZURE_CLIENT_ID=${AZURE_CLIENT_ID}
+    AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}
+    AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP}

pkg/cloudprovider/azure/common.go

@@ -17,10 +17,12 @@ limitations under the License.
 package azure
 
 import (
+	"os"
 	"strings"
 
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
 )
 
@@ -48,6 +50,19 @@ func GetResticEnvVars(config map[string]string) (map[string]string, error) {
 	}, nil
 }
 
+func loadEnv() error {
+	envFile := os.Getenv("AZURE_CREDENTIALS_FILE")
+	if envFile == "" {
+		return nil
+	}
+
+	if err := godotenv.Overload(envFile); err != nil {
+		return errors.Wrapf(err, "error loading environment from AZURE_CREDENTIALS_FILE (%s)", envFile)
+	}
+
+	return nil
+}
+
 func newServicePrincipalToken(tenantID, clientID, clientSecret, scope string) (*adal.ServicePrincipalToken, error) {
 	oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, tenantID)
 	if err != nil {

pkg/cloudprovider/azure/object_store.go

@@ -47,6 +47,11 @@ func NewObjectStore(logger logrus.FieldLogger) *ObjectStore {
 }
 
 func getStorageAccountKey(config map[string]string) (string, error) {
+	// load environment vars from $AZURE_CREDENTIALS_FILE, if it exists
+	if err := loadEnv(); err != nil {
+		return "", err
+	}
+
 	// 1. we need AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID
 	envVars, err := getRequiredValues(os.Getenv, tenantIDEnvVar, clientIDEnvVar, clientSecretEnvVar, subscriptionIDEnvVar)
 	if err != nil {

pkg/cloudprovider/azure/volume_snapshotter.go

@@ -76,6 +76,11 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 		return err
 	}
 
+	// load environment vars from $AZURE_CREDENTIALS_FILE, if it exists
+	if err := loadEnv(); err != nil {
+		return err
+	}
+
 	// 1. we need AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP
 	envVars, err := getRequiredValues(os.Getenv, tenantIDEnvVar, clientIDEnvVar, clientSecretEnvVar, subscriptionIDEnvVar, resourceGroupEnvVar)
 	if err != nil {

pkg/install/deployment.go

@@ -134,6 +134,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1beta1.Deploy
 									Name:  "AWS_SHARED_CREDENTIALS_FILE",
 									Value: "/credentials/cloud",
 								},
+								{
+									Name:  "AZURE_CREDENTIALS_FILE",
+									Value: "/credentials/cloud",
+								},
 							},
 						},
 					},

pkg/install/deployment_test.go

@@ -32,7 +32,7 @@ func TestDeployment(t *testing.T) {
 	assert.Equal(t, "--restore-only", deploy.Spec.Template.Spec.Containers[0].Args[1])
 
 	deploy = Deployment("velero", WithEnvFromSecretKey("my-var", "my-secret", "my-key"))
-	envSecret := deploy.Spec.Template.Spec.Containers[0].Env[3]
+	envSecret := deploy.Spec.Template.Spec.Containers[0].Env[4]
 	assert.Equal(t, "my-var", envSecret.Name)
 	assert.Equal(t, "my-secret", envSecret.ValueFrom.SecretKeyRef.LocalObjectReference.Name)
 	assert.Equal(t, "my-key", envSecret.ValueFrom.SecretKeyRef.Key)

vendor/github.com/joho/godotenv/LICENCE

@@ -0,0 +1,23 @@
+Copyright (c) 2013 John Barton
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

vendor/github.com/joho/godotenv/godotenv.go

@@ -0,0 +1,346 @@
+// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv)
+//
+// Examples/readme can be found on the github page at https://github.com/joho/godotenv
+//
+// The TL;DR is that you make a .env file that looks something like
+//
+// 		SOME_ENV_VAR=somevalue
+//
+// and then in your go code you can call
+//
+// 		godotenv.Load()
+//
+// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR")
+package godotenv
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+const doubleQuoteSpecialChars = "\\\n\r\"!$`"
+
+// Load will read your env file(s) and load them into ENV for this process.
+//
+// Call this function as close as possible to the start of your program (ideally in main)
+//
+// If you call Load without any args it will default to loading .env in the current path
+//
+// You can otherwise tell it which files to load (there can be more than one) like
+//
+//		godotenv.Load("fileone", "filetwo")
+//
+// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
+func Load(filenames ...string) (err error) {
+	filenames = filenamesOrDefault(filenames)
+
+	for _, filename := range filenames {
+		err = loadFile(filename, false)
+		if err != nil {
+			return // return early on a spazout
+		}
+	}
+	return
+}
+
+// Overload will read your env file(s) and load them into ENV for this process.
+//
+// Call this function as close as possible to the start of your program (ideally in main)
+//
+// If you call Overload without any args it will default to loading .env in the current path
+//
+// You can otherwise tell it which files to load (there can be more than one) like
+//
+//		godotenv.Overload("fileone", "filetwo")
+//
+// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars.
+func Overload(filenames ...string) (err error) {
+	filenames = filenamesOrDefault(filenames)
+
+	for _, filename := range filenames {
+		err = loadFile(filename, true)
+		if err != nil {
+			return // return early on a spazout
+		}
+	}
+	return
+}
+
+// Read all env (with same file loading semantics as Load) but return values as
+// a map rather than automatically writing values into env
+func Read(filenames ...string) (envMap map[string]string, err error) {
+	filenames = filenamesOrDefault(filenames)
+	envMap = make(map[string]string)
+
+	for _, filename := range filenames {
+		individualEnvMap, individualErr := readFile(filename)
+
+		if individualErr != nil {
+			err = individualErr
+			return // return early on a spazout
+		}
+
+		for key, value := range individualEnvMap {
+			envMap[key] = value
+		}
+	}
+
+	return
+}
+
+// Parse reads an env file from io.Reader, returning a map of keys and values.
+func Parse(r io.Reader) (envMap map[string]string, err error) {
+	envMap = make(map[string]string)
+
+	var lines []string
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+
+	if err = scanner.Err(); err != nil {
+		return
+	}
+
+	for _, fullLine := range lines {
+		if !isIgnoredLine(fullLine) {
+			var key, value string
+			key, value, err = parseLine(fullLine, envMap)
+
+			if err != nil {
+				return
+			}
+			envMap[key] = value
+		}
+	}
+	return
+}
+
+//Unmarshal reads an env file from a string, returning a map of keys and values.
+func Unmarshal(str string) (envMap map[string]string, err error) {
+	return Parse(strings.NewReader(str))
+}
+
+// Exec loads env vars from the specified filenames (empty map falls back to default)
+// then executes the cmd specified.
+//
+// Simply hooks up os.Stdin/err/out to the command and calls Run()
+//
+// If you want more fine grained control over your command it's recommended
+// that you use `Load()` or `Read()` and the `os/exec` package yourself.
+func Exec(filenames []string, cmd string, cmdArgs []string) error {
+	Load(filenames...)
+
+	command := exec.Command(cmd, cmdArgs...)
+	command.Stdin = os.Stdin
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+	return command.Run()
+}
+
+// Write serializes the given environment and writes it to a file
+func Write(envMap map[string]string, filename string) error {
+	content, error := Marshal(envMap)
+	if error != nil {
+		return error
+	}
+	file, error := os.Create(filename)
+	if error != nil {
+		return error
+	}
+	_, err := file.WriteString(content)
+	return err
+}
+
+// Marshal outputs the given environment as a dotenv-formatted environment file.
+// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped.
+func Marshal(envMap map[string]string) (string, error) {
+	lines := make([]string, 0, len(envMap))
+	for k, v := range envMap {
+		lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v)))
+	}
+	sort.Strings(lines)
+	return strings.Join(lines, "\n"), nil
+}
+
+func filenamesOrDefault(filenames []string) []string {
+	if len(filenames) == 0 {
+		return []string{".env"}
+	}
+	return filenames
+}
+
+func loadFile(filename string, overload bool) error {
+	envMap, err := readFile(filename)
+	if err != nil {
+		return err
+	}
+
+	currentEnv := map[string]bool{}
+	rawEnv := os.Environ()
+	for _, rawEnvLine := range rawEnv {
+		key := strings.Split(rawEnvLine, "=")[0]
+		currentEnv[key] = true
+	}
+
+	for key, value := range envMap {
+		if !currentEnv[key] || overload {
+			os.Setenv(key, value)
+		}
+	}
+
+	return nil
+}
+
+func readFile(filename string) (envMap map[string]string, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	return Parse(file)
+}
+
+func parseLine(line string, envMap map[string]string) (key string, value string, err error) {
+	if len(line) == 0 {
+		err = errors.New("zero length string")
+		return
+	}
+
+	// ditch the comments (but keep quoted hashes)
+	if strings.Contains(line, "#") {
+		segmentsBetweenHashes := strings.Split(line, "#")
+		quotesAreOpen := false
+		var segmentsToKeep []string
+		for _, segment := range segmentsBetweenHashes {
+			if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
+				if quotesAreOpen {
+					quotesAreOpen = false
+					segmentsToKeep = append(segmentsToKeep, segment)
+				} else {
+					quotesAreOpen = true
+				}
+			}
+
+			if len(segmentsToKeep) == 0 || quotesAreOpen {
+				segmentsToKeep = append(segmentsToKeep, segment)
+			}
+		}
+
+		line = strings.Join(segmentsToKeep, "#")
+	}
+
+	firstEquals := strings.Index(line, "=")
+	firstColon := strings.Index(line, ":")
+	splitString := strings.SplitN(line, "=", 2)
+	if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) {
+		//this is a yaml-style line
+		splitString = strings.SplitN(line, ":", 2)
+	}
+
+	if len(splitString) != 2 {
+		err = errors.New("Can't separate key from value")
+		return
+	}
+
+	// Parse the key
+	key = splitString[0]
+	if strings.HasPrefix(key, "export") {
+		key = strings.TrimPrefix(key, "export")
+	}
+	key = strings.Trim(key, " ")
+
+	// Parse the value
+	value = parseValue(splitString[1], envMap)
+	return
+}
+
+func parseValue(value string, envMap map[string]string) string {
+
+	// trim
+	value = strings.Trim(value, " ")
+
+	// check if we've got quoted values or possible escapes
+	if len(value) > 1 {
+		rs := regexp.MustCompile(`\A'(.*)'\z`)
+		singleQuotes := rs.FindStringSubmatch(value)
+
+		rd := regexp.MustCompile(`\A"(.*)"\z`)
+		doubleQuotes := rd.FindStringSubmatch(value)
+
+		if singleQuotes != nil || doubleQuotes != nil {
+			// pull the quotes off the edges
+			value = value[1 : len(value)-1]
+		}
+
+		if doubleQuotes != nil {
+			// expand newlines
+			escapeRegex := regexp.MustCompile(`\\.`)
+			value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string {
+				c := strings.TrimPrefix(match, `\`)
+				switch c {
+				case "n":
+					return "\n"
+				case "r":
+					return "\r"
+				default:
+					return match
+				}
+			})
+			// unescape characters
+			e := regexp.MustCompile(`\\([^$])`)
+			value = e.ReplaceAllString(value, "$1")
+		}
+
+		if singleQuotes == nil {
+			value = expandVariables(value, envMap)
+		}
+	}
+
+	return value
+}
+
+func expandVariables(v string, m map[string]string) string {
+	r := regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`)
+
+	return r.ReplaceAllStringFunc(v, func(s string) string {
+		submatch := r.FindStringSubmatch(s)
+
+		if submatch == nil {
+			return s
+		}
+		if submatch[1] == "\\" || submatch[2] == "(" {
+			return submatch[0][1:]
+		} else if submatch[4] != "" {
+			return m[submatch[4]]
+		}
+		return s
+	})
+}
+
+func isIgnoredLine(line string) bool {
+	trimmedLine := strings.Trim(line, " \n\t")
+	return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
+}
+
+func doubleQuoteEscape(line string) string {
+	for _, c := range doubleQuoteSpecialChars {
+		toReplace := "\\" + string(c)
+		if c == '\n' {
+			toReplace = `\n`
+		}
+		if c == '\r' {
+			toReplace = `\r`
+		}
+		line = strings.Replace(line, string(c), toReplace, -1)
+	}
+	return line
+}