From 3db43b72060148ccba679a8da57db08bab0cc605 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Wed, 3 Jan 2024 22:44:11 -0800 Subject: [PATCH] feat: add azure local env auth This is the recommended auth from the following: https://github.com/Azure-Samples/storage-blobs-go-quickstart/blob/master/storage-quickstart.go https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-go?toc=%2Fazure%2Fdeveloper%2Fgo%2Ftoc.json&bc=%2Fazure%2Fdeveloper%2Fgo%2Fbreadcrumb%2Ftoc.json&tabs=roles-azure-portal#authenticate-to-azure-and-authorize-access-to-blob-data --- backend/azure/azure.go | 78 ++++++++++++++++++++++++++++++------------ cmd/versitygw/azure.go | 7 +--- go.mod | 9 +++-- go.sum | 20 +++++++---- 4 files changed, 77 insertions(+), 37 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index f32831ec..876b71dd 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -24,11 +24,13 @@ import ( "fmt" "io" "math" + "os" "strconv" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" @@ -46,18 +48,16 @@ const aclKey string = "Acl" type Azure struct { backend.BackendUnsupported - client *azblob.Client - creds *azblob.SharedKeyCredential - serviceURL string - sasToken string + client *azblob.Client + sharedkeyCreds *azblob.SharedKeyCredential + defaultCreds *azidentity.DefaultAzureCredential + serviceURL string + sasToken string } var _ backend.Backend = &Azure{} func New(accountName, accountKey, serviceURL, sasToken string) (*Azure, error) { - if sasToken != "" && (accountName != "" || accountKey != "") { - return nil, fmt.Errorf("choose one of the authentication methods: with sas token or with access key/name") - } if sasToken != "" { client, err := azblob.NewClientWithNoCredential(serviceURL+"?"+sasToken, nil) if err != nil { @@ -65,17 +65,42 @@ func New(accountName, accountKey, serviceURL, sasToken string) (*Azure, error) { } return &Azure{client: client, serviceURL: serviceURL, sasToken: sasToken}, nil } + + if accountName == "" { + // if account name not provided, try to get from env var + accountName = os.Getenv("AZURE_CLIENT_ID") + } + + url := serviceURL + if serviceURL == "" && accountName != "" { + // if not otherwise specified, use the typical form: + // http(s)://.blob.core.windows.net/ + url = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + } + + if accountName == "" || accountKey == "" { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("init default credentials: %w", err) + } + client, err := azblob.NewClient(url, cred, nil) + if err != nil { + return nil, fmt.Errorf("init client: %w", err) + } + return &Azure{client: client, serviceURL: url, defaultCreds: cred}, nil + } + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return nil, fmt.Errorf("init credentials: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + client, err := azblob.NewClientWithSharedKeyCredential(url, cred, nil) if err != nil { return nil, fmt.Errorf("init client: %w", err) } - return &Azure{client: client, serviceURL: serviceURL, creds: cred}, nil + return &Azure{client: client, serviceURL: url, sharedkeyCreds: cred}, nil } func (az *Azure) Shutdown() {} @@ -656,36 +681,45 @@ func (az *Azure) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) return []byte(*aclPtr), nil } -func (az *Azure) getContainerURL(container string) string { - return fmt.Sprintf("%v/%v", az.serviceURL, container) +func (az *Azure) getContainerURL(cntr string) string { + return fmt.Sprintf("%v/%v", az.serviceURL, cntr) } -func (az *Azure) getBlobURL(container, blob string) string { - return az.getContainerURL(container) + "/" + blob +func (az *Azure) getBlobURL(cntr, blb string) string { + return fmt.Sprintf("%v/%v", az.getContainerURL(cntr), blb) } -func (az *Azure) getBlobClient(container, blb string) (*blob.Client, error) { - blobURL := az.getBlobURL(container, blb) +func (az *Azure) getBlobClient(cntr, blb string) (*blob.Client, error) { + blobURL := az.getBlobURL(cntr, blb) + if az.defaultCreds != nil { + return blob.NewClient(blobURL, az.defaultCreds, nil) + } if az.sasToken != "" { return blob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil) } - return blob.NewClientWithSharedKeyCredential(blobURL, az.creds, nil) + return blob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil) } -func (az *Azure) getContainerClient(ctr string) (*container.Client, error) { - containerURL := az.getContainerURL(ctr) +func (az *Azure) getContainerClient(cntr string) (*container.Client, error) { + containerURL := az.getContainerURL(cntr) + if az.defaultCreds != nil { + return container.NewClient(containerURL, az.defaultCreds, nil) + } if az.sasToken != "" { return container.NewClientWithNoCredential(containerURL+"?"+az.sasToken, nil) } - return container.NewClientWithSharedKeyCredential(containerURL, az.creds, nil) + return container.NewClientWithSharedKeyCredential(containerURL, az.sharedkeyCreds, nil) } -func (az *Azure) getBlockBlobClient(container, blob string) (*blockblob.Client, error) { - blobURL := az.getBlobURL(container, blob) +func (az *Azure) getBlockBlobClient(cntr, blb string) (*blockblob.Client, error) { + blobURL := az.getBlobURL(cntr, blb) + if az.defaultCreds != nil { + return blockblob.NewClient(blobURL, az.defaultCreds, nil) + } if az.sasToken != "" { return blockblob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil) } - return blockblob.NewClientWithSharedKeyCredential(blobURL, az.creds, nil) + return blockblob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil) } func parseMetadata(m map[string]string) map[string]*string { diff --git a/cmd/versitygw/azure.go b/cmd/versitygw/azure.go index ccd950f0..7ccf5d9b 100644 --- a/cmd/versitygw/azure.go +++ b/cmd/versitygw/azure.go @@ -65,14 +65,9 @@ func azureCommand() *cli.Command { } func runAzure(ctx *cli.Context) error { - if azServiceURL == "" { - // if not otherwise specified, use the typical form: http(s)://.blob.core.windows.net/ - azServiceURL = fmt.Sprintf("https://%s.blob.core.windows.net/", azAccount) - } - be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken) if err != nil { - return fmt.Errorf("init azure: %v", err) + return fmt.Errorf("init azure: %w", err) } return runGateway(ctx.Context, be) diff --git a/go.mod b/go.mod index 9bd15e3b..c874fa6f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 @@ -23,18 +24,22 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index ee4e5c1d..9c9a3fe6 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= @@ -8,7 +9,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cX github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= @@ -63,7 +65,8 @@ github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE= github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= @@ -76,6 +79,7 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -92,7 +96,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -132,8 +137,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -142,8 +147,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -154,6 +159,7 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=