From 5536d88fbb665c52e15f24f897b861bae269028c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Jul 2026 16:41:13 -0700 Subject: [PATCH] azure: let the blob endpoint be configured (#10460) * azure: let the blob endpoint be configured The service url was always derived as .blob.core.windows.net, which leaves out Azure Government, Azure China, and private endpoints. Name the blob service url instead and those accounts become reachable. The url has to be https, since the account key or the bearer token would otherwise travel in the clear. * azure: reject an endpoint that carries no hostname A url like https://:443/ has a host of ":443", so the emptiness check on Host let it through and the request only failed once it reached Azure. The hostname is what has to be there. --- seaweed-volume/proto/remote.proto | 1 + weed/command/scaffold/replication.toml | 3 ++ weed/pb/remote.proto | 1 + weed/pb/remote_pb/remote.pb.go | 13 ++++++- .../remote_storage/azure/azure_credentials.go | 29 ++++++++++++++- .../azure/azure_credentials_test.go | 37 ++++++++++++++++--- .../azure/azure_storage_client.go | 2 +- weed/replication/sink/azuresink/azure_sink.go | 5 ++- .../sink/azuresink/azure_sink_test.go | 10 ++--- weed/shell/command_remote_configure.go | 1 + 10 files changed, 84 insertions(+), 18 deletions(-) diff --git a/seaweed-volume/proto/remote.proto b/seaweed-volume/proto/remote.proto index 6298c33a0..d79a79df9 100644 --- a/seaweed-volume/proto/remote.proto +++ b/seaweed-volume/proto/remote.proto @@ -27,6 +27,7 @@ message RemoteConf { string azure_account_name = 15; string azure_account_key = 16; string azure_client_id = 17; + string azure_endpoint = 18; string backblaze_key_id = 20; string backblaze_application_key = 21; diff --git a/weed/command/scaffold/replication.toml b/weed/command/scaffold/replication.toml index 40530453e..d72c5148e 100644 --- a/weed/command/scaffold/replication.toml +++ b/weed/command/scaffold/replication.toml @@ -67,6 +67,9 @@ account_key = "" # takes its tenant and token from AZURE_TENANT_ID and AZURE_FEDERATED_TOKEN_FILE, # which the Azure workload identity webhook projects into the pod. client_id = "" +# blob service url, for accounts outside the public cloud. Empty resolves to +# https://.blob.core.windows.net/ +endpoint = "" container = "mycontainer" # an existing container directory = "/" # destination directory is_incremental = false diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto index 2bf7dac59..9654719de 100644 --- a/weed/pb/remote.proto +++ b/weed/pb/remote.proto @@ -27,6 +27,7 @@ message RemoteConf { string azure_account_name = 15; string azure_account_key = 16; string azure_client_id = 17; + string azure_endpoint = 18; string backblaze_key_id = 20; string backblaze_application_key = 21; diff --git a/weed/pb/remote_pb/remote.pb.go b/weed/pb/remote_pb/remote.pb.go index ee5580eb2..611819c08 100644 --- a/weed/pb/remote_pb/remote.pb.go +++ b/weed/pb/remote_pb/remote.pb.go @@ -41,6 +41,7 @@ type RemoteConf struct { AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` AzureClientId string `protobuf:"bytes,17,opt,name=azure_client_id,json=azureClientId,proto3" json:"azure_client_id,omitempty"` + AzureEndpoint string `protobuf:"bytes,18,opt,name=azure_endpoint,json=azureEndpoint,proto3" json:"azure_endpoint,omitempty"` BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` @@ -209,6 +210,13 @@ func (x *RemoteConf) GetAzureClientId() string { return "" } +func (x *RemoteConf) GetAzureEndpoint() string { + if x != nil { + return x.AzureEndpoint + } + return "" +} + func (x *RemoteConf) GetBackblazeKeyId() string { if x != nil { return x.BackblazeKeyId @@ -536,7 +544,7 @@ var File_remote_proto protoreflect.FileDescriptor const file_remote_proto_rawDesc = "" + "\n" + - "\fremote.proto\x12\tremote_pb\"\xc3\x0e\n" + + "\fremote.proto\x12\tremote_pb\"\xea\x0e\n" + "\n" + "RemoteConf\x12\x12\n" + "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + @@ -555,7 +563,8 @@ const file_remote_proto_rawDesc = "" + "\x0egcs_project_id\x18\f \x01(\tR\fgcsProjectId\x12,\n" + "\x12azure_account_name\x18\x0f \x01(\tR\x10azureAccountName\x12*\n" + "\x11azure_account_key\x18\x10 \x01(\tR\x0fazureAccountKey\x12&\n" + - "\x0fazure_client_id\x18\x11 \x01(\tR\razureClientId\x12(\n" + + "\x0fazure_client_id\x18\x11 \x01(\tR\razureClientId\x12%\n" + + "\x0eazure_endpoint\x18\x12 \x01(\tR\razureEndpoint\x12(\n" + "\x10backblaze_key_id\x18\x14 \x01(\tR\x0ebackblazeKeyId\x12:\n" + "\x19backblaze_application_key\x18\x15 \x01(\tR\x17backblazeApplicationKey\x12-\n" + "\x12backblaze_endpoint\x18\x16 \x01(\tR\x11backblazeEndpoint\x12)\n" + diff --git a/weed/remote_storage/azure/azure_credentials.go b/weed/remote_storage/azure/azure_credentials.go index 94516b438..d749b4136 100644 --- a/weed/remote_storage/azure/azure_credentials.go +++ b/weed/remote_storage/azure/azure_credentials.go @@ -2,6 +2,7 @@ package azure import ( "fmt" + "net/url" "os" "regexp" @@ -23,7 +24,10 @@ var validAzureAccountName = regexp.MustCompile(`^[a-z0-9]{3,24}$`) // the environment, and access is granted through RBAC. Fleets that cannot // distribute and rotate storage account keys authenticate that way. clientID // pins a user-assigned identity when the environment offers more than one. -func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, error) { +// +// endpoint is the blob service URL, for accounts outside the public cloud. An +// empty endpoint derives the public one from accountName. +func NewAzBlobClient(accountName, accountKey, clientID, endpoint string) (*azblob.Client, error) { if accountName == "" { return nil, fmt.Errorf("azure account name is required") @@ -32,7 +36,10 @@ func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, return nil, fmt.Errorf("invalid azure account name %q: expecting 3 to 24 lowercase letters and digits", accountName) } - serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + serviceURL, err := azureServiceURL(accountName, endpoint) + if err != nil { + return nil, err + } if accountKey == "" { credential, err := newAzureTokenCredential(clientID) @@ -58,6 +65,24 @@ func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, return client, nil } +// azureServiceURL locates the blob service. Sovereign clouds and private +// endpoints do not live under blob.core.windows.net, so they name the service +// URL outright rather than having it derived from the account. +func azureServiceURL(accountName, endpoint string) (string, error) { + if endpoint == "" { + return fmt.Sprintf("https://%s.blob.core.windows.net/", accountName), nil + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("invalid azure endpoint %q: %w", endpoint, err) + } + // plain http would carry the account key or the bearer token in the clear + if parsed.Scheme != "https" || parsed.Hostname() == "" { + return "", fmt.Errorf("invalid azure endpoint %q: expecting an https service url, such as https://%s.blob.core.usgovcloudapi.net/", endpoint, accountName) + } + return endpoint, nil +} + // newAzureTokenCredential resolves an Entra ID credential. Without a pinned // clientID the default chain discovers whatever the host offers. With one, the // federated token file projected by the Azure workload identity webhook tells diff --git a/weed/remote_storage/azure/azure_credentials_test.go b/weed/remote_storage/azure/azure_credentials_test.go index 0d9c7a4d3..6f63a9155 100644 --- a/weed/remote_storage/azure/azure_credentials_test.go +++ b/weed/remote_storage/azure/azure_credentials_test.go @@ -6,21 +6,21 @@ import ( ) func TestNewAzBlobClientRequiresAccountName(t *testing.T) { - if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", ""); err == nil { + if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", "", ""); err == nil { t.Error("expected an error without an account name") } } func TestNewAzBlobClientRejectsMalformedAccountName(t *testing.T) { for _, accountName := range []string{"ab", "TestAccount", "test-account", "evil.com/x", "evil@host.com", "account?x=1"} { - if _, err := NewAzBlobClient(accountName, "", ""); err == nil { + if _, err := NewAzBlobClient(accountName, "", "", ""); err == nil { t.Errorf("expected an error for account name %q", accountName) } } } func TestNewAzBlobClientSharedKey(t *testing.T) { - client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "") + client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "", "") if err != nil { t.Fatalf("failed to create a shared key client: %v", err) } @@ -30,14 +30,39 @@ func TestNewAzBlobClientSharedKey(t *testing.T) { } func TestNewAzBlobClientSharedKeyRejectsMalformedKey(t *testing.T) { - if _, err := NewAzBlobClient("testaccount", "not base64", ""); err == nil { + if _, err := NewAzBlobClient("testaccount", "not base64", "", ""); err == nil { t.Error("expected an error with a malformed account key") } } +func TestAzureServiceURL(t *testing.T) { + serviceURL, err := azureServiceURL("testaccount", "") + if err != nil { + t.Fatalf("failed to derive the public service url: %v", err) + } + if serviceURL != "https://testaccount.blob.core.windows.net/" { + t.Errorf("unexpected public service url %q", serviceURL) + } + + government := "https://testaccount.blob.core.usgovcloudapi.net/" + serviceURL, err = azureServiceURL("testaccount", government) + if err != nil { + t.Fatalf("failed to keep the configured service url: %v", err) + } + if serviceURL != government { + t.Errorf("expected %q, got %q", government, serviceURL) + } + + for _, endpoint := range []string{"core.usgovcloudapi.net", "http://testaccount.blob.core.windows.net/", "https://", "https://:443/", "://x"} { + if _, err := azureServiceURL("testaccount", endpoint); err == nil { + t.Errorf("expected an error for endpoint %q", endpoint) + } + } +} + // no account key: authenticate through the Entra ID chain instead func TestNewAzBlobClientEntraID(t *testing.T) { - client, err := NewAzBlobClient("testaccount", "", "") + client, err := NewAzBlobClient("testaccount", "", "", "") if err != nil { t.Fatalf("failed to create an Entra ID client: %v", err) } @@ -50,7 +75,7 @@ func TestNewAzBlobClientWorkloadIdentity(t *testing.T) { t.Setenv("AZURE_FEDERATED_TOKEN_FILE", filepath.Join(t.TempDir(), "token")) t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000") - client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111") + client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111", "") if err != nil { t.Fatalf("failed to create a workload identity client: %v", err) } diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go index e0888f662..705a5720a 100644 --- a/weed/remote_storage/azure/azure_storage_client.go +++ b/weed/remote_storage/azure/azure_storage_client.go @@ -125,7 +125,7 @@ func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storag return nil, fmt.Errorf("neither azure_account_name nor the AZURE_STORAGE_ACCOUNT environment variable is set") } - azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId) + azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId, conf.AzureEndpoint) if err != nil { return nil, err } diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 4ca8a9c4e..6688483a4 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -52,6 +52,7 @@ func (g *AzureSink) Initialize(configuration util.Configuration, prefix string) configuration.GetString(prefix+"account_name"), configuration.GetString(prefix+"account_key"), configuration.GetString(prefix+"client_id"), + configuration.GetString(prefix+"endpoint"), configuration.GetString(prefix+"container"), configuration.GetString(prefix+"directory"), ) @@ -61,11 +62,11 @@ func (g *AzureSink) SetSourceFiler(s *source.FilerSource) { g.filerSource = s } -func (g *AzureSink) initialize(accountName, accountKey, clientID, container, dir string) error { +func (g *AzureSink) initialize(accountName, accountKey, clientID, endpoint, container, dir string) error { g.container = container g.dir = dir - client, err := azure.NewAzBlobClient(accountName, accountKey, clientID) + client, err := azure.NewAzBlobClient(accountName, accountKey, clientID, endpoint) if err != nil { return err } diff --git a/weed/replication/sink/azuresink/azure_sink_test.go b/weed/replication/sink/azuresink/azure_sink_test.go index 33282df6d..2d3fe7369 100644 --- a/weed/replication/sink/azuresink/azure_sink_test.go +++ b/weed/replication/sink/azuresink/azure_sink_test.go @@ -104,7 +104,7 @@ func TestAzureSinkInitialization(t *testing.T) { sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize Azure sink: %v", err) } @@ -190,7 +190,7 @@ func TestAzureSinkEntryOperations(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -285,7 +285,7 @@ func TestAzureSinkPrecondition(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -354,7 +354,7 @@ func TestAzureSinkIdempotentCreate(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -460,7 +460,7 @@ func BenchmarkCleanKey(b *testing.B) { func TestAzureSinkInvalidCredentials(t *testing.T) { sink := &AzureSink{} - err := sink.initialize("invalid-account", "aW52YWxpZGtleQ==", "", "test-container", "/test") + err := sink.initialize("invalid-account", "aW52YWxpZGtleQ==", "", "", "test-container", "/test") if err != nil { t.Skip("Invalid credentials correctly rejected at initialization") } diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index 44b9c5e9f..2189a408f 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -84,6 +84,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.AzureAccountName, "azure.account_name", "", "azure account name, default to use env AZURE_STORAGE_ACCOUNT") remoteConfigureCommand.StringVar(&conf.AzureAccountKey, "azure.account_key", "", "azure account key, default to use env AZURE_STORAGE_ACCESS_KEY. Leave empty to authenticate with Entra ID") remoteConfigureCommand.StringVar(&conf.AzureClientId, "azure.client_id", "", "azure user-assigned identity to authenticate, when no account key is given. Workload identity also reads env AZURE_TENANT_ID and AZURE_FEDERATED_TOKEN_FILE") + remoteConfigureCommand.StringVar(&conf.AzureEndpoint, "azure.endpoint", "", "azure blob service url, for accounts outside the public cloud, e.g. https://xxx.blob.core.usgovcloudapi.net/") remoteConfigureCommand.StringVar(&conf.BackblazeKeyId, "b2.key_id", "", "backblaze keyID") remoteConfigureCommand.StringVar(&conf.BackblazeApplicationKey, "b2.application_key", "", "backblaze applicationKey. Note that your Master Application Key will not work with the S3 Compatible API. You must create a new key that is eligible for use. For more information: https://help.backblaze.com/hc/en-us/articles/360047425453")