diff --git a/seaweed-volume/proto/remote.proto b/seaweed-volume/proto/remote.proto index 6298c33a0..d79a79df9 100644 --- a/seaweed-volume/proto/remote.proto +++ b/seaweed-volume/proto/remote.proto @@ -27,6 +27,7 @@ message RemoteConf { string azure_account_name = 15; string azure_account_key = 16; string azure_client_id = 17; + string azure_endpoint = 18; string backblaze_key_id = 20; string backblaze_application_key = 21; diff --git a/weed/command/scaffold/replication.toml b/weed/command/scaffold/replication.toml index 40530453e..d72c5148e 100644 --- a/weed/command/scaffold/replication.toml +++ b/weed/command/scaffold/replication.toml @@ -67,6 +67,9 @@ account_key = "" # takes its tenant and token from AZURE_TENANT_ID and AZURE_FEDERATED_TOKEN_FILE, # which the Azure workload identity webhook projects into the pod. client_id = "" +# blob service url, for accounts outside the public cloud. Empty resolves to +# https://.blob.core.windows.net/ +endpoint = "" container = "mycontainer" # an existing container directory = "/" # destination directory is_incremental = false diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto index 2bf7dac59..9654719de 100644 --- a/weed/pb/remote.proto +++ b/weed/pb/remote.proto @@ -27,6 +27,7 @@ message RemoteConf { string azure_account_name = 15; string azure_account_key = 16; string azure_client_id = 17; + string azure_endpoint = 18; string backblaze_key_id = 20; string backblaze_application_key = 21; diff --git a/weed/pb/remote_pb/remote.pb.go b/weed/pb/remote_pb/remote.pb.go index ee5580eb2..611819c08 100644 --- a/weed/pb/remote_pb/remote.pb.go +++ b/weed/pb/remote_pb/remote.pb.go @@ -41,6 +41,7 @@ type RemoteConf struct { AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` AzureClientId string `protobuf:"bytes,17,opt,name=azure_client_id,json=azureClientId,proto3" json:"azure_client_id,omitempty"` + AzureEndpoint string `protobuf:"bytes,18,opt,name=azure_endpoint,json=azureEndpoint,proto3" json:"azure_endpoint,omitempty"` BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` @@ -209,6 +210,13 @@ func (x *RemoteConf) GetAzureClientId() string { return "" } +func (x *RemoteConf) GetAzureEndpoint() string { + if x != nil { + return x.AzureEndpoint + } + return "" +} + func (x *RemoteConf) GetBackblazeKeyId() string { if x != nil { return x.BackblazeKeyId @@ -536,7 +544,7 @@ var File_remote_proto protoreflect.FileDescriptor const file_remote_proto_rawDesc = "" + "\n" + - "\fremote.proto\x12\tremote_pb\"\xc3\x0e\n" + + "\fremote.proto\x12\tremote_pb\"\xea\x0e\n" + "\n" + "RemoteConf\x12\x12\n" + "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + @@ -555,7 +563,8 @@ const file_remote_proto_rawDesc = "" + "\x0egcs_project_id\x18\f \x01(\tR\fgcsProjectId\x12,\n" + "\x12azure_account_name\x18\x0f \x01(\tR\x10azureAccountName\x12*\n" + "\x11azure_account_key\x18\x10 \x01(\tR\x0fazureAccountKey\x12&\n" + - "\x0fazure_client_id\x18\x11 \x01(\tR\razureClientId\x12(\n" + + "\x0fazure_client_id\x18\x11 \x01(\tR\razureClientId\x12%\n" + + "\x0eazure_endpoint\x18\x12 \x01(\tR\razureEndpoint\x12(\n" + "\x10backblaze_key_id\x18\x14 \x01(\tR\x0ebackblazeKeyId\x12:\n" + "\x19backblaze_application_key\x18\x15 \x01(\tR\x17backblazeApplicationKey\x12-\n" + "\x12backblaze_endpoint\x18\x16 \x01(\tR\x11backblazeEndpoint\x12)\n" + diff --git a/weed/remote_storage/azure/azure_credentials.go b/weed/remote_storage/azure/azure_credentials.go index 94516b438..d749b4136 100644 --- a/weed/remote_storage/azure/azure_credentials.go +++ b/weed/remote_storage/azure/azure_credentials.go @@ -2,6 +2,7 @@ package azure import ( "fmt" + "net/url" "os" "regexp" @@ -23,7 +24,10 @@ var validAzureAccountName = regexp.MustCompile(`^[a-z0-9]{3,24}$`) // the environment, and access is granted through RBAC. Fleets that cannot // distribute and rotate storage account keys authenticate that way. clientID // pins a user-assigned identity when the environment offers more than one. -func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, error) { +// +// endpoint is the blob service URL, for accounts outside the public cloud. An +// empty endpoint derives the public one from accountName. +func NewAzBlobClient(accountName, accountKey, clientID, endpoint string) (*azblob.Client, error) { if accountName == "" { return nil, fmt.Errorf("azure account name is required") @@ -32,7 +36,10 @@ func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, return nil, fmt.Errorf("invalid azure account name %q: expecting 3 to 24 lowercase letters and digits", accountName) } - serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + serviceURL, err := azureServiceURL(accountName, endpoint) + if err != nil { + return nil, err + } if accountKey == "" { credential, err := newAzureTokenCredential(clientID) @@ -58,6 +65,24 @@ func NewAzBlobClient(accountName, accountKey, clientID string) (*azblob.Client, return client, nil } +// azureServiceURL locates the blob service. Sovereign clouds and private +// endpoints do not live under blob.core.windows.net, so they name the service +// URL outright rather than having it derived from the account. +func azureServiceURL(accountName, endpoint string) (string, error) { + if endpoint == "" { + return fmt.Sprintf("https://%s.blob.core.windows.net/", accountName), nil + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("invalid azure endpoint %q: %w", endpoint, err) + } + // plain http would carry the account key or the bearer token in the clear + if parsed.Scheme != "https" || parsed.Hostname() == "" { + return "", fmt.Errorf("invalid azure endpoint %q: expecting an https service url, such as https://%s.blob.core.usgovcloudapi.net/", endpoint, accountName) + } + return endpoint, nil +} + // newAzureTokenCredential resolves an Entra ID credential. Without a pinned // clientID the default chain discovers whatever the host offers. With one, the // federated token file projected by the Azure workload identity webhook tells diff --git a/weed/remote_storage/azure/azure_credentials_test.go b/weed/remote_storage/azure/azure_credentials_test.go index 0d9c7a4d3..6f63a9155 100644 --- a/weed/remote_storage/azure/azure_credentials_test.go +++ b/weed/remote_storage/azure/azure_credentials_test.go @@ -6,21 +6,21 @@ import ( ) func TestNewAzBlobClientRequiresAccountName(t *testing.T) { - if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", ""); err == nil { + if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", "", ""); err == nil { t.Error("expected an error without an account name") } } func TestNewAzBlobClientRejectsMalformedAccountName(t *testing.T) { for _, accountName := range []string{"ab", "TestAccount", "test-account", "evil.com/x", "evil@host.com", "account?x=1"} { - if _, err := NewAzBlobClient(accountName, "", ""); err == nil { + if _, err := NewAzBlobClient(accountName, "", "", ""); err == nil { t.Errorf("expected an error for account name %q", accountName) } } } func TestNewAzBlobClientSharedKey(t *testing.T) { - client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "") + client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "", "") if err != nil { t.Fatalf("failed to create a shared key client: %v", err) } @@ -30,14 +30,39 @@ func TestNewAzBlobClientSharedKey(t *testing.T) { } func TestNewAzBlobClientSharedKeyRejectsMalformedKey(t *testing.T) { - if _, err := NewAzBlobClient("testaccount", "not base64", ""); err == nil { + if _, err := NewAzBlobClient("testaccount", "not base64", "", ""); err == nil { t.Error("expected an error with a malformed account key") } } +func TestAzureServiceURL(t *testing.T) { + serviceURL, err := azureServiceURL("testaccount", "") + if err != nil { + t.Fatalf("failed to derive the public service url: %v", err) + } + if serviceURL != "https://testaccount.blob.core.windows.net/" { + t.Errorf("unexpected public service url %q", serviceURL) + } + + government := "https://testaccount.blob.core.usgovcloudapi.net/" + serviceURL, err = azureServiceURL("testaccount", government) + if err != nil { + t.Fatalf("failed to keep the configured service url: %v", err) + } + if serviceURL != government { + t.Errorf("expected %q, got %q", government, serviceURL) + } + + for _, endpoint := range []string{"core.usgovcloudapi.net", "http://testaccount.blob.core.windows.net/", "https://", "https://:443/", "://x"} { + if _, err := azureServiceURL("testaccount", endpoint); err == nil { + t.Errorf("expected an error for endpoint %q", endpoint) + } + } +} + // no account key: authenticate through the Entra ID chain instead func TestNewAzBlobClientEntraID(t *testing.T) { - client, err := NewAzBlobClient("testaccount", "", "") + client, err := NewAzBlobClient("testaccount", "", "", "") if err != nil { t.Fatalf("failed to create an Entra ID client: %v", err) } @@ -50,7 +75,7 @@ func TestNewAzBlobClientWorkloadIdentity(t *testing.T) { t.Setenv("AZURE_FEDERATED_TOKEN_FILE", filepath.Join(t.TempDir(), "token")) t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000") - client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111") + client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111", "") if err != nil { t.Fatalf("failed to create a workload identity client: %v", err) } diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go index e0888f662..705a5720a 100644 --- a/weed/remote_storage/azure/azure_storage_client.go +++ b/weed/remote_storage/azure/azure_storage_client.go @@ -125,7 +125,7 @@ func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storag return nil, fmt.Errorf("neither azure_account_name nor the AZURE_STORAGE_ACCOUNT environment variable is set") } - azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId) + azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId, conf.AzureEndpoint) if err != nil { return nil, err } diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 4ca8a9c4e..6688483a4 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -52,6 +52,7 @@ func (g *AzureSink) Initialize(configuration util.Configuration, prefix string) configuration.GetString(prefix+"account_name"), configuration.GetString(prefix+"account_key"), configuration.GetString(prefix+"client_id"), + configuration.GetString(prefix+"endpoint"), configuration.GetString(prefix+"container"), configuration.GetString(prefix+"directory"), ) @@ -61,11 +62,11 @@ func (g *AzureSink) SetSourceFiler(s *source.FilerSource) { g.filerSource = s } -func (g *AzureSink) initialize(accountName, accountKey, clientID, container, dir string) error { +func (g *AzureSink) initialize(accountName, accountKey, clientID, endpoint, container, dir string) error { g.container = container g.dir = dir - client, err := azure.NewAzBlobClient(accountName, accountKey, clientID) + client, err := azure.NewAzBlobClient(accountName, accountKey, clientID, endpoint) if err != nil { return err } diff --git a/weed/replication/sink/azuresink/azure_sink_test.go b/weed/replication/sink/azuresink/azure_sink_test.go index 33282df6d..2d3fe7369 100644 --- a/weed/replication/sink/azuresink/azure_sink_test.go +++ b/weed/replication/sink/azuresink/azure_sink_test.go @@ -104,7 +104,7 @@ func TestAzureSinkInitialization(t *testing.T) { sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize Azure sink: %v", err) } @@ -190,7 +190,7 @@ func TestAzureSinkEntryOperations(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -285,7 +285,7 @@ func TestAzureSinkPrecondition(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -354,7 +354,7 @@ func TestAzureSinkIdempotentCreate(t *testing.T) { } sink := &AzureSink{} - err := sink.initialize(accountName, accountKey, "", testContainer, "/test") + err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test") if err != nil { t.Fatalf("Failed to initialize: %v", err) } @@ -460,7 +460,7 @@ func BenchmarkCleanKey(b *testing.B) { func TestAzureSinkInvalidCredentials(t *testing.T) { sink := &AzureSink{} - err := sink.initialize("invalid-account", "aW52YWxpZGtleQ==", "", "test-container", "/test") + err := sink.initialize("invalid-account", "aW52YWxpZGtleQ==", "", "", "test-container", "/test") if err != nil { t.Skip("Invalid credentials correctly rejected at initialization") } diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index 44b9c5e9f..2189a408f 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -84,6 +84,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.AzureAccountName, "azure.account_name", "", "azure account name, default to use env AZURE_STORAGE_ACCOUNT") remoteConfigureCommand.StringVar(&conf.AzureAccountKey, "azure.account_key", "", "azure account key, default to use env AZURE_STORAGE_ACCESS_KEY. Leave empty to authenticate with Entra ID") remoteConfigureCommand.StringVar(&conf.AzureClientId, "azure.client_id", "", "azure user-assigned identity to authenticate, when no account key is given. Workload identity also reads env AZURE_TENANT_ID and AZURE_FEDERATED_TOKEN_FILE") + remoteConfigureCommand.StringVar(&conf.AzureEndpoint, "azure.endpoint", "", "azure blob service url, for accounts outside the public cloud, e.g. https://xxx.blob.core.usgovcloudapi.net/") remoteConfigureCommand.StringVar(&conf.BackblazeKeyId, "b2.key_id", "", "backblaze keyID") remoteConfigureCommand.StringVar(&conf.BackblazeApplicationKey, "b2.application_key", "", "backblaze applicationKey. Note that your Master Application Key will not work with the S3 Compatible API. You must create a new key that is eligible for use. For more information: https://help.backblaze.com/hc/en-us/articles/360047425453")