s3: register the advertised ip with the master (#10482)

* s3: register the advertised ip with the master

The cluster address came from the bind ip, falling back to the
auto-detected interface, so -ip never reached the S3 registration.
weed mini -ip=localhost binds the wildcard and ended up registering
whatever interface happened to sort first -- on a host with VPN
interfaces, an address that stops routing once the tunnel drops.

IAM changes are pushed to registered S3 servers over gRPC, so every
mutation then blocked the full 10s propagation deadline before logging
a failure, and cluster.ps and the admin UI listed a node nothing could
reach. Identities still arrived through the /etc/iam metadata
subscription, so this cost latency and visibility, not credentials.

Add an advertise ip to the gateway option, preferring it over the bind
address, and wire the parent -ip through server, filer and mini.

* s3: treat any unspecified bind address as a wildcard

net.ParseIP + IsUnspecified covers ::, [::] and the expanded IPv6 forms
instead of only the 0.0.0.0 literal, so an IPv6 wildcard bind no longer
registers an address peers cannot dial. Host names parse as nil and stay
addresses in their own right. Apply the same guard to the advertised ip.

weed/command/filer.go

@@ -257,6 +257,7 @@ func runFiler(cmd *Command, args []string) bool {
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
+		filerS3Options.ip = f.ip
 		if *filerS3Options.bindIp == "" {
 			filerS3Options.bindIp = f.bindIp
 		}

weed/command/mini.go

@@ -1223,6 +1223,7 @@ func runMini(cmd *Command, args []string) bool {
 	miniFilerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
 	miniFilerOptions.ip = miniIp
 	miniFilerOptions.bindIp = miniBindIp
+	miniS3Options.ip = miniIp
 	miniS3Options.bindIp = miniBindIp
 	miniWebDavOptions.ipBind = miniBindIp
 	miniOptions.v.ip = miniIp

weed/command/s3.go

@@ -42,6 +42,7 @@ var (
 // When adding a new field, update all four flag registration sites.
 type S3Options struct {
 	filer                     *string
+	ip                        *string
 	bindIp                    *string
 	port                      *int
 	portHttps                 *int
@@ -83,6 +84,7 @@ type S3Options struct {
 func init() {
 	cmdS3.Run = runS3 // break init cycle
 	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
+	s3StandaloneOptions.ip = cmdS3.Flag.String("ip", "", "ip address advertised to the cluster. If empty, default to -ip.bind, or the auto-detected address.")
 	s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to 0.0.0.0.")
 	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
 	s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
@@ -358,6 +360,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		EnableIam:                 *s3opt.enableIam, // Embedded IAM API (enabled by default)
 		IamReadOnly:               *s3opt.iamReadOnly,
 		Cipher:                    *s3opt.cipher, // encrypt data on volume servers
+		Ip:                        *s3opt.ip,
 		BindIp:                    *s3opt.bindIp,
 		GrpcPort:                  *s3opt.portGrpc,
 		ExternalUrl:               s3opt.resolveExternalUrl(),

weed/command/server.go

@@ -289,6 +289,7 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
+	s3Options.ip = serverIp
 	if *s3Options.bindIp == "" {
 		s3Options.bindIp = serverBindIp
 	}

weed/s3api/s3api_server.go

@@ -58,9 +58,10 @@ type S3ApiServerOption struct {
 	IamConfig                 string // Advanced IAM configuration file path
 	ConcurrentUploadLimit     int64
 	ConcurrentFileUploadLimit int64
-	EnableIam                 bool // Enable embedded IAM API on the same port
-	IamReadOnly               bool // Disable IAM write operations on this server
-	Cipher                    bool // encrypt data on volume servers
+	EnableIam                 bool   // Enable embedded IAM API on the same port
+	IamReadOnly               bool   // Disable IAM write operations on this server
+	Cipher                    bool   // encrypt data on volume servers
+	Ip                        string // address advertised to the cluster; empty falls back to BindIp
 	BindIp                    string
 	GrpcPort                  int
 	ExternalUrl               string // external URL clients use, for signature verification behind a reverse proxy
@@ -132,6 +133,30 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
 	return NewS3ApiServerWithStore(router, option, "")
 }
 
+// advertisedHost is the address this server registers with the master, which is
+// how peers reach it — IAM changes are pushed to it over gRPC. It must be the
+// advertised -ip, not the bind address: binding 0.0.0.0 and registering the
+// auto-detected interface makes those pushes dial a host that may not route
+// back here at all (a VPN address, a container-internal IP), and the push then
+// fails silently after a 10s deadline.
+func (option *S3ApiServerOption) advertisedHost() string {
+	if option.Ip != "" && !isWildcardHost(option.Ip) {
+		return option.Ip
+	}
+	if option.BindIp != "" && !isWildcardHost(option.BindIp) {
+		return option.BindIp
+	}
+	return util.DetectedHostAddress()
+}
+
+// isWildcardHost reports whether host is an unspecified address (0.0.0.0, ::,
+// [::]) — one that accepts connections but tells a peer nothing about where to
+// reach us. Host names parse as nil and are addresses in their own right.
+func isWildcardHost(host string) bool {
+	ip := net.ParseIP(strings.TrimSuffix(strings.TrimPrefix(host, "["), "]"))
+	return ip != nil && ip.IsUnspecified()
+}
+
 func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, explicitStore string) (s3ApiServer *S3ApiServer, err error) {
 	if len(option.Filers) == 0 {
 		return nil, fmt.Errorf("at least one filer address is required")
@@ -174,10 +199,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 		for i, addr := range option.Masters {
 			masterMap[fmt.Sprintf("master%d", i)] = addr
 		}
-		clientHost := option.BindIp
-		if clientHost == "0.0.0.0" || clientHost == "" {
-			clientHost = util.DetectedHostAddress()
-		}
+		clientHost := option.advertisedHost()
 		masterClient = wdclient.NewMasterClient(option.GrpcDialOption, option.FilerGroup, cluster.S3Type, pb.ServerAddress(util.JoinHostPort(clientHost, option.GrpcPort)), option.DataCenter, "", *pb.NewServiceDiscoveryFromMap(masterMap))
 		// Build the object-write lock client and subscribe to the master's
 		// lock-ring updates BEFORE starting the master loop, so the initial

weed/s3api/s3api_server_advertise_test.go

@@ -0,0 +1,37 @@
+package s3api
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestAdvertisedHost(t *testing.T) {
+	detected := util.DetectedHostAddress()
+
+	for _, tc := range []struct {
+		name   string
+		ip     string
+		bindIp string
+		want   string
+	}{
+		{"advertised ip wins over wildcard bind", "localhost", "0.0.0.0", "localhost"},
+		{"advertised ip wins over specific bind", "s3.example.com", "10.0.0.5", "s3.example.com"},
+		{"bind ip used when no advertised ip", "", "10.0.0.5", "10.0.0.5"},
+		{"ipv6 bind ip used when no advertised ip", "", "2001:db8::1", "2001:db8::1"},
+		{"wildcard bind falls back to detected", "", "0.0.0.0", detected},
+		{"ipv6 wildcard bind falls back to detected", "", "::", detected},
+		{"bracketed ipv6 wildcard bind falls back to detected", "", "[::]", detected},
+		{"expanded ipv6 wildcard bind falls back to detected", "", "0:0:0:0:0:0:0:0", detected},
+		{"wildcard advertised ip falls back to bind ip", "0.0.0.0", "10.0.0.5", "10.0.0.5"},
+		{"wildcard advertised ip and bind fall back to detected", "::", "0.0.0.0", detected},
+		{"empty falls back to detected", "", "", detected},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			option := &S3ApiServerOption{Ip: tc.ip, BindIp: tc.bindIp}
+			if got := option.advertisedHost(); got != tc.want {
+				t.Errorf("advertisedHost() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}