feat: version the private IAM protocol between gateway and standalone service

The S3 gateway and the standalone IAM service exchange authorization decisions over the private endpoints, where a version skew is silently unsafe in both directions: an older service drops a request field it does not know (a `Condition` block, say) and evaluates fail-open, while an older gateway ignores a response field it does not know and misses a deny the service intended. Neither side could previously detect either case.

Both peers now declare a protocol version on every exchange via the `X-Vgw-Private-Protocol` header — the gateway on each request, the service on each response, error responses included — and each refuses a peer it cannot serve safely. The service rejects a gateway below `MinClientProtocol` with a `ProtocolMismatch` code; the gateway rejects a service older than the `ProtocolVersion` it speaks, and rejects a response carrying no version at all, since no build of this protocol omits the header and something else answering on that address should not be interpreted as an IAM decision. `ParseProtocolVersion` is shared by both sides and deliberately strict: an unreadable value is a mismatch, never an assumed default.

A new root-signed `/private/version` endpoint reports the protocol version, the minimum client the service will serve, and the build tag (`WithPrivateServerVersion`). It is exempt from the service's own client-version check so it can still answer a gateway the service refuses — which is how that gateway learns why. Being authenticated like every other private endpoint, it also lets the gateway's startup probe verify its own credential and its mTLS transport in the same round trip.

The gateway probes it once in `NewIAMServiceStandalone` rather than discovering a skew as an opaque per-request 500. An incompatible service is fatal after a 30s window, since a gateway that cannot authorize a single request is more useful refusing to start with the reason in its log; an unreachable one is only a warning, because the two processes legitimately start in parallel and every request checks the version regardless. Only conditions that can resolve on their own are retried — a rejected credential is reported immediately.

iamapi/private/errors.go

@@ -30,9 +30,10 @@ import (
 // credential was rotated would tell the *user* their access key doesn't
 // exist.
 const (
-	CodeNoSuchIdentity = "NoSuchIdentity"
-	CodeInvalidToken   = "InvalidToken"
-	CodeBadRequest     = "BadRequest"
+	CodeNoSuchIdentity   = "NoSuchIdentity"
+	CodeInvalidToken     = "InvalidToken"
+	CodeBadRequest       = "BadRequest"
+	CodeProtocolMismatch = "ProtocolMismatch"
 )
 
 // privateAPIError is a minimal local error for failures (like a malformed
@@ -68,6 +69,18 @@ var (
 	}
 )
 
+// errProtocolMismatch reports that the calling gateway speaks a private
+// protocol this build will not serve. Unlike the sentinels above it carries
+// a message built at the call site, since which versions disagreed is the
+// whole diagnosis.
+func errProtocolMismatch(message string) *privateAPIError {
+	return &privateAPIError{
+		status:  http.StatusBadRequest,
+		code:    CodeProtocolMismatch,
+		message: message,
+	}
+}
+
 // mapResolveError translates iamutil's identity-resolution sentinels into
 // the wire errors this protocol reports. Anything unrecognized falls through
 // unchanged and renders as a 500, which is the correct signal: it is a fault

iamapi/private/handlers.go

@@ -25,6 +25,20 @@ import (
 	"github.com/versity/versitygw/internal/sigv4auth"
 )
 
+// handleVersion reports what this build speaks. It is root-signed like every
+// other endpoint here, which is what lets the gateway's startup probe verify
+// its own credential and its mTLS transport in the same round trip that
+// verifies the protocol — a rotated gateway credential is a far more common
+// misconfiguration than a version skew, and an unauthenticated probe would
+// report success right through one.
+func (p *PrivateAPI) handleVersion(ctx fiber.Ctx) error {
+	return ctx.JSON(VersionResponse{
+		Protocol:      ProtocolVersion,
+		MinClient:     MinClientProtocol,
+		ServerVersion: p.serverVersion,
+	})
+}
+
 func (p *PrivateAPI) handleDeriveSigningKey(ctx fiber.Ctx) error {
 	var req DeriveSigningKeyRequest
 	if err := json.Unmarshal(ctx.Body(), &req); err != nil {

iamapi/private/listener.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v3"
+	"github.com/versity/versitygw/debuglogger"
 	"github.com/versity/versitygw/internal/netutil"
 )
 
@@ -53,6 +54,7 @@ func (p *PrivateAPI) ServeMultiPort(addrs []string, tlsOpts netutil.TLSOptions)
 			ln, err = netutil.NewMultiAddrTLSListenerWithOptions(fiber.NetworkTCP, addr, tlsOpts, netutil.ListenerOptions{SocketPerm: p.socketPerm})
 		}
 		if err != nil {
+			closeListeners(listeners)
 			return fmt.Errorf("failed to bind private iam listener %s: %w", addr, err)
 		}
 		listeners = append(listeners, ln)
@@ -62,6 +64,17 @@ func (p *PrivateAPI) ServeMultiPort(addrs []string, tlsOpts netutil.TLSOptions)
 	return p.app.Listener(finalListener, fiber.ListenConfig{DisableStartupMessage: true})
 }
 
+// closeListeners closes already bound listeners so a failed bind part way
+// through ServeMultiPort does not leave the earlier addresses (and unix
+// socket files) held open.
+func closeListeners(listeners []net.Listener) {
+	for _, ln := range listeners {
+		if err := ln.Close(); err != nil {
+			debuglogger.InternalError(fmt.Errorf("close private iam listener %v: %w", ln.Addr(), err))
+		}
+	}
+}
+
 // Shutdown gracefully stops the private endpoint listeners.
 func (p *PrivateAPI) Shutdown() error {
 	return p.app.ShutdownWithTimeout(shutDownDuration)

iamapi/private/private_test.go

@@ -20,6 +20,8 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -115,6 +117,7 @@ func doPrivateRequest(t *testing.T, p *PrivateAPI, method, target, access, secre
 
 	req := httptest.NewRequest(method, target, bytes.NewReader(body))
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(ProtocolHeader, strconv.Itoa(ProtocolVersion))
 	req.ContentLength = int64(len(body))
 
 	hash := sigv4auth.PayloadSHA256Hex(body)
@@ -127,6 +130,28 @@ func doPrivateRequest(t *testing.T, p *PrivateAPI, method, target, access, secre
 	return resp
 }
 
+// doPrivateRequestWithProtocol is doPrivateRequest with the protocol header
+// set to an arbitrary value — including "" for a gateway build that predates
+// versioning and sends none at all.
+func doPrivateRequestWithProtocol(t *testing.T, p *PrivateAPI, target, protocol string, body []byte) *http.Response {
+	t.Helper()
+
+	req := httptest.NewRequest(http.MethodPost, target, bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	if protocol != "" {
+		req.Header.Set(ProtocolHeader, protocol)
+	}
+	req.ContentLength = int64(len(body))
+
+	signPrivateRequest(t, req, testRoot.Access, testRoot.Secret, sigv4auth.PayloadSHA256Hex(body))
+
+	resp, err := p.app.Test(req)
+	if err != nil {
+		t.Fatalf("app.Test: %v", err)
+	}
+	return resp
+}
+
 func readBody(t *testing.T, resp *http.Response) string {
 	t.Helper()
 
@@ -636,6 +661,157 @@ func TestPrivateAPIRejectsUnsignedRequest(t *testing.T) {
 	}
 }
 
+func TestPrivateAPIVersion(t *testing.T) {
+	store, err := storage.New(storage.Config{Dir: t.TempDir()})
+	if err != nil {
+		t.Fatalf("storage.New: %v", err)
+	}
+	p, err := New(store, testRoot, WithPrivateServerVersion("v1.2.3"))
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+
+	resp := doPrivateRequest(t, p, http.MethodPost, VersionPath, testRoot.Access, testRoot.Secret, []byte("{}"))
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", resp.StatusCode, readBody(t, resp))
+	}
+
+	var got VersionResponse
+	if err := json.Unmarshal([]byte(readBody(t, resp)), &got); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if got.Protocol != ProtocolVersion || got.MinClient != MinClientProtocol {
+		t.Errorf("VersionResponse = %+v, want protocol %d minClient %d", got, ProtocolVersion, MinClientProtocol)
+	}
+	if got.ServerVersion != "v1.2.3" {
+		t.Errorf("ServerVersion = %q, want %q", got.ServerVersion, "v1.2.3")
+	}
+}
+
+// TestPrivateAPIVersionRequiresRootCredential confirms the version endpoint is
+// authenticated like every other one here — that is what lets the gateway's
+// startup probe verify its own credential in the same round trip.
+func TestPrivateAPIVersionRequiresRootCredential(t *testing.T) {
+	p, store := newTestServer(t)
+	createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "")
+
+	resp := doPrivateRequest(t, p, http.MethodPost, VersionPath, "AKIAALICE", "alicesecret", []byte("{}"))
+	if resp.StatusCode == http.StatusOK {
+		t.Fatalf("version endpoint served a non-root credential: %s", readBody(t, resp))
+	}
+}
+
+// TestPrivateAPIProtocolHeaderOnEveryResponse covers the success path, an
+// application error, and an unknown route. The last two go through
+// errorHandler, which must not drop the header — a mismatch response that
+// carries no version is the one response an operator most needs it on.
+func TestPrivateAPIProtocolHeaderOnEveryResponse(t *testing.T) {
+	p, store := newTestServer(t)
+	createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "")
+
+	for _, tc := range []struct {
+		name string
+		path string
+		body string
+	}{
+		{"success", ResolveIdentityPath, `{"accessKeyIds":["AKIAALICE"]}`},
+		{"application error", DerivePath, "not json"},
+		{"unknown route", "/private/nope", "{}"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			resp := doPrivateRequest(t, p, http.MethodPost, tc.path, testRoot.Access, testRoot.Secret, []byte(tc.body))
+			if got := resp.Header.Get(ProtocolHeader); got != strconv.Itoa(ProtocolVersion) {
+				t.Errorf("%s = %q, want %q", ProtocolHeader, got, strconv.Itoa(ProtocolVersion))
+			}
+		})
+	}
+}
+
+// TestPrivateAPIRejectsIncompatibleClientProtocol covers every request-header
+// value this build refuses. A gateway too old to be served safely, and one
+// whose version cannot be read at all, are both refused with a code the
+// gateway dispatches on — never served on an assumed version.
+func TestPrivateAPIRejectsIncompatibleClientProtocol(t *testing.T) {
+	p, store := newTestServer(t)
+	createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "")
+
+	for _, tc := range []struct {
+		name     string
+		protocol string
+	}{
+		{"absent", ""},
+		{"empty", " "},
+		{"not a number", "one"},
+		{"signed", "+1"},
+		{"zero", "0"},
+		{"absurdly long", "11111111111111111111"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			resp := doPrivateRequestWithProtocol(t, p, ResolveIdentityPath, tc.protocol, []byte(`{"accessKeyIds":["AKIAALICE"]}`))
+			if resp.StatusCode != http.StatusBadRequest {
+				t.Fatalf("status = %d, want %d; body = %s", resp.StatusCode, http.StatusBadRequest, readBody(t, resp))
+			}
+			body := readBody(t, resp)
+			if !strings.Contains(body, CodeProtocolMismatch) {
+				t.Errorf("body = %s, want code %s", body, CodeProtocolMismatch)
+			}
+			if got := resp.Header.Get(ProtocolHeader); got != strconv.Itoa(ProtocolVersion) {
+				t.Errorf("%s = %q, want the refusing build's own version", ProtocolHeader, got)
+			}
+		})
+	}
+}
+
+// TestPrivateAPIVersionExemptFromClientProtocolCheck confirms the version
+// endpoint answers a gateway this build would otherwise refuse. Without it, a
+// future service that raised MinClientProtocol could not tell an older gateway
+// why it was being turned away.
+func TestPrivateAPIVersionExemptFromClientProtocolCheck(t *testing.T) {
+	p, _ := newTestServer(t)
+
+	resp := doPrivateRequestWithProtocol(t, p, VersionPath, "", []byte("{}"))
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body = %s", resp.StatusCode, readBody(t, resp))
+	}
+	if got := resp.Header.Get(ProtocolHeader); got != strconv.Itoa(ProtocolVersion) {
+		t.Errorf("%s = %q, want %q", ProtocolHeader, got, strconv.Itoa(ProtocolVersion))
+	}
+}
+
+func TestParseProtocolVersion(t *testing.T) {
+	for _, tc := range []struct {
+		value string
+		want  int
+	}{
+		{"1", 1},
+		{"2", 2},
+		{"1000", 1000},
+		{"", 0},
+		{" 1", 0},
+		{"1 ", 0},
+		{"+1", 0},
+		{"-1", 0},
+		{"0", 0},
+		{"1.0", 0},
+		{"v1", 0},
+		{"99999", 0},
+	} {
+		got, err := ParseProtocolVersion(tc.value)
+		if tc.want == 0 {
+			if err == nil {
+				t.Errorf("ParseProtocolVersion(%q) = %d, want an error", tc.value, got)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("ParseProtocolVersion(%q): %v", tc.value, err)
+		}
+		if got != tc.want {
+			t.Errorf("ParseProtocolVersion(%q) = %d, want %d", tc.value, got, tc.want)
+		}
+	}
+}
+
 // createTestRole creates a role with an optional inline permission policy
 // directly against store, the same way createTestUser bypasses the
 // control-plane API. Arn and RoleID are set explicitly because

iamapi/private/server.go

@@ -29,6 +29,7 @@ package private
 import (
 	"fmt"
 	"os"
+	"strconv"
 
 	"github.com/gofiber/fiber/v3"
 	"github.com/gofiber/fiber/v3/middleware/logger"
@@ -44,6 +45,39 @@ const (
 	DerivePath          = "/private/derive-signing-key"
 	EvaluatePath        = "/private/evaluate-policy"
 	ResolveIdentityPath = "/private/resolve-identity"
+	VersionPath         = "/private/version"
+
+	// ProtocolHeader carries the private protocol version each peer speaks.
+	// Both send it: the S3 gateway on every request, this service on every
+	// response, including error responses.
+	ProtocolHeader = "X-Vgw-Private-Protocol"
+
+	// ProtocolVersion is the private protocol version this build speaks, and
+	// MinClientProtocol the oldest S3 gateway it will serve. Together they
+	// express compatibility in both directions, since a skew can be unsafe
+	// from either side:
+	//
+	//   - Bump ProtocolVersion when the gateway starts relying on something
+	//     an older service would silently ignore — a new request field, or a
+	//     new endpoint. An unrecognized field is dropped by json.Unmarshal,
+	//     so an older service evaluating without one (Condition being the
+	//     worked example) is fail-open. The gateway catches this itself by
+	//     refusing a service older than the version it speaks.
+	//
+	//   - Bump both when this service starts returning something an older
+	//     gateway must understand to stay fail-closed — a new deny dimension,
+	//     or a decision matrix that narrows an Allow, as HasSessionPolicy/
+	//     SessionDecisions would have been had they landed later. An older
+	//     gateway cannot detect this on its own: it does not know the field
+	//     exists. This service refuses it instead.
+	//
+	//   - Bump neither for an addition an older gateway can safely ignore.
+	//
+	// Changing an existing field's meaning in place is not a bump; it is a
+	// new route. The operational rule that falls out of all this: upgrade
+	// the IAM service before the gateways.
+	ProtocolVersion   = 1
+	MinClientProtocol = 1
 
 	// privateService is the SigV4 credential-scope service name the S3
 	// gateway signs its own requests to these endpoints with. It's an
@@ -55,10 +89,11 @@ const (
 
 // PrivateAPI is the standalone IAM service's private endpoint set
 type PrivateAPI struct {
-	app        *fiber.App
-	store      storage.Storer
-	socketPerm os.FileMode
-	quiet      bool
+	app           *fiber.App
+	store         storage.Storer
+	socketPerm    os.FileMode
+	quiet         bool
+	serverVersion string
 }
 
 type PrivateAPIOption func(*PrivateAPI)
@@ -77,6 +112,14 @@ func WithPrivateQuiet() PrivateAPIOption {
 	return func(p *PrivateAPI) { p.quiet = true }
 }
 
+// WithPrivateServerVersion sets the build version the version endpoint
+// reports. It is what lets an operator map a protocol number back to an
+// image, so it is worth passing even though nothing decides compatibility
+// on it.
+func WithPrivateServerVersion(version string) PrivateAPIOption {
+	return func(p *PrivateAPI) { p.serverVersion = version }
+}
+
 // New constructs the private endpoint set. root is the identity these
 // endpoints authenticate every request against.
 func New(store storage.Storer, root iammiddleware.RootCredentials, opts ...PrivateAPIOption) (*PrivateAPI, error) {
@@ -102,7 +145,10 @@ func New(store storage.Storer, root iammiddleware.RootCredentials, opts ...Priva
 		}))
 	}
 
+	app.Use("*", p.checkProtocolVersion)
+
 	rootAuth := iammiddleware.VerifyRootOnlySigV4(privateService, &root)
+	app.Post(VersionPath, chainHandlers(rootAuth, p.handleVersion))
 	app.Post(DerivePath, chainHandlers(rootAuth, p.handleDeriveSigningKey))
 	app.Post(EvaluatePath, chainHandlers(rootAuth, p.handleEvaluatePolicy))
 	app.Post(ResolveIdentityPath, chainHandlers(rootAuth, p.handleResolveIdentity))
@@ -122,3 +168,68 @@ func chainHandlers(handlers ...fiber.Handler) fiber.Handler {
 		return nil
 	}
 }
+
+// checkProtocolVersion answers every response with this build's protocol
+// version and refuses a gateway too old for it to serve safely.
+//
+// It runs before root authentication so a version refusal is reported and
+// logged as exactly that, rather than as a misleading 403 about the
+// gateway's credential. The only thing that discloses to an unauthenticated
+// peer is the protocol version, which the response header carries either
+// way, on a listener that is already mTLS- or unix-socket-only.
+func (p *PrivateAPI) checkProtocolVersion(ctx fiber.Ctx) error {
+	ctx.Set(ProtocolHeader, strconv.Itoa(ProtocolVersion))
+
+	// The version endpoint answers even a gateway this build refuses to
+	// serve: it is how that gateway finds out what it is talking to. Keyed
+	// on the path rather than on registration order, since a route
+	// registered ahead of this middleware would not get the header set
+	// above either.
+	if ctx.Path() == VersionPath {
+		return ctx.Next()
+	}
+
+	client, err := ParseProtocolVersion(ctx.Get(ProtocolHeader))
+	if err != nil {
+		return errProtocolMismatch(err.Error())
+	}
+	if client < MinClientProtocol {
+		return errProtocolMismatch(fmt.Sprintf(
+			"gateway speaks private protocol %d, this IAM service requires %d or newer: upgrade the gateway",
+			client, MinClientProtocol))
+	}
+
+	return ctx.Next()
+}
+
+// ParseProtocolVersion reads a ProtocolHeader value. It is shared by both
+// peers so they agree on what the header means, and is deliberately strict:
+// a value it cannot read is a mismatch, never a default. Treating an absent
+// or unreadable version as some assumed one is the fail-open direction for
+// the check everything else is gated on.
+func ParseProtocolVersion(value string) (int, error) {
+	if value == "" {
+		return 0, fmt.Errorf("no %s header", ProtocolHeader)
+	}
+	// Bounded before it is echoed into an error: this value is attacker-
+	// controlled and ends up in a log line.
+	if len(value) > maxProtocolDigits {
+		return 0, fmt.Errorf("malformed %s header", ProtocolHeader)
+	}
+	// Digits only. Atoi by itself would also accept a leading sign, which is
+	// not a version.
+	for i := 0; i < len(value); i++ {
+		if value[i] < '0' || value[i] > '9' {
+			return 0, fmt.Errorf("malformed %s header %q", ProtocolHeader, value)
+		}
+	}
+	version, err := strconv.Atoi(value)
+	if err != nil || version < 1 {
+		return 0, fmt.Errorf("malformed %s header %q", ProtocolHeader, value)
+	}
+	return version, nil
+}
+
+// maxProtocolDigits bounds a protocol version's wire length, so an
+// arbitrarily long header value never reaches a log line.
+const maxProtocolDigits = 4

iamapi/private/types.go

@@ -112,3 +112,17 @@ type EvaluatePolicyResponse struct {
 	HasSessionPolicy bool       `json:"hasSessionPolicy,omitempty"`
 	PrincipalArn     string     `json:"principalArn,omitempty"`
 }
+
+// VersionResponse is the version endpoint's body.
+//
+// MinClient is load-bearing, and this is the only place it appears: the
+// version endpoint is exempt from the service's own client-version check so
+// that it can answer a gateway the service will not serve, which leaves the
+// gateway to draw that conclusion itself from this field. Protocol duplicates
+// the ProtocolHeader every response carries, and ServerVersion is the build
+// tag — what maps a protocol number back to an image during a rollout.
+type VersionResponse struct {
+	Protocol      int    `json:"protocol"`
+	MinClient     int    `json:"minClient"`
+	ServerVersion string `json:"serverVersion,omitempty"`
+}