Files
versitygw/iamapi/server.go
T
Tung Lam e63998fb31 fix: log the client IP from a trusted proxy header
Behind a reverse proxy the request log and the S3, admin, website and WebUI
access logs record the proxy's address instead of the client's. Add a
client-ip-header option that takes the logged address from X-Forwarded-For or
X-Real-Ip. It is off by default, so an existing deployment logs exactly what it
logged before.

The header is only honoured when the connecting peer is listed in the new
trusted-proxies list of IPs and CIDRs. Without that, the value is whatever the
client sent: nginx's standard
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for appends to a
client-supplied header, leaving the client's own address leftmost.
X-Forwarded-For is walked right to left, skipping the listed proxies, so the
first unlisted address is the client and a forged leftmost entry is never
reached; the two-proxy case from #2243 works once both proxies are listed.
Every header line is walked (a proxy may add its own line instead of
appending), ip:port entries are accepted, and a request from an unlisted peer
keeps the socket address. Setting the header with an empty trusted-proxies list
fails startup rather than silently trusting every peer.

X-Real-Ip reads the last header line, so a proxy that adds a line rather than
replacing the client's is the one that speaks. An address carrying an IPv6 zone
is rejected: the zone is free text that the space-delimited access log would
write verbatim, shifting every field after RemoteIP. When every hop in
X-Forwarded-For is a trusted proxy, the leftmost hop is logged, as nginx's
real_ip_recursive does, since the socket peer is then the nearest proxy. The
trusted-proxies list accepts the entry "unix" to trust the peer of a listener
bound to a unix socket path, which fasthttp reports as 0.0.0.0.

The resolver, middleware, log tag and parsers live in internal/clientip, shared
by s3api, iamapi, website and webui; s3api/utils re-exports them for the S3
callers, so the standalone IAM service no longer pulls in the S3 side. On this
branch go list -deps ./iamapi drops from 375 to 299 packages, against 298 on
main.

ctx.IP() is unchanged and still returns the socket peer address, because it
feeds aws:SourceIp in IAM policy conditions and SourceIP in policy evaluation;
only the address written to the logs and S3 event records comes from the
header.

Fixes #2243.
2026-10-16 00:51:53 +00:00

412 lines
14 KiB
Go

// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package iamapi
import (
"fmt"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/gofiber/fiber/v3"
"github.com/gofiber/fiber/v3/middleware/logger"
"github.com/gofiber/fiber/v3/middleware/recover"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/iamapi/internal/iammiddleware"
"github.com/versity/versitygw/iamapi/internal/iamutil"
"github.com/versity/versitygw/iamapi/storage"
"github.com/versity/versitygw/internal/clientip"
"github.com/versity/versitygw/internal/netutil"
)
const (
shutDownDuration = time.Second * 10
requestHeaderMaxSize = 8 * 1024
)
// RootCredentials re-exports the type from iammiddleware so callers only need
// to import iamapi.
type RootCredentials = iammiddleware.RootCredentials
type CertStorage = netutil.CertStorage
func NewCertStorage() *CertStorage {
return netutil.NewCertStorage()
}
type IAMApiServer struct {
Router *IAMApiRouter
app *fiber.App
store storage.Storer
rootCreds *RootCredentials
CertStorage *CertStorage
quiet bool
keepAlive bool
health string
maxConnections int
maxRequests int
socketPerm os.FileMode
onListen func()
// oidc holds the OIDC provider settings threaded into the router,
// controller, and policy middleware; see OIDCConfig.
oidc OIDCConfig
// corsAllowOrigin is the single origin browsers may call this API from
corsAllowOrigin string
// clientIPHeader and trustedProxies control the address recorded in the
// request log; see WithClientIPHeader and WithTrustedProxies.
clientIPHeader string
trustedProxies clientip.TrustedProxies
}
// OIDCConfig groups the settings that govern how this API treats OIDC
// identity providers: whether CreateOpenIDConnectProvider may reach out for
// a thumbprint at all, and how strictly a provider's endpoint is validated
// and fetched from. The zero value is the default AWS-matching posture.
type OIDCConfig struct {
// ThumbprintAutoFetchDisabled disables CreateOpenIDConnectProvider's
// TLS auto-fetch fallback when ThumbprintList is omitted; see
// WithOIDCThumbprintAutoFetchDisabled.
ThumbprintAutoFetchDisabled bool
// AllowPrivateEndpoints permits OIDC provider URLs that resolve to
// loopback/private/link-local addresses, and that carry an explicit
// port; see WithOIDCAllowPrivateEndpoints.
AllowPrivateEndpoints bool
// AllowInsecureTransport permits plaintext http OIDC provider URLs and
// drops TLS verification for https ones; see
// WithOIDCAllowInsecureTransport.
AllowInsecureTransport bool
// DiscoveryURLs holds "<provider url>=<discovery url>" pairs, each
// redirecting one provider's discovery-document fetch; see
// WithOIDCDiscoveryURLs.
DiscoveryURLs []string
// discovery is DiscoveryURLs parsed and keyed by stored provider Url,
// built by New.
discovery map[string]string
}
// endpointPolicy projects the endpoint relaxations into the form iamutil's
// URL-validation and fetch helpers take.
func (c OIDCConfig) endpointPolicy() iamutil.OIDCEndpointPolicy {
return iamutil.OIDCEndpointPolicy{
AllowPrivateEndpoints: c.AllowPrivateEndpoints,
AllowInsecureTransport: c.AllowInsecureTransport,
DiscoveryURLs: c.discovery,
}
}
// parseDiscoveryURLs turns DiscoveryURLs' pairs into the map the endpoint
// policy takes, keyed by stored provider Url so a lookup by a provider's
// stored form hits directly. The provider Url is held to the same rules
// CreateOpenIDConnectProvider applies, so a pair naming a provider that could
// never be registered fails here; the discovery URL must be an absolute
// http or https URL, since it is fetched exactly as written.
func (c *OIDCConfig) parseDiscoveryURLs() error {
if len(c.DiscoveryURLs) == 0 {
return nil
}
c.discovery = make(map[string]string, len(c.DiscoveryURLs))
for _, pair := range c.DiscoveryURLs {
providerURL, discoveryURL, ok := cutDiscoveryURLPair(pair)
providerURL, discoveryURL = strings.TrimSpace(providerURL), strings.TrimSpace(discoveryURL)
if !ok || providerURL == "" {
return fmt.Errorf("iamapi: oidc discovery url %q must be in <provider url>=<discovery url> form, with an http:// or https:// discovery url", pair)
}
stored, err := iamutil.ValidateOIDCProviderURL(providerURL, c.endpointPolicy())
if err != nil {
return fmt.Errorf("iamapi: oidc discovery url %q: invalid provider url %q: %w", pair, providerURL, err)
}
parsed, err := url.Parse(discoveryURL)
if err != nil || (parsed.Scheme != "https" && parsed.Scheme != "http") || parsed.Host == "" {
return fmt.Errorf("iamapi: oidc discovery url %q: invalid discovery url %q", pair, discoveryURL)
}
if parsed.Scheme == "http" && !c.AllowInsecureTransport {
return fmt.Errorf("iamapi: plaintext oidc discovery url %q requires insecure transport to be allowed", discoveryURL)
}
c.discovery[stored] = discoveryURL
}
return nil
}
// cutDiscoveryURLPair splits a "<provider url>=<discovery url>" pair at the
// "=" immediately preceding the discovery URL's scheme rather than at the
// first "=", since a provider Url's path may itself contain "=".
func cutDiscoveryURLPair(pair string) (providerURL, discoveryURL string, ok bool) {
i := -1
for _, sep := range []string{"=https://", "=http://"} {
if j := strings.Index(pair, sep); j >= 0 && (i < 0 || j < i) {
i = j
}
}
if i < 0 {
return "", "", false
}
return pair[:i], pair[i+1:], true
}
func New(store storage.Storer, root RootCredentials, opts ...Option) (*IAMApiServer, error) {
if store == nil {
return nil, fmt.Errorf("iamapi: storer is required")
}
server := &IAMApiServer{
store: store,
rootCreds: &root,
Router: &IAMApiRouter{
store: store,
},
}
for _, opt := range opts {
opt(server)
}
if err := server.oidc.parseDiscoveryURLs(); err != nil {
return nil, err
}
// validate the header once here, so the request path can compare it
// directly instead of re-normalizing on every request
if server.clientIPHeader != "" {
header, err := clientip.NormalizeClientIPHeader(server.clientIPHeader)
if err != nil {
return nil, err
}
server.clientIPHeader = header
}
app := fiber.New(fiber.Config{
AppName: "versitygw-iam",
ServerHeader: "VERSITYGW",
DisableKeepalive: !server.keepAlive,
ErrorHandler: iammiddleware.GlobalErrorHandler,
Concurrency: server.maxConnections,
ReadBufferSize: requestHeaderMaxSize,
StreamRequestBody: false,
})
server.app = app
server.Router.app = app
server.Router.rootCreds = server.rootCreds
server.Router.oidc = server.oidc
app.Use("*", recover.New(recover.Config{
EnableStackTrace: true,
StackTraceHandler: iammiddleware.StackTraceHandler,
}))
if !server.quiet {
app.Use("*", logger.New(logger.Config{
Format: "${time} | vgw-iam | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
CustomTags: map[string]logger.LogFunc{
logger.TagQueryStringParams: debuglogger.RedactedQueryParamsTag,
logger.TagIP: clientip.Tag,
},
}))
}
if server.corsAllowOrigin != "" {
app.Use("*", iammiddleware.CORS(server.corsAllowOrigin))
}
app.Use("*", iammiddleware.RequestIDs())
// resolve the logged client address from a trusted proxy header, when
// one is configured
if server.clientIPHeader != "" {
app.Use("*", clientip.Middleware(server.clientIPHeader, server.trustedProxies))
}
if server.health != "" {
app.Get(server.health, func(ctx fiber.Ctx) error {
return ctx.SendStatus(http.StatusOK)
})
}
if server.maxRequests > 0 {
app.Use("*", iammiddleware.RateLimiter(server.maxRequests))
}
if debuglogger.IsDebugEnabled() {
app.Use("*", iammiddleware.DebugLogger())
}
server.Router.Init()
return server, nil
}
type Option func(*IAMApiServer)
func WithTLS(cs *CertStorage) Option {
return func(s *IAMApiServer) { s.CertStorage = cs }
}
func WithQuiet() Option {
return func(s *IAMApiServer) { s.quiet = true }
}
func WithHealth(health string) Option {
return func(s *IAMApiServer) { s.health = health }
}
func WithKeepAlive() Option {
return func(s *IAMApiServer) { s.keepAlive = true }
}
func WithConcurrencyLimiter(maxConnections, maxRequests int) Option {
return func(s *IAMApiServer) {
s.maxConnections = maxConnections
s.maxRequests = maxRequests
}
}
func WithSocketPerm(perm os.FileMode) Option {
return func(s *IAMApiServer) { s.socketPerm = perm }
}
// WithClientIPHeader records the client address from the named proxy header in
// the request log. The accepted values are clientip.ClientIPHeaderForwardedFor
// and clientip.ClientIPHeaderRealIP; an empty value (the default) keeps the
// socket peer address. The header is only honoured from a peer listed in
// WithTrustedProxies. ctx.IP() is not changed, so SourceIP in policy
// evaluation keeps using the socket address.
func WithClientIPHeader(header string) Option {
return func(s *IAMApiServer) { s.clientIPHeader = header }
}
// WithTrustedProxies sets the peers whose client IP header is trusted, as
// parsed by clientip.ParseTrustedProxies. It only has an effect together with
// WithClientIPHeader.
func WithTrustedProxies(trusted clientip.TrustedProxies) Option {
return func(s *IAMApiServer) { s.trustedProxies = trusted }
}
// WithCORSAllowOrigin sets the Access-Control-Allow-Origin value returned to
// browsers, and enables preflight handling. Required for the WebUI, which
// never shares a port with the IAM API. Empty (the default) skips the CORS
// middleware, leaving the API usable by CLI and SDK clients only.
func WithCORSAllowOrigin(origin string) Option {
return func(s *IAMApiServer) { s.corsAllowOrigin = strings.TrimSpace(origin) }
}
func WithOnListen(fn func()) Option {
return func(s *IAMApiServer) { s.onListen = fn }
}
// WithOIDCThumbprintAutoFetchDisabled disables CreateOpenIDConnectProvider's
// TLS auto-fetch fallback for when ThumbprintList is omitted. When set, an
// omitted ThumbprintList is rejected with a MissingValue error instead of
// the gateway making an outbound TLS connection to the caller-supplied URL
// — an operational safety valve for restricted/air-gapped deployments.
func WithOIDCThumbprintAutoFetchDisabled() Option {
return func(s *IAMApiServer) { s.oidc.ThumbprintAutoFetchDisabled = true }
}
// WithOIDCAllowPrivateEndpoints permits an OIDC provider Url that resolves
// to a loopback/private/link-local address, and one carrying an explicit
// port. Both are refused by default, which makes an IdP that only exists on
// an internal network — a SPIFFE/SPIRE OIDC discovery provider on a cluster
// Service, say — impossible to register or verify tokens against. Transport
// is unaffected: still https, still fully verified.
func WithOIDCAllowPrivateEndpoints() Option {
return func(s *IAMApiServer) { s.oidc.AllowPrivateEndpoints = true }
}
// WithOIDCAllowInsecureTransport permits plaintext http OIDC provider URLs
// and drops TLS certificate verification (ThumbprintList pinning included)
// for https ones, leaving the network path as the only thing authenticating
// the IdP. Intended for an IdP reachable only over a path that is itself
// trusted — a discovery provider bound to loopback as a sidecar in this
// process's own pod.
func WithOIDCAllowInsecureTransport() Option {
return func(s *IAMApiServer) { s.oidc.AllowInsecureTransport = true }
}
// WithOIDCDiscoveryURLs redirects the discovery-document fetch of individual
// providers, taking "<provider url>=<discovery url>" pairs. The discovery URL
// is fetched exactly as given, so it must include the
// "/.well-known/openid-configuration" path when the IdP serves it there.
//
// Only the fetch moves: the provider Url is still what a token's iss claim
// and the fetched document's own issuer field must match, and the JWKS is
// still fetched from the jwks_uri that document publishes. That is what lets
// an IdP hand out tokens naming a public issuer while this gateway reads its
// keys over a cluster-internal path — the endpoints being private is the
// point, so a configured discovery URL and the jwks_uri it publishes are
// exempt from the private-address check without WithOIDCAllowPrivateEndpoints.
func WithOIDCDiscoveryURLs(pairs []string) Option {
return func(s *IAMApiServer) { s.oidc.DiscoveryURLs = pairs }
}
func (s *IAMApiServer) ServeMultiPort(ports []string) error {
if len(ports) == 0 {
return fmt.Errorf("no ports specified")
}
var listeners []net.Listener
for _, portSpec := range ports {
var ln net.Listener
var err error
if s.CertStorage != nil {
ln, err = netutil.NewMultiAddrTLSListener(fiber.NetworkTCP, portSpec, s.CertStorage.GetCertificate, netutil.ListenerOptions{SocketPerm: s.socketPerm})
} else {
ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, portSpec, netutil.ListenerOptions{SocketPerm: s.socketPerm})
}
if err != nil {
closeListeners(listeners)
return fmt.Errorf("failed to bind iam listener %s: %w", portSpec, err)
}
listeners = append(listeners, ln)
}
if len(listeners) == 0 {
return fmt.Errorf("failed to create any iam listeners")
}
finalListener := netutil.NewMultiListener(listeners...)
if s.onListen != nil {
fn := s.onListen
s.app.Hooks().OnListen(func(fiber.ListenData) error {
fn()
return nil
})
}
return s.app.Listener(finalListener, fiber.ListenConfig{
DisableStartupMessage: true,
})
}
// closeListeners closes already bound listeners so a failed bind part way
// through ServeMultiPort does not leave the earlier ports (and unix socket
// files) held open.
func closeListeners(listeners []net.Listener) {
for _, ln := range listeners {
if err := ln.Close(); err != nil {
debuglogger.InternalError(fmt.Errorf("close iam listener %v: %w", ln.Addr(), err))
}
}
}
func (s *IAMApiServer) Shutdown() error {
return s.app.ShutdownWithTimeout(shutDownDuration)
}