// Copyright 2026 Versity Software // This file is licensed under the Apache License, Version 2.0 // (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an "AS // IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either // express or implied. See the License for the specific language // governing permissions and limitations under the License. // Package rdmamode resolves which RDMA paths a gateway runs and // orders their shutdown. It is deliberately free of cgo so the // behavior is testable on any build platform. package rdmamode import ( "fmt" "math" "strings" "sync" "sync/atomic" "time" "github.com/versity/versitygw/backend" ) // Mode resolves which RDMA paths run: the cuObject v1 backend // follows the legacy --rdma-ip contract, and the hipobj-rc-v2 // control routes follow --rdma-rc-enable. Neither flag implies // the other. func Mode(rdmaIP string, rcEnable bool) (v1, v2 bool) { return strings.TrimSpace(rdmaIP) != "", rcEnable } // V1Settings carries the cuObject v1 tunings that are validated // only while the v1 backend runs. type V1Settings struct { Port uint RetryCount uint PoolBufSize int PoolBufCnt int TunablesSet bool NumDCIs int CQDepth uint } // V1ValidationError describes the first invalid v1 setting, or // the empty string when every setting is valid. Stale v1 values // in the environment must not block v2-only or plain-S3 // startup, so the gateway consults this only when v1 is on. func V1ValidationError(s V1Settings) string { switch { case s.Port > 65535: return fmt.Sprintf("rdma-port %d is out of range (0-65535)", s.Port) case s.RetryCount > 7: return fmt.Sprintf("rdma-retry-count %d is out of range (0-7)", s.RetryCount) case s.PoolBufSize <= 0: return fmt.Sprintf("pool-buf-size %d must be positive", s.PoolBufSize) case s.PoolBufCnt <= 0: return fmt.Sprintf("pool-buf-count %d must be positive", s.PoolBufCnt) case s.TunablesSet && s.NumDCIs <= 0: return fmt.Sprintf("rdma-num-dcis %d must be positive", s.NumDCIs) case s.TunablesSet && s.CQDepth > math.MaxUint32: return fmt.Sprintf("rdma-cq-depth %d exceeds the 32-bit limit", s.CQDepth) default: return "" } } // V2Settings carries the hipobj-rc-v2 tunings that are validated // only while the RC data plane runs. Counts arrive as uint64 from // the CLI and narrow to uint32 at the DeviceOpts boundary, so the // range is checked before the narrowing cast. Timeouts feed // deadline arithmetic (nowMs + timeout), so an upper bound keeps // the sum from wrapping. type V2Settings struct { MaxSessions uint64 MaxUserSessions uint64 MaxStagingBytes uint64 MaxUserStagingBytes uint64 MaxQPs uint64 MaxUserQPs uint64 MaxReadySlots uint64 TPrepMs uint64 TExecMs uint64 } // v2TimeoutCeiling bounds the RC timeouts well below the uint64 // wrap point of nowMs + timeout arithmetic in the C core. const v2TimeoutCeiling = uint64(24 * time.Hour / time.Millisecond) // V2ValidationError describes the first invalid v2 setting, or // the empty string when every setting is valid. Stale v2 values // in the environment must not block v1-only or plain-S3 startup, // so the gateway consults this only when v2 is on. func V2ValidationError(s V2Settings) string { switch { case s.MaxSessions < 1 || s.MaxSessions > math.MaxUint32: return fmt.Sprintf("rdma-rc-max-sessions %d is out of range (1-%d)", s.MaxSessions, uint64(math.MaxUint32)) case s.MaxUserSessions < 1 || s.MaxUserSessions > math.MaxUint32: return fmt.Sprintf("rdma-rc-max-user-sessions %d is out of range (1-%d)", s.MaxUserSessions, uint64(math.MaxUint32)) case s.MaxStagingBytes < 1: return fmt.Sprintf("rdma-rc-max-staging-bytes %d must be positive", s.MaxStagingBytes) case s.MaxUserStagingBytes < 1: return fmt.Sprintf("rdma-rc-max-user-staging-bytes %d must be positive", s.MaxUserStagingBytes) case s.MaxQPs < 1 || s.MaxQPs > math.MaxUint32: return fmt.Sprintf("rdma-rc-max-qps %d is out of range (1-%d)", s.MaxQPs, uint64(math.MaxUint32)) case s.MaxUserQPs < 1 || s.MaxUserQPs > math.MaxUint32: return fmt.Sprintf("rdma-rc-max-user-qps %d is out of range (1-%d)", s.MaxUserQPs, uint64(math.MaxUint32)) case s.MaxReadySlots < 1 || s.MaxReadySlots > math.MaxUint32: return fmt.Sprintf("rdma-rc-max-ready-slots %d is out of range (1-%d)", s.MaxReadySlots, uint64(math.MaxUint32)) case s.TPrepMs < 1 || s.TPrepMs > v2TimeoutCeiling: return fmt.Sprintf("rdma-rc-prep-timeout-ms %d is out of range (1-%d)", s.TPrepMs, v2TimeoutCeiling) case s.TExecMs < 1 || s.TExecMs > v2TimeoutCeiling: return fmt.Sprintf("rdma-rc-exec-timeout-ms %d is out of range (1-%d)", s.TExecMs, v2TimeoutCeiling) default: return "" } } // Closer is the close operation of the RC session service. It is // idempotent. type Closer interface{ Close() } // OpsDrainer drains operational publications (audit records, // events) that the RC teardown path queued before the RC service // closes, so nothing is left waiting on sinks that are about to // close. It is idempotent. type OpsDrainer interface{ Shutdown() } // BackendShutdownAfterRC forwards a backend and closes the RC // service before the wrapped backend shuts down. The RC handlers // reference the backend and IAM service, so the RC service must // stop accepting and drain before either dependency is closed by // the gateway lifecycle. The whole shutdown, including the // delegated backend call, runs exactly once: several closers // (the gateway lifecycle and a deferred rollback in the startup // path) may call Shutdown on the same instance, and the wrapped // backends are not guaranteed to be idempotent. type BackendShutdownAfterRC struct { backend.Backend rc Closer ops atomic.Pointer[OpsDrainer] closed sync.Once } // SetOpsDrainer installs the operational publication drainer. The // route handler that owns the publications is built after this // wrapper (it needs the wrapped backend), so the drainer arrives // via this setter; installs after Shutdown ran are dropped, since // the drain window has passed. func (b *BackendShutdownAfterRC) SetOpsDrainer(d OpsDrainer) { b.ops.Store(&d) } // Shutdown closes the RC service, drains operational publications, // then shuts the wrapped backend down, once. The drain runs AFTER // the RC close: closing RC quiesces the native reaper (every // teardown callback has returned by the time Close returns), so no // producer can enqueue behind the drain - enqueue-then-worker-exit // stranding is impossible by ordering rather than by locking. func (b *BackendShutdownAfterRC) Shutdown() { b.closed.Do(func() { b.rc.Close() if d := b.ops.Load(); d != nil { (*d).Shutdown() } b.Backend.Shutdown() }) } // ShutdownOnceBackend makes a backend Shutdown idempotent. The // gateway startup path defers a rollback close while the gateway // lifecycle also shuts its backend down; a bare backend has no // idempotency guarantee of its own. type ShutdownOnceBackend struct { backend.Backend once sync.Once } // Shutdown closes the wrapped backend at most once. func (b *ShutdownOnceBackend) Shutdown() { b.once.Do(func() { b.Backend.Shutdown() }) } // WrapShutdownOnce returns be with an idempotent Shutdown. func WrapShutdownOnce(be backend.Backend) backend.Backend { return &ShutdownOnceBackend{Backend: be} } // WrapBackendShutdownAfterRC returns be with a Shutdown that // closes rc first. func WrapBackendShutdownAfterRC(be backend.Backend, rc Closer) backend.Backend { return &BackendShutdownAfterRC{Backend: be, rc: rc} }