mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-23 16:34:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

rdma: expose the RC data plane resource limits as gateway flags

Browse Source 
The hipobj-rc-v2 data plane started with its session, queue pair,
staging and timeout limits hardcoded at the rcserver.Init call
site, so operators could not size the RC plane for their hardware
the way they can for the cuObject backend. Add one flag per limit
plus the READY admission slot count, all defaulting to the values
the gateway passes today, and validate them through a new
rdmamode.V2ValidationError consulted only when the RC data plane
is enabled, mirroring the stale-value handling of the v1 settings.
Counts are parsed as uint64 and range-checked against the uint32
narrowing at the DeviceOpts boundary, and the timeouts carry an
upper bound that keeps the nowMs + timeout deadline arithmetic in
the C core from wrapping.
This commit is contained in:
Jihyeon Gim
2026-09-05 17:53:05 +09:00
parent 7f0a793150
commit 980078d822
3 changed files with 249 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/vgwrdma/main.go
+100 -8
View File
@@ -120,6 +120,15 @@ var (
rdmaCQDepth uint
rdmaRetryCount uint
rdmaTunablesSet bool
rcMaxSessions uint64
rcMaxUserSessions uint64
rcMaxStagingBytes uint64
rcMaxUserStagingBytes uint64
rcMaxQPs uint64
rcMaxUserQPs uint64
rcMaxReadySlots uint64
rcPrepTimeoutMs uint64
rcExecTimeoutMs uint64
)
var (
@@ -898,6 +907,69 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_RDMA_RC_ENABLE"},
Destination: &rdmaRCEnable,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-sessions",
Usage: "maximum concurrent hipobj-rc-v2 sessions (default 1024)",
EnvVars: []string{"VGW_RDMA_RC_MAX_SESSIONS"},
Value: 1024,
Destination: &rcMaxSessions,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-user-sessions",
Usage: "per-principal session limit for the hipobj-rc-v2 data plane (default 64)",
EnvVars: []string{"VGW_RDMA_RC_MAX_USER_SESSIONS"},
Value: 64,
Destination: &rcMaxUserSessions,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-staging-bytes",
Usage: "total staging buffer budget for hipobj-rc-v2 sessions in bytes (default 4294967296)",
EnvVars: []string{"VGW_RDMA_RC_MAX_STAGING_BYTES"},
Value: 4 << 30,
Destination: &rcMaxStagingBytes,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-user-staging-bytes",
Usage: "per-principal staging buffer budget for the hipobj-rc-v2 data plane in bytes (default 1073741824)",
EnvVars: []string{"VGW_RDMA_RC_MAX_USER_STAGING_BYTES"},
Value: 1 << 30,
Destination: &rcMaxUserStagingBytes,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-qps",
Usage: "maximum queue pairs for the hipobj-rc-v2 data plane (default 1024)",
EnvVars: []string{"VGW_RDMA_RC_MAX_QPS"},
Value: 1024,
Destination: &rcMaxQPs,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-user-qps",
Usage: "per-principal queue pair limit for the hipobj-rc-v2 data plane (default 16)",
EnvVars: []string{"VGW_RDMA_RC_MAX_USER_QPS"},
Value: 16,
Destination: &rcMaxUserQPs,
},
&cli.Uint64Flag{
Name: "rdma-rc-max-ready-slots",
Usage: "concurrent READY transfers admitted by the hipobj-rc-v2 data plane (default 64)",
EnvVars: []string{"VGW_RDMA_RC_MAX_READY_SLOTS"},
Value: 64,
Destination: &rcMaxReadySlots,
},
&cli.Uint64Flag{
Name: "rdma-rc-prep-timeout-ms",
Usage: "milliseconds a hipobj-rc-v2 session may wait for READY after PREPARE (default 100000)",
EnvVars: []string{"VGW_RDMA_RC_PREP_TIMEOUT_MS"},
Value: 100000,
Destination: &rcPrepTimeoutMs,
},
&cli.Uint64Flag{
Name: "rdma-rc-exec-timeout-ms",
Usage: "milliseconds a hipobj-rc-v2 READY transfer may run (default 30000)",
EnvVars: []string{"VGW_RDMA_RC_EXEC_TIMEOUT_MS"},
Value: 30000,
Destination: &rcExecTimeoutMs,
},
&cli.UintFlag{
Name: "rdma-port",
Usage: "port for RDMA listener",
@@ -1006,6 +1078,25 @@ func runGateway(ctx context.Context, be backend.Backend) error {
return errors.New(msg)
}
}
if v2On {
// RC-only settings are irrelevant when the hipobj-rc-v2
// data plane is not running; stale environment values
// must not block v1-only or plain-S3 startup.
v2s := rdmamode.V2Settings{
MaxSessions: rcMaxSessions,
MaxUserSessions: rcMaxUserSessions,
MaxStagingBytes: rcMaxStagingBytes,
MaxUserStagingBytes: rcMaxUserStagingBytes,
MaxQPs: rcMaxQPs,
MaxUserQPs: rcMaxUserQPs,
MaxReadySlots: rcMaxReadySlots,
TPrepMs: rcPrepTimeoutMs,
TExecMs: rcExecTimeoutMs,
}
if msg := rdmamode.V2ValidationError(v2s); msg != "" {
return errors.New(msg)
}
}
var s3Opts []s3api.Option
if v1On {
@@ -1163,14 +1254,15 @@ func runGateway(ctx context.Context, be backend.Backend) error {
rcSvc, err := rcserver.Init(rcserver.DeviceOpts{
GidHint: rcGidHint,
Port: 1,
MaxSessions: 1024,
MaxUserSessions: 64,
MaxStagingBytes: 4 << 30,
MaxUserStagingBytes: 1 << 30,
MaxQPs: 1024,
MaxUserQPs: 16,
TPrepMs: 100000,
TExecMs: 30000,
MaxSessions: uint32(rcMaxSessions),
MaxUserSessions: uint32(rcMaxUserSessions),
MaxStagingBytes: rcMaxStagingBytes,
MaxUserStagingBytes: rcMaxUserStagingBytes,
MaxQPs: uint32(rcMaxQPs),
MaxUserQPs: uint32(rcMaxUserQPs),
MaxReadySlots: uint32(rcMaxReadySlots),
TPrepMs: rcPrepTimeoutMs,
TExecMs: rcExecTimeoutMs,
})
if err != nil {
return err
internal/rdmamode/rdmamode.go
+52
View File
@@ -21,6 +21,7 @@ import (
"math"
"strings"
"sync"
"time"
"github.com/versity/versitygw/backend"
)
@@ -68,6 +69,57 @@ func V1ValidationError(s V1Settings) string {
}
}
// V2Settings carries the hipobj-rc-v2 tunings that are validated
// only while the RC data plane runs. Counts arrive as uint64 from
// the CLI and narrow to uint32 at the DeviceOpts boundary, so the
// range is checked before the narrowing cast. Timeouts feed
// deadline arithmetic (nowMs + timeout), so an upper bound keeps
// the sum from wrapping.
type V2Settings struct {
MaxSessions uint64
MaxUserSessions uint64
MaxStagingBytes uint64
MaxUserStagingBytes uint64
MaxQPs uint64
MaxUserQPs uint64
MaxReadySlots uint64
TPrepMs uint64
TExecMs uint64
}
// v2TimeoutCeiling bounds the RC timeouts well below the uint64
// wrap point of nowMs + timeout arithmetic in the C core.
const v2TimeoutCeiling = uint64(24 * time.Hour / time.Millisecond)
// V2ValidationError describes the first invalid v2 setting, or
// the empty string when every setting is valid. Stale v2 values
// in the environment must not block v1-only or plain-S3 startup,
// so the gateway consults this only when v2 is on.
func V2ValidationError(s V2Settings) string {
switch {
case s.MaxSessions < 1 || s.MaxSessions > math.MaxUint32:
return fmt.Sprintf("rdma-rc-max-sessions %d is out of range (1-%d)", s.MaxSessions, uint64(math.MaxUint32))
case s.MaxUserSessions < 1 || s.MaxUserSessions > math.MaxUint32:
return fmt.Sprintf("rdma-rc-max-user-sessions %d is out of range (1-%d)", s.MaxUserSessions, uint64(math.MaxUint32))
case s.MaxStagingBytes < 1:
return fmt.Sprintf("rdma-rc-max-staging-bytes %d must be positive", s.MaxStagingBytes)
case s.MaxUserStagingBytes < 1:
return fmt.Sprintf("rdma-rc-max-user-staging-bytes %d must be positive", s.MaxUserStagingBytes)
case s.MaxQPs < 1 || s.MaxQPs > math.MaxUint32:
return fmt.Sprintf("rdma-rc-max-qps %d is out of range (1-%d)", s.MaxQPs, uint64(math.MaxUint32))
case s.MaxUserQPs < 1 || s.MaxUserQPs > math.MaxUint32:
return fmt.Sprintf("rdma-rc-max-user-qps %d is out of range (1-%d)", s.MaxUserQPs, uint64(math.MaxUint32))
case s.MaxReadySlots < 1 || s.MaxReadySlots > math.MaxUint32:
return fmt.Sprintf("rdma-rc-max-ready-slots %d is out of range (1-%d)", s.MaxReadySlots, uint64(math.MaxUint32))
case s.TPrepMs < 1 || s.TPrepMs > v2TimeoutCeiling:
return fmt.Sprintf("rdma-rc-prep-timeout-ms %d is out of range (1-%d)", s.TPrepMs, v2TimeoutCeiling)
case s.TExecMs < 1 || s.TExecMs > v2TimeoutCeiling:
return fmt.Sprintf("rdma-rc-exec-timeout-ms %d is out of range (1-%d)", s.TExecMs, v2TimeoutCeiling)
default:
return ""
}
}
// Closer is the close operation of the RC session service. It is
// idempotent.
type Closer interface{ Close() }
internal/rdmamode/rdmamode_test.go
+97
View File
@@ -256,3 +256,100 @@ func TestRCWrapperChainsOntoOnceBackend(t *testing.T) {
t.Fatalf("base backend shutdown %d times, want 1", shutdowns)
}
}
func v2Defaults() V2Settings {
return V2Settings{
MaxSessions: 1024,
MaxUserSessions: 64,
MaxStagingBytes: 4 << 30,
MaxUserStagingBytes: 1 << 30,
MaxQPs: 1024,
MaxUserQPs: 16,
MaxReadySlots: 64,
TPrepMs: 100000,
TExecMs: 30000,
}
}
func TestV2ValidationDefaultsPass(t *testing.T) {
// The defaults mirror the values the gateway passed before
// the flags existed, so a deployment that sets nothing must
// keep starting.
if msg := V2ValidationError(v2Defaults()); msg != "" {
t.Fatalf("defaults rejected: %q", msg)
}
}
func TestV2ValidationRejectsZero(t *testing.T) {
// Every knob must be positive: counts narrow to uint32 and
// zero would disable a limit or a deadline in the C core.
for name, mutate := range map[string]func(*V2Settings){
"max-sessions": func(s *V2Settings) { s.MaxSessions = 0 },
"max-user-sessions": func(s *V2Settings) { s.MaxUserSessions = 0 },
"max-staging-bytes": func(s *V2Settings) { s.MaxStagingBytes = 0 },
"max-user-staging-bytes": func(s *V2Settings) { s.MaxUserStagingBytes = 0 },
"max-qps": func(s *V2Settings) { s.MaxQPs = 0 },
"max-user-qps": func(s *V2Settings) { s.MaxUserQPs = 0 },
"max-ready-slots": func(s *V2Settings) { s.MaxReadySlots = 0 },
"prep-timeout": func(s *V2Settings) { s.TPrepMs = 0 },
"exec-timeout": func(s *V2Settings) { s.TExecMs = 0 },
} {
s := v2Defaults()
mutate(&s)
if msg := V2ValidationError(s); msg == "" {
t.Fatalf("%s: zero accepted", name)
}
}
}
func TestV2ValidationCountBoundaries(t *testing.T) {
// Counts arrive as uint64 and narrow to uint32 at the
// DeviceOpts boundary: MaxUint32 passes, the first value
// beyond it fails.
fields := []struct {
name string
field *uint64
}{
{"max-sessions", new(uint64)},
{"max-user-sessions", new(uint64)},
{"max-qps", new(uint64)},
{"max-user-qps", new(uint64)},
{"max-ready-slots", new(uint64)},
}
for _, f := range fields {
base := v2Defaults()
f.field = &base.MaxSessions
switch f.name {
case "max-user-sessions":
f.field = &base.MaxUserSessions
case "max-qps":
f.field = &base.MaxQPs
case "max-user-qps":
f.field = &base.MaxUserQPs
case "max-ready-slots":
f.field = &base.MaxReadySlots
}
*f.field = math.MaxUint32
if msg := V2ValidationError(base); msg != "" {
t.Fatalf("%s at MaxUint32 rejected: %q", f.name, msg)
}
*f.field = math.MaxUint32 + 1
if msg := V2ValidationError(base); msg == "" {
t.Fatalf("%s beyond MaxUint32 accepted", f.name)
}
}
}
func TestV2ValidationTimeoutCeiling(t *testing.T) {
// Timeouts feed nowMs + timeout deadline arithmetic in the
// C core, so values past the ceiling are refused.
base := v2Defaults()
base.TPrepMs = v2TimeoutCeiling
if msg := V2ValidationError(base); msg != "" {
t.Fatalf("ceiling rejected: %q", msg)
}
base.TPrepMs = v2TimeoutCeiling + 1
if msg := V2ValidationError(base); msg == "" {
t.Fatal("ceiling+1 accepted")
}
}