diff --git a/cmd/vgwrdma/main.go b/cmd/vgwrdma/main.go index 33306dd3..d771d438 100644 --- a/cmd/vgwrdma/main.go +++ b/cmd/vgwrdma/main.go @@ -120,6 +120,15 @@ var ( rdmaCQDepth uint rdmaRetryCount uint rdmaTunablesSet bool + rcMaxSessions uint64 + rcMaxUserSessions uint64 + rcMaxStagingBytes uint64 + rcMaxUserStagingBytes uint64 + rcMaxQPs uint64 + rcMaxUserQPs uint64 + rcMaxReadySlots uint64 + rcPrepTimeoutMs uint64 + rcExecTimeoutMs uint64 ) var ( @@ -898,6 +907,69 @@ func initFlags() []cli.Flag { EnvVars: []string{"VGW_RDMA_RC_ENABLE"}, Destination: &rdmaRCEnable, }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-sessions", + Usage: "maximum concurrent hipobj-rc-v2 sessions (default 1024)", + EnvVars: []string{"VGW_RDMA_RC_MAX_SESSIONS"}, + Value: 1024, + Destination: &rcMaxSessions, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-user-sessions", + Usage: "per-principal session limit for the hipobj-rc-v2 data plane (default 64)", + EnvVars: []string{"VGW_RDMA_RC_MAX_USER_SESSIONS"}, + Value: 64, + Destination: &rcMaxUserSessions, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-staging-bytes", + Usage: "total staging buffer budget for hipobj-rc-v2 sessions in bytes (default 4294967296)", + EnvVars: []string{"VGW_RDMA_RC_MAX_STAGING_BYTES"}, + Value: 4 << 30, + Destination: &rcMaxStagingBytes, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-user-staging-bytes", + Usage: "per-principal staging buffer budget for the hipobj-rc-v2 data plane in bytes (default 1073741824)", + EnvVars: []string{"VGW_RDMA_RC_MAX_USER_STAGING_BYTES"}, + Value: 1 << 30, + Destination: &rcMaxUserStagingBytes, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-qps", + Usage: "maximum queue pairs for the hipobj-rc-v2 data plane (default 1024)", + EnvVars: []string{"VGW_RDMA_RC_MAX_QPS"}, + Value: 1024, + Destination: &rcMaxQPs, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-user-qps", + Usage: "per-principal queue pair limit for the hipobj-rc-v2 data plane (default 16)", + EnvVars: []string{"VGW_RDMA_RC_MAX_USER_QPS"}, + Value: 16, + Destination: &rcMaxUserQPs, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-max-ready-slots", + Usage: "concurrent READY transfers admitted by the hipobj-rc-v2 data plane (default 64)", + EnvVars: []string{"VGW_RDMA_RC_MAX_READY_SLOTS"}, + Value: 64, + Destination: &rcMaxReadySlots, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-prep-timeout-ms", + Usage: "milliseconds a hipobj-rc-v2 session may wait for READY after PREPARE (default 100000)", + EnvVars: []string{"VGW_RDMA_RC_PREP_TIMEOUT_MS"}, + Value: 100000, + Destination: &rcPrepTimeoutMs, + }, + &cli.Uint64Flag{ + Name: "rdma-rc-exec-timeout-ms", + Usage: "milliseconds a hipobj-rc-v2 READY transfer may run (default 30000)", + EnvVars: []string{"VGW_RDMA_RC_EXEC_TIMEOUT_MS"}, + Value: 30000, + Destination: &rcExecTimeoutMs, + }, &cli.UintFlag{ Name: "rdma-port", Usage: "port for RDMA listener", @@ -1006,6 +1078,25 @@ func runGateway(ctx context.Context, be backend.Backend) error { return errors.New(msg) } } + if v2On { + // RC-only settings are irrelevant when the hipobj-rc-v2 + // data plane is not running; stale environment values + // must not block v1-only or plain-S3 startup. + v2s := rdmamode.V2Settings{ + MaxSessions: rcMaxSessions, + MaxUserSessions: rcMaxUserSessions, + MaxStagingBytes: rcMaxStagingBytes, + MaxUserStagingBytes: rcMaxUserStagingBytes, + MaxQPs: rcMaxQPs, + MaxUserQPs: rcMaxUserQPs, + MaxReadySlots: rcMaxReadySlots, + TPrepMs: rcPrepTimeoutMs, + TExecMs: rcExecTimeoutMs, + } + if msg := rdmamode.V2ValidationError(v2s); msg != "" { + return errors.New(msg) + } + } var s3Opts []s3api.Option if v1On { @@ -1163,14 +1254,15 @@ func runGateway(ctx context.Context, be backend.Backend) error { rcSvc, err := rcserver.Init(rcserver.DeviceOpts{ GidHint: rcGidHint, Port: 1, - MaxSessions: 1024, - MaxUserSessions: 64, - MaxStagingBytes: 4 << 30, - MaxUserStagingBytes: 1 << 30, - MaxQPs: 1024, - MaxUserQPs: 16, - TPrepMs: 100000, - TExecMs: 30000, + MaxSessions: uint32(rcMaxSessions), + MaxUserSessions: uint32(rcMaxUserSessions), + MaxStagingBytes: rcMaxStagingBytes, + MaxUserStagingBytes: rcMaxUserStagingBytes, + MaxQPs: uint32(rcMaxQPs), + MaxUserQPs: uint32(rcMaxUserQPs), + MaxReadySlots: uint32(rcMaxReadySlots), + TPrepMs: rcPrepTimeoutMs, + TExecMs: rcExecTimeoutMs, }) if err != nil { return err diff --git a/internal/rdmamode/rdmamode.go b/internal/rdmamode/rdmamode.go index a5b40aca..92cad9fa 100644 --- a/internal/rdmamode/rdmamode.go +++ b/internal/rdmamode/rdmamode.go @@ -21,6 +21,7 @@ import ( "math" "strings" "sync" + "time" "github.com/versity/versitygw/backend" ) @@ -68,6 +69,57 @@ func V1ValidationError(s V1Settings) string { } } +// V2Settings carries the hipobj-rc-v2 tunings that are validated +// only while the RC data plane runs. Counts arrive as uint64 from +// the CLI and narrow to uint32 at the DeviceOpts boundary, so the +// range is checked before the narrowing cast. Timeouts feed +// deadline arithmetic (nowMs + timeout), so an upper bound keeps +// the sum from wrapping. +type V2Settings struct { + MaxSessions uint64 + MaxUserSessions uint64 + MaxStagingBytes uint64 + MaxUserStagingBytes uint64 + MaxQPs uint64 + MaxUserQPs uint64 + MaxReadySlots uint64 + TPrepMs uint64 + TExecMs uint64 +} + +// v2TimeoutCeiling bounds the RC timeouts well below the uint64 +// wrap point of nowMs + timeout arithmetic in the C core. +const v2TimeoutCeiling = uint64(24 * time.Hour / time.Millisecond) + +// V2ValidationError describes the first invalid v2 setting, or +// the empty string when every setting is valid. Stale v2 values +// in the environment must not block v1-only or plain-S3 startup, +// so the gateway consults this only when v2 is on. +func V2ValidationError(s V2Settings) string { + switch { + case s.MaxSessions < 1 || s.MaxSessions > math.MaxUint32: + return fmt.Sprintf("rdma-rc-max-sessions %d is out of range (1-%d)", s.MaxSessions, uint64(math.MaxUint32)) + case s.MaxUserSessions < 1 || s.MaxUserSessions > math.MaxUint32: + return fmt.Sprintf("rdma-rc-max-user-sessions %d is out of range (1-%d)", s.MaxUserSessions, uint64(math.MaxUint32)) + case s.MaxStagingBytes < 1: + return fmt.Sprintf("rdma-rc-max-staging-bytes %d must be positive", s.MaxStagingBytes) + case s.MaxUserStagingBytes < 1: + return fmt.Sprintf("rdma-rc-max-user-staging-bytes %d must be positive", s.MaxUserStagingBytes) + case s.MaxQPs < 1 || s.MaxQPs > math.MaxUint32: + return fmt.Sprintf("rdma-rc-max-qps %d is out of range (1-%d)", s.MaxQPs, uint64(math.MaxUint32)) + case s.MaxUserQPs < 1 || s.MaxUserQPs > math.MaxUint32: + return fmt.Sprintf("rdma-rc-max-user-qps %d is out of range (1-%d)", s.MaxUserQPs, uint64(math.MaxUint32)) + case s.MaxReadySlots < 1 || s.MaxReadySlots > math.MaxUint32: + return fmt.Sprintf("rdma-rc-max-ready-slots %d is out of range (1-%d)", s.MaxReadySlots, uint64(math.MaxUint32)) + case s.TPrepMs < 1 || s.TPrepMs > v2TimeoutCeiling: + return fmt.Sprintf("rdma-rc-prep-timeout-ms %d is out of range (1-%d)", s.TPrepMs, v2TimeoutCeiling) + case s.TExecMs < 1 || s.TExecMs > v2TimeoutCeiling: + return fmt.Sprintf("rdma-rc-exec-timeout-ms %d is out of range (1-%d)", s.TExecMs, v2TimeoutCeiling) + default: + return "" + } +} + // Closer is the close operation of the RC session service. It is // idempotent. type Closer interface{ Close() } diff --git a/internal/rdmamode/rdmamode_test.go b/internal/rdmamode/rdmamode_test.go index abeaa515..3da8d035 100644 --- a/internal/rdmamode/rdmamode_test.go +++ b/internal/rdmamode/rdmamode_test.go @@ -256,3 +256,100 @@ func TestRCWrapperChainsOntoOnceBackend(t *testing.T) { t.Fatalf("base backend shutdown %d times, want 1", shutdowns) } } + +func v2Defaults() V2Settings { + return V2Settings{ + MaxSessions: 1024, + MaxUserSessions: 64, + MaxStagingBytes: 4 << 30, + MaxUserStagingBytes: 1 << 30, + MaxQPs: 1024, + MaxUserQPs: 16, + MaxReadySlots: 64, + TPrepMs: 100000, + TExecMs: 30000, + } +} + +func TestV2ValidationDefaultsPass(t *testing.T) { + // The defaults mirror the values the gateway passed before + // the flags existed, so a deployment that sets nothing must + // keep starting. + if msg := V2ValidationError(v2Defaults()); msg != "" { + t.Fatalf("defaults rejected: %q", msg) + } +} + +func TestV2ValidationRejectsZero(t *testing.T) { + // Every knob must be positive: counts narrow to uint32 and + // zero would disable a limit or a deadline in the C core. + for name, mutate := range map[string]func(*V2Settings){ + "max-sessions": func(s *V2Settings) { s.MaxSessions = 0 }, + "max-user-sessions": func(s *V2Settings) { s.MaxUserSessions = 0 }, + "max-staging-bytes": func(s *V2Settings) { s.MaxStagingBytes = 0 }, + "max-user-staging-bytes": func(s *V2Settings) { s.MaxUserStagingBytes = 0 }, + "max-qps": func(s *V2Settings) { s.MaxQPs = 0 }, + "max-user-qps": func(s *V2Settings) { s.MaxUserQPs = 0 }, + "max-ready-slots": func(s *V2Settings) { s.MaxReadySlots = 0 }, + "prep-timeout": func(s *V2Settings) { s.TPrepMs = 0 }, + "exec-timeout": func(s *V2Settings) { s.TExecMs = 0 }, + } { + s := v2Defaults() + mutate(&s) + if msg := V2ValidationError(s); msg == "" { + t.Fatalf("%s: zero accepted", name) + } + } +} + +func TestV2ValidationCountBoundaries(t *testing.T) { + // Counts arrive as uint64 and narrow to uint32 at the + // DeviceOpts boundary: MaxUint32 passes, the first value + // beyond it fails. + fields := []struct { + name string + field *uint64 + }{ + {"max-sessions", new(uint64)}, + {"max-user-sessions", new(uint64)}, + {"max-qps", new(uint64)}, + {"max-user-qps", new(uint64)}, + {"max-ready-slots", new(uint64)}, + } + for _, f := range fields { + base := v2Defaults() + f.field = &base.MaxSessions + switch f.name { + case "max-user-sessions": + f.field = &base.MaxUserSessions + case "max-qps": + f.field = &base.MaxQPs + case "max-user-qps": + f.field = &base.MaxUserQPs + case "max-ready-slots": + f.field = &base.MaxReadySlots + } + *f.field = math.MaxUint32 + if msg := V2ValidationError(base); msg != "" { + t.Fatalf("%s at MaxUint32 rejected: %q", f.name, msg) + } + *f.field = math.MaxUint32 + 1 + if msg := V2ValidationError(base); msg == "" { + t.Fatalf("%s beyond MaxUint32 accepted", f.name) + } + } +} + +func TestV2ValidationTimeoutCeiling(t *testing.T) { + // Timeouts feed nowMs + timeout deadline arithmetic in the + // C core, so values past the ceiling are refused. + base := v2Defaults() + base.TPrepMs = v2TimeoutCeiling + if msg := V2ValidationError(base); msg != "" { + t.Fatalf("ceiling rejected: %q", msg) + } + base.TPrepMs = v2TimeoutCeiling + 1 + if msg := V2ValidationError(base); msg == "" { + t.Fatal("ceiling+1 accepted") + } +}