rdma: expose live RC sessions on the admin server

Add a SessionsSnapshot view over the new C ABI entry point and
serve it from the admin server as GET /rc-sessions. The admin server
gains a WithAdminRoute option so an embedding binary can register
extra admin routes that run with the same signature verification and
admin checks as the built-in endpoints; vgwrdma registers the
snapshot there when the RC feature is enabled. The route replies
with the usual XML error surface so unsigned or non-admin requests
get a 403 rather than a generic 500. Stub builds return a not
supported error, keeping the build matrix unchanged.
This commit is contained in:
Jihyeon Gim
2026-09-07 16:14:53 +09:00
parent c07f75a612
commit 6eb574dc1a
8 changed files with 288 additions and 0 deletions
+34
View File
@@ -35,11 +35,14 @@ import (
"github.com/versity/versitygw/embedgw"
"github.com/versity/versitygw/internal/netutil"
"github.com/versity/versitygw/internal/rdmamode"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/rdma"
"github.com/versity/versitygw/rdma/rcroutes"
"github.com/versity/versitygw/rdma/rcserver"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
)
var (
@@ -1296,6 +1299,37 @@ func runGateway(ctx context.Context, be backend.Backend) error {
s3api.WithRoute("POST", "/.hipobj-rc/ready", rcAuth, rcH.Ready),
s3api.WithRoute("POST", "/.hipobj-rc/cancel", rcAuth, rcH.Cancel),
)
// The RC session snapshot rides the standalone admin server
// (started when --admin-port is given) behind the same
// signature verification and admin role check as the other
// admin endpoints.
rcAdminAuth := func(ctx fiber.Ctx) error {
if err := rcVerify(ctx); err != nil {
return err
}
if err := middlewares.IsAdmin(metrics.ActionAdminListBuckets)(ctx); err != nil {
return err
}
return nil
}
// Admin routes answer with the same XML error surface as the
// built-in admin endpoints; the default fiber error handler
// would turn auth failures into a bare 500.
rcAdminRoute := func(ctx fiber.Ctx) error {
err := rcAdminAuth(ctx)
if err == nil {
err = rcH.AdminSnapshot(ctx)
}
if serr, ok := err.(s3err.S3Error); ok {
requestID, hostID := utils.EnsureRequestIDs(ctx)
return ctx.Status(serr.StatusCode()).Send(
serr.XMLBody(requestID, hostID))
}
return err
}
cfg.AdminOptions = append(cfg.AdminOptions,
s3api.WithAdminRoute("GET", "/rc-sessions", rcAdminRoute, rcH.AdminSnapshot),
)
} else {
cfg.S3Options = s3Opts
}
+6
View File
@@ -75,6 +75,11 @@ type Config struct {
// control over the admin endpoint with optionally separate TLS certs.
AdminPorts []string
// AdminOptions carries extra standalone-admin-server options from
// the embedding binary (e.g. additional admin routes). Only used
// when AdminPorts is non-empty.
AdminOptions []s3api.AdminOpt
// MaxConnections is the maximum number of concurrent TCP connections
// accepted by the S3 API server.
MaxConnections int
@@ -873,6 +878,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error {
admOpts := []s3api.AdminOpt{
s3api.WithAdminConcurrencyLimiter(cfg.AdminMaxConnections, cfg.AdminMaxRequests),
}
admOpts = append(admOpts, cfg.AdminOptions...)
if corsAllowOrigin != "" {
admOpts = append(admOpts, s3api.WithAdminCORSAllowOrigin(corsAllowOrigin))
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2026 Versity Software
// Copyright 2026 Gluesys Inc. and Jihyeon Gim
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// Admin session snapshot route for the RC data plane.
//go:build linux && amd64 && cgo
package rcroutes
import (
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/rdma/rcserver"
)
// AdminSnapshot serves the RC session snapshot on the admin surface.
// The route is registered only when the RC plane is enabled and the
// standalone admin server runs; middleware before it has already
// verified the admin signature and role.
func (h *Handler) AdminSnapshot(ctx fiber.Ctx) error {
sessions, err := h.svc.SessionsSnapshot()
if err != nil {
return WriteRouteError(ctx, mapRcError(err))
}
type sessionRecord struct {
SessionID string `json:"session_id"`
Op string `json:"op"`
Target string `json:"target"`
State string `json:"state"`
ReapPending bool `json:"reap_pending"`
AgeMs uint64 `json:"age_ms"`
StagingBytes uint64 `json:"staging_bytes"`
}
stateNames := map[uint8]string{
rcserver.SnapshotStatePrepared: "prepared",
rcserver.SnapshotStatePublishing: "publishing",
rcserver.SnapshotStateTransferring: "transferring",
rcserver.SnapshotStateCompleting: "completing",
rcserver.SnapshotStateReaping: "reaping",
}
out := make([]sessionRecord, 0, len(sessions))
for _, s := range sessions {
name, ok := stateNames[s.State]
if !ok {
name = "unknown"
}
out = append(out, sessionRecord{
SessionID: s.SessionID,
Op: s.Op,
Target: s.Target,
State: name,
ReapPending: s.ReapPending,
AgeMs: s.AgeMs,
StagingBytes: s.StagingBytes,
})
}
return ctx.JSON(fiber.Map{"sessions": out})
}
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2026 Versity Software
// Copyright 2026 Gluesys Inc. and Jihyeon Gim
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// Admin session snapshot route (stub for platforms without RDMA
// support).
//go:build !(linux && amd64 && cgo)
package rcroutes
import (
"errors"
"github.com/gofiber/fiber/v3"
)
// AdminSnapshot is a stub; the route is never registered without the
// RC data plane.
func (h *Handler) AdminSnapshot(ctx fiber.Ctx) error {
return fiber.NewError(fiber.StatusNotFound,
errors.New("rdma rc data plane not available").Error())
}
+15
View File
@@ -1,3 +1,18 @@
// Copyright 2026 Versity Software
// Copyright 2026 Gluesys Inc. and Jihyeon Gim
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build linux && cgo
package rcserver
+73
View File
@@ -33,6 +33,7 @@ package rcserver
// level/file/line/msg are marshalled through this fixed trampoline.
extern void rcgo_log_sink(void *ctx, int level, char *msg,
char *file, int line);
extern void rcgo_snapshot_cb(rc_session_snapshot *rec, void *ctx);
*/
import "C"
@@ -517,6 +518,78 @@ func (s *RCSvc) SessionInfo(sessionID string, who PrincipalID) (*SessionInfo, er
}, nil
}
// SessionSnapshot describes a live RC session for observability.
type SessionSnapshot struct {
SessionID string
Op string
Target string
State uint8 // SessState value, may be ORed with ReapPending
ReapPending bool
AgeMs uint64
StagingBytes uint64
}
// SnapshotState constants mirrored from the C ABI.
const (
SnapshotStatePrepared = 0
SnapshotStatePublishing = 1
SnapshotStateTransferring = 2
SnapshotStateCompleting = 3
SnapshotStateReaping = 4
SnapshotReapPending = 0x80
)
// snapshotReceiver is set by SessionsSnapshot for the duration of the
// C call; the fixed trampoline copies each record into it. The mutex
// covers the case of concurrent snapshot calls.
var (
snapshotMu sync.Mutex
snapshotSink *[]SessionSnapshot
)
//export rcgo_snapshot_cb
func rcgo_snapshot_cb(rec *C.rc_session_snapshot, _ unsafe.Pointer) {
sink := snapshotSink
if sink == nil || rec == nil {
return
}
st := uint8(rec.state)
*sink = append(*sink, SessionSnapshot{
SessionID: C.GoString(&rec.session_id[0]),
Op: C.GoString(&rec.op[0]),
Target: C.GoStringN(&rec.target[0], C.int(rec.target_len)),
State: st &^ SnapshotReapPending,
ReapPending: st&SnapshotReapPending != 0,
AgeMs: uint64(rec.age_ms),
StagingBytes: uint64(rec.staging_bytes),
})
}
// SessionsSnapshot copies every live session into Go-owned records.
// The C side snapshots under its map lock and delivers the copies
// outside it, so the callback cannot block the data plane.
func (s *RCSvc) SessionsSnapshot() ([]SessionSnapshot, error) {
if s.srv == nil {
return nil, errors.New("rcserver: service closed")
}
if !s.TryEnter() {
return nil, ErrInternal
}
defer s.Leave()
snapshotMu.Lock()
defer snapshotMu.Unlock()
var out []SessionSnapshot
snapshotSink = &out
defer func() { snapshotSink = nil }()
rc := C.rc_server_sessions_snapshot(s.srv,
(*[0]byte)(C.rcgo_snapshot_cb), nil)
if rc != C.RC_OK {
return nil, statusError(rc)
}
return out, nil
}
// ReadyTransfer runs the data phase (READY).
func (s *RCSvc) ReadyTransfer(req ReadyRequest) (*ReadyResponse, error) {
if s.srv == nil {
+26
View File
@@ -111,6 +111,27 @@ type SessionInfo struct {
Target string
}
// SessionSnapshot describes a live RC session for observability.
type SessionSnapshot struct {
SessionID string
Op string
Target string
State uint8
ReapPending bool
AgeMs uint64
StagingBytes uint64
}
// Snapshot state values mirrored from the cgo ABI.
const (
SnapshotStatePrepared = 0
SnapshotStatePublishing = 1
SnapshotStateTransferring = 2
SnapshotStateCompleting = 3
SnapshotStateReaping = 4
SnapshotReapPending = 0x80
)
// RCSvc owns the RC session server (stub).
type RCSvc struct{}
@@ -152,6 +173,11 @@ func (s *RCSvc) SessionInfo(sessionID string, who PrincipalID) (*SessionInfo, er
return nil, errNotSupported
}
// SessionsSnapshot is a stub.
func (s *RCSvc) SessionsSnapshot() ([]SessionSnapshot, error) {
return nil, errNotSupported
}
// ReadyTransfer is a stub.
func (s *RCSvc) ReadyTransfer(req ReadyRequest) (*ReadyResponse, error) {
return nil, errNotSupported
+32
View File
@@ -42,6 +42,15 @@ type S3AdminServer struct {
maxConnections int
maxRequests int
socketPerm os.FileMode
extraRoutes []adminRouteMount
}
// adminRouteMount is a route registered on the admin app after the
// built-in admin router ran.
type adminRouteMount struct {
method string
path string
handlers []fiber.Handler
}
func NewAdminServer(be backend.Backend, root middlewares.RootUserConfig, region string, iam auth.IAMService, l s3log.AuditLogger, ctrl controllers.S3ApiController, opts ...AdminOpt) *S3AdminServer {
@@ -95,6 +104,14 @@ func NewAdminServer(be backend.Backend, root middlewares.RootUserConfig, region
server.router.Init(app, be, iam, l, root, region, server.debug, server.corsAllowOrigin)
for _, r := range server.extraRoutes {
args := make([]any, 0, len(r.handlers))
for _, h := range r.handlers {
args = append(args, h)
}
app.Add([]string{r.method}, r.path, args[0], args[1:]...)
}
return server
}
@@ -136,6 +153,21 @@ func WithAdminSocketPerm(perm os.FileMode) AdminOpt {
return func(s *S3AdminServer) { s.socketPerm = perm }
}
// WithAdminRoute registers a route on the standalone admin server,
// after the built-in admin routes and their middleware chain. Use it
// for admin-surface endpoints that do not fit the S3 admin controller
// shape.
func WithAdminRoute(method, path string, handlers ...fiber.Handler) AdminOpt {
return func(s *S3AdminServer) {
copied := append([]fiber.Handler(nil), handlers...)
s.extraRoutes = append(s.extraRoutes, adminRouteMount{
method: method,
path: path,
handlers: copied,
})
}
}
// ServeMultiPort creates listeners for multiple port specifications and serves
// on all of them simultaneously. This supports listening on multiple ports and/or
// addresses (e.g., [":8080", "localhost:8081"]).