rdma: publish RC transfer outcomes into the gateway operational services

Two-phase RC transfers were invisible to the access log, metrics,
and bucket notifications: every outcome record on the S3 surface is
driven by the fiber request context, and the RC wire requests carry
the transfer session rather than the object.

Publish one record per session by tracking the operational context
from PREPARE through completion. A tracker table registers each
successfully prepared session with its account, region, object, and
start time; whichever path confirms the final outcome first (READY
completion, READY failure, or the reaper teardown callback) claims
the entry and publishes exactly once. Sessions that end before
PREPARE succeeds publish a request record directly.

The record is emitted through a synthetic fiber context carrying
the object path and the captured locals, so the existing logger,
metrics manager, and event sender produce the same schema as the
S3 surface without any interface change. Bucket notifications fire
for completed PUTs.

The gateway creates the operational services inside RunVersityGW,
after the RC routes exist. Add an OnServicesReady callback to the
embedded gateway config and wire it in vgwrdma to hand the services
to the RC routes and install the teardown callback.
This commit is contained in:
Jihyeon Gim
2026-09-09 13:16:52 +09:00
parent 138504dbc9
commit 49b1c6f6bc
8 changed files with 526 additions and 2 deletions
+11
View File
@@ -1294,6 +1294,17 @@ func runGateway(ctx context.Context, be backend.Backend) error {
return ctx.Next()
}
rcH := rcroutes.New(rcSvc, be, iamSvc, readonly, disableACLs)
// The gateway builds the access logger, metrics manager,
// and event sender inside RunVersityGW; hand them to the
// RC routes as soon as they exist so finished transfers
// publish into them.
cfg.OnServicesReady = func(s embedgw.OpsServices) {
rcH.SetOpsServices(rcroutes.OpsServices{
Logger: s.Logger,
Metrics: s.Metrics,
Events: s.Events,
})
}
cfg.S3Options = append(s3Opts,
s3api.WithRoute("POST", "/.hipobj-rc/prepare", rcAuth, rcH.Prepare),
s3api.WithRoute("POST", "/.hipobj-rc/ready", rcAuth, rcH.Ready),
+23
View File
@@ -520,6 +520,13 @@ type Config struct {
// as request middleware.
S3Options []s3api.Option
// OnServicesReady runs after the operational services (access
// logger, metrics manager, event sender) are created and before
// the S3 server is built, so embedders can wire them into
// components constructed earlier (such as the RDMA control
// routes). A nil callback is skipped.
OnServicesReady func(OpsServices)
// Version, Build, and BuildTime are displayed in the startup banner.
// All three are optional; omit or leave empty to suppress the field.
Version string
@@ -527,6 +534,14 @@ type Config struct {
BuildTime string
}
// OpsServices bundles the operational service instances handed to
// the Config.OnServicesReady callback.
type OpsServices struct {
Logger s3log.AuditLogger
Metrics metrics.Manager
Events s3event.S3EventSender
}
// TODO: remove gatewayRunning once package-level globals (bucket-name
// validation, debug logging) are eliminated and concurrent calls are safe.
var gatewayRunning atomic.Bool
@@ -853,6 +868,14 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error {
}))
}
if cfg.OnServicesReady != nil {
cfg.OnServicesReady(OpsServices{
Logger: loggers.S3Logger,
Metrics: metricsManager,
Events: evSender,
})
}
srv, err := s3api.New(be, middlewares.RootUserConfig{
Access: cfg.RootUserAccess,
Secret: cfg.RootUserSecret,
+277
View File
@@ -0,0 +1,277 @@
// Copyright 2026 Versity Software
// Copyright 2026 Gluesys Inc. and Jihyeon Gim
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build linux && amd64 && cgo
package rcroutes
import (
"errors"
"fmt"
"sync"
"time"
"github.com/gofiber/fiber/v3"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/rdma/rcserver"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
// OpsServices carries the operational service instances the RC routes
// report into. All three may be nil; publication then becomes a no-op
// so the control plane works without any configured backend.
type OpsServices struct {
Logger s3log.AuditLogger
Metrics metrics.Manager
Events s3event.S3EventSender
}
// opsEmitter is the operational context captured at PREPARE and held
// until the final outcome is known: enough to synthesize an access
// record carrying the session's object rather than the wire path.
type opsEmitter struct {
ops OpsServices
app *fiber.App
acct auth.Account
region string
bucket string
key string
isPut bool
start time.Time
}
// synthesize builds a fiber context whose path and request locals
// describe the session's logical object operation, so the standard
// access-log and event pipelines observe GET/PUT of bucket/key
// instead of the fixed RDMA control path.
func (e *opsEmitter) synthesize() (fiber.Ctx, func()) {
ctx := e.app.AcquireCtx(&fasthttp.RequestCtx{})
method := fiber.MethodGet
if e.isPut {
method = fiber.MethodPut
}
ctx.Method(method)
// The access logger and the event schema both split this path
// into bucket/key, so the synthesized path must be the object
// path in canonical form.
ctx.Path("/" + e.bucket + "/" + e.key)
utils.ContextKeyAccount.Set(ctx, e.acct)
utils.ContextKeyRegion.Set(ctx, e.region)
utils.ContextKeyStartTime.Set(ctx, e.start)
utils.ContextKeyIsRoot.Set(ctx, false)
requestID, hostID := utils.EnsureRequestIDs(ctx)
ctx.Request().Header.Add("X-Amz-Request-Id", requestID)
ctx.Request().Header.Add("X-Amz-Id-2", hostID)
return ctx, func() { e.app.ReleaseCtx(ctx) }
}
// publish emits the final audit record, request metric, and (for a
// committed PUT) the object-created event. Exactly-once delivery is
// the tracker's job; this method just performs one emission.
func (e *opsEmitter) publish(err error, bytes int64) {
if e == nil || (e.ops.Logger == nil && e.ops.Metrics == nil && e.ops.Events == nil) {
return
}
ctx, release := e.synthesize()
defer release()
action := metrics.ActionGetObject
if e.isPut {
action = metrics.ActionPutObject
}
status := httpStatusFromError(err)
if e.ops.Metrics != nil {
e.ops.Metrics.Send(ctx, err, action, bytes, status)
}
if e.ops.Logger != nil {
e.ops.Logger.Log(ctx, err, nil, s3log.LogMeta{
Action: action,
})
}
// The object-created event fires at commit time only; error
// publications never carry it.
if e.ops.Events != nil && err == nil && e.isPut {
e.ops.Events.SendEvent(ctx, s3event.EventMeta{
EventName: s3event.EventObjectCreatedPut,
ObjectSize: bytes,
})
}
}
// httpStatusFromError maps an operation error to the HTTP status
// the S3 surface would have answered with.
func httpStatusFromError(err error) int {
if err == nil {
return 200
}
var serr s3err.S3Error
if errors.As(err, &serr) {
return serr.StatusCode()
}
return 500
}
// sessionRecord is one tracked session with its captured context.
type sessionRecord struct {
emit *opsEmitter
done bool
pending bool
}
// opsTracker owns terminal publication: each session (and each
// pre-session request) is published exactly once, by whichever path
// confirms the final outcome first. It carries its own throwaway
// fiber app: the synthesized contexts only carry path and locals,
// never route state, so they must not share the gateway app.
type opsTracker struct {
mu sync.Mutex
sessions map[string]*sessionRecord
ops OpsServices
app *fiber.App
}
func newOpsTracker() *opsTracker {
return &opsTracker{
sessions: map[string]*sessionRecord{},
app: fiber.New(),
}
}
// SetOpsServices installs the operational service instances. The
// gateway creates the logger, metrics manager, and event sender
// after the RC routes exist, so the tracker starts empty and the
// embedder injects them once RunVersityGW has built them. Sessions
// registered before the injection publish nothing (there are none:
// the server is not listening yet).
func (t *opsTracker) SetOpsServices(ops OpsServices) {
t.mu.Lock()
defer t.mu.Unlock()
t.ops = ops
}
// opsSnapshot returns the current service set under the lock.
func (t *opsTracker) opsSnapshot() OpsServices {
t.mu.Lock()
defer t.mu.Unlock()
return t.ops
}
// register captures the operational context of a successfully created
// session so a later terminal path can publish its final outcome.
func (t *opsTracker) register(sessionID string, acct auth.Account,
region, bucket, key string, isPut bool, start time.Time) {
emit := &opsEmitter{
ops: t.opsSnapshot(),
app: t.app,
acct: acct,
region: region,
bucket: bucket,
key: key,
isPut: isPut,
start: start,
}
t.mu.Lock()
defer t.mu.Unlock()
t.sessions[sessionID] = &sessionRecord{emit: emit, pending: true}
}
// claim removes the session's publication slot and returns its
// captured context; the second caller gets nil and publishes nothing.
func (t *opsTracker) claim(sessionID string) *opsEmitter {
t.mu.Lock()
defer t.mu.Unlock()
rec, ok := t.sessions[sessionID]
if !ok {
return nil
}
delete(t.sessions, sessionID)
if !rec.pending {
return nil
}
return rec.emit
}
// onTerminal is the native teardown callback path: publish sessions
// that no handler ever claimed (expired, abandoned, or canceled
// without a READY).
func (t *opsTracker) onTerminal(ev rcserver.TerminalEvent) {
if t == nil {
return
}
emit := t.claim(ev.SessionID)
if emit == nil {
return
}
// An expired or abandoned session never reached a final object
// result; the bytes staged for it did not become a transfer.
emit.publish(errSessionExpired, 0)
}
// publishClaimed publishes the terminal record from the request
// path (READY completion or failure). A nil tracker (no operational
// services) and an unknown session (already claimed or never
// registered) are both silent no-ops.
func (t *opsTracker) publishClaimed(sessionID string, err error, bytes ...int64) {
if t == nil {
return
}
emit := t.claim(sessionID)
if emit == nil {
return
}
var n int64
if len(bytes) > 0 {
n = bytes[0]
}
emit.publish(err, n)
}
// publishRequest emits an operation record for a request that ended
// before any session existed (authentication, authorization, or
// header failures): no tracking table entry, single emission.
func (t *opsTracker) publishRequest(ctx fiber.Ctx, err error,
bucket, key string, isPut bool) {
if t == nil {
return
}
emit := &opsEmitter{
ops: t.ops,
app: t.app,
region: regionFromCtx(ctx),
bucket: bucket,
key: key,
isPut: isPut,
start: time.Now(),
}
emit.publish(err, 0)
}
// regionFromCtx reads the region the gateway middleware stored on
// the live request; the synthesized publication reuses it.
func regionFromCtx(ctx fiber.Ctx) string {
if v, ok := utils.ContextKeyRegion.Get(ctx).(string); ok {
return v
}
return ""
}
var errSessionExpired = fmt.Errorf("session expired")
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2026 Versity Software
// Copyright 2026 Gluesys Inc. and Jihyeon Gim
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
<<<<<<< Updated upstream
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific
// language governing permissions and limitations under the
// License.
=======
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
>>>>>>> Stashed changes
//go:build linux && amd64 && cgo
package rcroutes
import (
"errors"
"testing"
"time"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/rdma/rcserver"
"github.com/versity/versitygw/s3err"
)
// The tests exercise the tracker's ownership semantics: the session
// table is the single source of truth for who may publish, so the
// assertions watch table membership rather than the emission itself
// (emission is a no-op without operational services wired in).
func TestOpsTrackerExpiryPublication(t *testing.T) {
tr := newOpsTracker()
tr.register("sess-1", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", false, time.Now())
if got := len(tr.sessions); got != 1 {
t.Fatalf("registered sessions = %d, want 1", got)
}
// The reaper path claims and removes the session.
tr.onTerminal(rcserver.TerminalEvent{SessionID: "sess-1"})
if got := len(tr.sessions); got != 0 {
t.Fatalf("session survived terminal: %d", got)
}
// A second terminal (double reap) finds nothing.
tr.onTerminal(rcserver.TerminalEvent{SessionID: "sess-1"})
if got := len(tr.sessions); got != 0 {
t.Fatalf("double terminal left residue: %d", got)
}
}
func TestOpsTrackerRequestPathClaims(t *testing.T) {
tr := newOpsTracker()
tr.register("sess-2", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", true, time.Now())
// READY completion claims the publication first.
tr.publishClaimed("sess-2", nil, 4096)
if got := len(tr.sessions); got != 0 {
t.Fatalf("request path left session: %d", got)
}
// The late reaper callback finds nothing left.
tr.onTerminal(rcserver.TerminalEvent{SessionID: "sess-2"})
if got := len(tr.sessions); got != 0 {
t.Fatalf("reaper re-added session: %d", got)
}
}
func TestOpsTrackerUnknownSession(t *testing.T) {
tr := newOpsTracker()
// Unknown sessions and the nil tracker are silent no-ops.
var nilTracker *opsTracker
nilTracker.publishClaimed("ghost", nil, 1)
nilTracker.onTerminal(rcserver.TerminalEvent{SessionID: "ghost"})
tr.publishClaimed("ghost", nil, 1)
tr.onTerminal(rcserver.TerminalEvent{SessionID: "ghost"})
if got := len(tr.sessions); got != 0 {
t.Fatalf("ghost session materialized: %d", got)
}
}
func TestHttpStatusFromError(t *testing.T) {
if got := httpStatusFromError(nil); got != 200 {
t.Fatalf("nil error => %d, want 200", got)
}
if got := httpStatusFromError(errors.New("x")); got != 500 {
t.Fatalf("plain error => %d, want 500", got)
}
if got := httpStatusFromError(s3err.GetAPIError(s3err.ErrAccessDenied)); got != 403 {
t.Fatalf("access denied => %d, want 403", got)
}
}
+39 -2
View File
@@ -32,6 +32,7 @@ import (
"net/url"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -76,13 +77,31 @@ type Handler struct {
iam auth.IAMService
readonly bool
disableACL bool
// ops owns terminal publication into the operational services
// (access log, request metrics, object events); nil keeps the
// routes uninstrumented.
ops *opsTracker
}
// New builds the route handler around a started RC service.
// SetOpsServices injects the operational service instances once the
// gateway has created them, and wires the native teardown callback
// that publishes sessions no request path ever completed.
func (h *Handler) SetOpsServices(ops OpsServices) {
if h.ops == nil {
return
}
h.ops.SetOpsServices(ops)
h.svc.SetTerminalNotify(h.ops.onTerminal)
}
// New builds the route handler around a started RC service. The
// operational services arrive later through SetOpsServices, once
// the gateway has created them.
func New(svc *rcserver.RCSvc, be backend.Backend, iam auth.IAMService,
readonly, disableACL bool) *Handler {
return &Handler{svc: svc, be: be, iam: iam,
readonly: readonly, disableACL: disableACL}
readonly: readonly, disableACL: disableACL,
ops: newOpsTracker()}
}
// principalID derives the session identity digest from the
@@ -159,6 +178,7 @@ func (h *Handler) prepareCore(ctx fiber.Ctx) error {
// Authorize through the regular object-access chain.
if err := h.authorize(ctx, acct, isRoot, bucket, key, isPut); err != nil {
h.ops.publishRequest(ctx, err, bucket, key, isPut)
return err
}
@@ -173,6 +193,7 @@ func (h *Handler) prepareCore(ctx fiber.Ctx) error {
ClientToken: ctx.Get(hdrToken),
})
if err != nil {
h.ops.publishRequest(ctx, mapRcError(err), bucket, key, isPut)
return mapRcError(err)
}
@@ -181,13 +202,23 @@ func (h *Handler) prepareCore(ctx fiber.Ctx) error {
if !isPut {
if err := h.stageGet(ctx, resp.SessionID, bucket, key, offset, size); err != nil {
_ = h.svc.FinishPrepare(resp.SessionID, false)
h.ops.publishRequest(ctx, err, bucket, key, isPut)
return err
}
}
if err := h.svc.FinishPrepare(resp.SessionID, true); err != nil {
h.ops.publishRequest(ctx, mapRcError(err), bucket, key, isPut)
return mapRcError(err)
}
// The session now owns the operation record: the terminal
// path (READY/FinishPut completion, CANCEL, or the expiry
// reaper) publishes the final outcome exactly once.
if h.ops != nil {
h.ops.register(resp.SessionID, acct,
regionFromCtx(ctx), bucket, key, isPut, time.Now())
}
// Wire reply per the hipobj-rc-v2 contract: protocol echo,
// the server endpoint as "200:<token>", session id, and PSN.
ctx.Set(hdrProtocol, protocolValue)
@@ -380,6 +411,7 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
finalized = true
}
if err != nil {
h.ops.publishClaimed(sessionID, err)
return err
}
// The FINAL wire reply carries the stored object's
@@ -387,11 +419,16 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
resp.Etag = put.ETag
resp.VersionID = put.VersionID
} else if err := h.svc.FinishFinal(sessionID); err != nil {
h.ops.publishClaimed(sessionID, mapRcError(err))
return mapRcError(err)
} else {
finalized = true
}
// The transfer completed: publish the terminal record with
// the byte count the data plane reported.
h.ops.publishClaimed(sessionID, nil, int64(resp.BytesTransferred))
// Wire reply per the hipobj-rc-v2 contract: protocol echo,
// cookie echo, transferred bytes, and object metadata.
ctx.Set(hdrProtocol, protocolValue)
+13
View File
@@ -48,3 +48,16 @@ func (h *Handler) Ready(ctx fiber.Ctx) error { return notImplemented(ctx) }
// Cancel is a stub handler that answers 501 Not Implemented.
func (h *Handler) Cancel(ctx fiber.Ctx) error { return notImplemented(ctx) }
// OpsServices carries the operational service instances (stub
// mirror; the fields exist only to keep the embedding surface
// platform-independent).
type OpsServices struct {
Logger any
Metrics any
Events any
}
// SetOpsServices is a stub: without RDMA support there is nothing
// to publish into.
func (h *Handler) SetOpsServices(ops OpsServices) {}
+49
View File
@@ -34,6 +34,8 @@ package rcserver
extern void rcgo_log_sink(void *ctx, int level, char *msg,
char *file, int line);
extern void rcgo_snapshot_cb(rc_session_snapshot *rec, void *ctx);
extern void rcgo_terminal_cb(void *ctx, char *id, int outcome,
uint64_t bytes);
*/
import "C"
@@ -547,6 +549,53 @@ var (
snapshotSink *[]SessionSnapshot
)
// TerminalEvent describes one reaped session, delivered through the
// terminal notification sink.
type TerminalEvent struct {
SessionID string
Outcome int // RC_READY_* value observed at teardown
Bytes uint64
}
// terminalNotify serializes the trampoline callback and stores the
// subscriber. The RC service has a single instance per gateway, so
// one process-wide sink matches the log-sink pattern.
var (
terminalNotifyMu sync.Mutex
terminalNotify func(TerminalEvent)
)
//export rcgo_terminal_cb
func rcgo_terminal_cb(_ unsafe.Pointer, id *C.char, outcome C.int, bytes C.uint64_t) {
if id == nil {
return
}
terminalNotifyMu.Lock()
fn := terminalNotify
terminalNotifyMu.Unlock()
if fn == nil {
return
}
fn(TerminalEvent{
SessionID: C.GoString(id),
Outcome: int(outcome),
Bytes: uint64(bytes),
})
}
// SetTerminalNotify installs the session-teardown callback. The C
// side invokes it with no lock held; the callback must return
// promptly and must not call back into the service.
func (s *RCSvc) SetTerminalNotify(fn func(TerminalEvent)) {
terminalNotifyMu.Lock()
terminalNotify = fn
terminalNotifyMu.Unlock()
if s.srv != nil {
C.rc_server_set_terminal_notify(s.srv,
(*[0]byte)(C.rcgo_terminal_cb), nil)
}
}
//export rcgo_snapshot_cb
func rcgo_snapshot_cb(rec *C.rc_session_snapshot, _ unsafe.Pointer) {
sink := snapshotSink
+10
View File
@@ -178,6 +178,16 @@ func (s *RCSvc) SessionsSnapshot() ([]SessionSnapshot, error) {
return nil, errNotSupported
}
// TerminalEvent is a stub mirror of the linux record.
type TerminalEvent struct {
SessionID string
Outcome int
Bytes uint64
}
// SetTerminalNotify is a stub.
func (s *RCSvc) SetTerminalNotify(fn func(TerminalEvent)) {}
// ReadyTransfer is a stub.
func (s *RCSvc) ReadyTransfer(req ReadyRequest) (*ReadyResponse, error) {
return nil, errNotSupported