rdma: size publications to session capacity and preserve committed PUT facts

Size the publication queue from the configured session limit instead
of a fixed depth with an overflow semaphore: each session publishes
exactly one terminal record, so the queue can never fill and native
callbacks hand off without waiting. Remove the semaphore fallback.

Admit verification failures under the shutdown barrier so an
authentication publication cannot land after the sinks close. Give
the metrics manager a lifetime independent of the gateway context so
shutdown drain updates are counted, and pass the captured bucket
explicitly so RC datapoints appear in bucket-filtered metrics.

Track reservations by claim generation: release and publication
validate the generation, so a stale claim cannot consume a newer
owner's record. Install the reservation cleanup defer immediately
after acquisition so a panic during authorization cannot orphan it.

Preserve committed PUT facts independent of the native finalizer:
when the backend object exists, record the commit, keep the
committed byte count on the error publication, and still emit the
object-created event.
This commit is contained in:
Jihyeon Gim
2026-09-09 13:16:52 +09:00
parent d3e45fdb41
commit 55c82f5e4e
8 changed files with 240 additions and 115 deletions
+10
View File
@@ -1300,6 +1300,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
// logger and event schema read the region from
// there, so set it for every verified request.
utils.ContextKeyRegion.Set(ctx, region)
// Admission barrier: verification can block on IAM
// lookups, and its failure publications must not
// outlive the RC shutdown sequence (sinks close
// right after the RC service drains). Requests that
// lose the race to shutdown are dropped - their
// audit record would land after the sinks closed.
if !rcSvc.TryEnter() {
return rcroutes.WriteRouteError(ctx, rcroutes.ErrNotAdmitted())
}
defer rcSvc.Leave()
if err := rcVerify(ctx); err != nil {
rcH.PublishAuthFailure(ctx, err)
return rcroutes.WriteRouteError(ctx, err)
+10 -1
View File
@@ -799,7 +799,16 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error {
return fmt.Errorf("setup logger: %w", err)
}
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
// The metrics manager must outlive the gateway context: RC
// teardown publications drain during backend shutdown, after
// this context is cancelled. A manager bound to ctx would
// silently discard those final datapoints, so it runs on its
// own context and closes with the other sinks below.
metricsCtx, metricsStop := context.WithCancel(context.Background())
// The cancel runs when this function returns - after the
// shutdown sequence below finishes draining every sink.
defer metricsStop()
metricsManager, err := metrics.NewManager(metricsCtx, metrics.Config{
ServiceName: cfg.MetricsService,
StatsdServers: cfg.StatsdServers,
DogStatsdServers: cfg.DogstatsServers,
+28
View File
@@ -44,6 +44,12 @@ type Tag struct {
// Manager is the interface definition for metrics manager
type Manager interface {
Send(ctx fiber.Ctx, err error, action string, count int64, status int)
// SendWithBucket is Send with the bucket dimension stated
// by the caller. The S3 middleware derives the bucket from
// the matched route, which synthesized contexts (RDMA
// operational records) cannot reproduce: they pass the
// captured bucket explicitly instead.
SendWithBucket(ctx fiber.Ctx, err error, action string, count int64, status int, bucket string)
Close()
}
@@ -137,6 +143,28 @@ func (m *manager) Send(ctx fiber.Ctx, err error, action string, count int64, sta
reqTags = append(reqTags, Tag{Key: "bucket", Value: bucket})
}
m.send(ctx, err, action, count, status, reqTags)
}
// SendWithBucket reports with the bucket dimension supplied by the
// caller; see the Manager interface.
func (m *manager) SendWithBucket(ctx fiber.Ctx, err error, action string, count int64, status int, bucket string) {
if action == "" {
action = ActionUndetected
}
a := ActionMap[action]
reqTags := []Tag{
{Key: "method", Value: ctx.Method()},
{Key: "api", Value: a.Service},
{Key: "action", Value: a.Name},
}
if bucket != "" {
reqTags = append(reqTags, Tag{Key: "bucket", Value: bucket})
}
m.send(ctx, err, action, count, status, reqTags)
}
func (m *manager) send(ctx fiber.Ctx, err error, action string, count int64, status int, reqTags []Tag) {
reqStatus := status
if err != nil {
+81 -58
View File
@@ -68,11 +68,23 @@ type opsEmitter struct {
version string
hasEtag bool
hasVer bool
// committed records that the backend created the object.
// The creation event keys off this fact, not off the final
// publication's error status: a committed PUT whose native
// finalizer later failed still created the object, and its
// creation event must not be lost.
committed bool
// eventSent guards the creation event against a second
// publication path (stashed terminal consumed by a release).
eventSent bool
}
// setCommitMeta records the backend-assigned object metadata for
// the success publication's event payload.
func (e *opsEmitter) setCommitMeta(etag, version string) {
// markCommitted records the backend commit fact together with the
// backend-assigned object metadata. The creation event fires for
// any publication after this, including an error publication from
// a failed native finalizer: the object exists regardless of the
// finalizer's fate.
func (e *opsEmitter) markCommitted(etag, version string) {
if e == nil {
return
}
@@ -80,6 +92,7 @@ func (e *opsEmitter) setCommitMeta(etag, version string) {
e.hasEtag = true
e.version = version
e.hasVer = true
e.committed = true
}
// synthesize builds a fiber context whose path and request locals
@@ -141,7 +154,10 @@ func (e *opsEmitter) publish(err error, bytes int64) {
}
if e.ops.Metrics != nil {
e.ops.Metrics.Send(ctx, sinkErr, action, bytes, status)
// The bucket dimension comes from the captured session,
// not the route: the synthesized context has no matched
// route, so Params("bucket") would be empty here.
e.ops.Metrics.SendWithBucket(ctx, sinkErr, action, bytes, status, e.bucket)
}
if e.ops.Logger != nil {
e.ops.Logger.Log(ctx, sinkErr, nil, s3log.LogMeta{
@@ -152,9 +168,12 @@ func (e *opsEmitter) publish(err error, bytes int64) {
ObjectSize: bytes,
})
}
// The object-created event fires at commit time only; error
// publications never carry it.
if e.ops.Events != nil && err == nil && e.isPut {
// The object-created event keys off the backend commit fact,
// not the publication's error status: a committed PUT whose
// native finalizer failed still created the object, so its
// creation event must survive. Uncommitted PUTs (backend
// failure) never carry it.
if e.ops.Events != nil && e.committed && e.isPut && !e.eventSent {
meta := s3event.EventMeta{
EventName: s3event.EventObjectCreatedPut,
ObjectSize: bytes,
@@ -167,6 +186,7 @@ func (e *opsEmitter) publish(err error, bytes int64) {
ver := e.version
meta.VersionId = &ver
}
e.eventSent = true
e.ops.Events.SendEvent(ctx, meta)
}
}
@@ -219,6 +239,11 @@ type sessionRecord struct {
// will come, so a later release of the reservation resolves
// the stashed outcome instead of leaving an orphan.
terminal bool
// claimGen identifies the current reservation. Each reserve
// bumps it, so a release or publication from an earlier
// reservation is rejected even though the record's emitter
// pointer is reused across claims.
claimGen uint64
}
// opsTracker owns terminal publication: each session publishes
@@ -243,7 +268,6 @@ type opsTracker struct {
sessions map[string]*sessionRecord
app *fiber.App
pubq chan pubJob
overflow chan struct{}
done chan struct{}
drain chan struct{}
drainOnce sync.Once
@@ -256,25 +280,30 @@ type pubJob struct {
byt int64
}
// pubQueueDepth bounds how many publications may wait in the
// handoff queue before the caller falls back to inline execution.
const pubQueueDepth = 256
// pubQueueCapacity returns the publication queue depth for a
// given session limit. Each session publishes exactly one
// terminal record (enforced by the reservation protocol), and the
// session count never exceeds the configured limit, so a queue
// this deep can never fill - native callbacks always hand off
// without waiting. A floor keeps tests and degenerate zero
// configs sane.
func pubQueueCapacity(maxSessions uint32) int {
n := int(maxSessions) + 1 // +1: shutdown drain may race one last dispatch
if n < 64 {
n = 64
}
return n
}
// pubOverflowSlots bounds how many callers may run overflow
// publications inline at once; further callers block on the
// semaphore until a slot frees.
const pubOverflowSlots = 8
func newOpsTracker() *opsTracker {
func newOpsTracker(queueDepth int) *opsTracker {
t := &opsTracker{
sessions: map[string]*sessionRecord{},
app: fiber.New(fiber.Config{
Immutable: true,
}),
pubq: make(chan pubJob, pubQueueDepth),
overflow: make(chan struct{}, pubOverflowSlots),
done: make(chan struct{}),
drain: make(chan struct{}),
pubq: make(chan pubJob, queueDepth),
done: make(chan struct{}),
drain: make(chan struct{}),
}
go func() {
defer close(t.done)
@@ -321,24 +350,14 @@ func (t *opsTracker) Shutdown() {
})
}
// dispatch hands a publication to the worker. It must never block
// indefinitely: when the queue is full (the worker itself stuck in
// a sink), the publication runs inline so the record is still
// delivered and the caller - possibly the native reaper - returns.
// dispatch hands a publication to the worker. It must never block
// indefinitely: when the queue is full the caller runs the
// publication itself under an overflow semaphore, which bounds how
// many overflow publications may wait at once. Overflow callers
// beyond the semaphore block - the alternative (one goroutine per
// job) grows without bound under a stalled sink, and dropping the
// record loses the publication entirely. The callers that reach
// overflow are the native reaper or a request handler; waiting
// there is bounded by the semaphore and by the worker draining,
// and is the price of never losing a record.
// dispatch hands a publication to the worker without ever
// blocking the caller. The queue is sized to the session limit
// and each session publishes exactly once, so the send below
// always has room while the worker runs; after the worker exits
// (shutdown drain) the publication runs inline, because the queue
// no longer moves. Either way the native callback - which may be
// the reaper - returns promptly and the record is never lost.
func (t *opsTracker) dispatch(job pubJob) {
// After the worker exited (shutdown drain), the queue no
// longer moves: publish inline so the record is not
// stranded behind a send nobody will receive.
select {
case <-t.done:
job.emit.publish(job.err, job.byt)
@@ -347,14 +366,12 @@ func (t *opsTracker) dispatch(job pubJob) {
}
select {
case t.pubq <- job:
return
default:
// Unreachable while the capacity contract holds; kept
// as a safety net so a mis-sized queue degrades to
// inline publication instead of blocking the reaper.
job.emit.publish(job.err, job.byt)
}
// Queue full and worker alive. Run inline under the overflow
// semaphore.
t.overflow <- struct{}{}
defer func() { <-t.overflow }()
job.emit.publish(job.err, job.byt)
}
// SetOpsServices installs the operational service instances. The
@@ -435,11 +452,18 @@ func (t *opsTracker) failOutcome(sessionID string, err error) {
t.dispatch(pubJob{emit: rec.emit, err: err})
}
// reservation couples the emitter with the generation of the
// claim that owns it: release and publication validate the
// generation, so a stale claim cannot act on a newer one.
type reservation struct {
emit *opsEmitter
gen uint64
}
// reserve marks a session record as owned by its request path: the
// teardown callback skips a reserved record because the request
// path publishes the real outcome itself. Returns the emitter when
// the record exists and was not reserved yet.
func (t *opsTracker) reserve(sessionID string) *opsEmitter {
// path publishes the real outcome itself. Returns the reservation
// when the record exists and was not reserved yet.
func (t *opsTracker) reserve(sessionID string) *reservation {
if t == nil {
return nil
}
@@ -450,7 +474,8 @@ func (t *opsTracker) reserve(sessionID string) *opsEmitter {
return nil
}
rec.reserved = true
return rec.emit
rec.claimGen++
return &reservation{emit: rec.emit, gen: rec.claimGen}
}
// releaseReservation returns a reserved record to the pool
@@ -461,13 +486,13 @@ func (t *opsTracker) reserve(sessionID string) *opsEmitter {
// reserved (terminal stashed), the session is gone: consume the
// record and publish the stashed outcome, since no second
// callback will arrive.
func (t *opsTracker) releaseReservation(sessionID string, emit *opsEmitter) {
func (t *opsTracker) releaseReservation(sessionID string, rsv *reservation) {
if t == nil {
return
}
t.mu.Lock()
rec, ok := t.sessions[sessionID]
if !ok || !rec.reserved || rec.emit != emit {
if !ok || !rec.reserved || rsv == nil || rec.claimGen != rsv.gen {
t.mu.Unlock()
return
}
@@ -483,25 +508,23 @@ func (t *opsTracker) releaseReservation(sessionID string, emit *opsEmitter) {
// publishReserved publishes through a reserved record and drops it:
// the single publication of a request-owned session outcome.
func (t *opsTracker) publishReserved(sessionID string, emit *opsEmitter, err error, bytes int64) {
func (t *opsTracker) publishReserved(sessionID string, rsv *reservation, err error, bytes int64) {
if t == nil {
return
}
t.mu.Lock()
rec, ok := t.sessions[sessionID]
// Ownership check: only the reservation holder publishes. A
// stale holder (its reservation was released or the record
// was replaced) must not delete or publish the current
// Ownership check: only the current reservation generation
// publishes. A stale claim (its reservation was released or
// superseded) must not delete or publish the current
// owner's record.
if !ok || rec.emit != emit {
if !ok || rsv == nil || rec.claimGen != rsv.gen {
t.mu.Unlock()
return
}
delete(t.sessions, sessionID)
t.mu.Unlock()
if emit != nil {
t.dispatch(pubJob{emit: emit, err: err, byt: bytes})
}
t.dispatch(pubJob{emit: rsv.emit, err: err, byt: bytes})
}
func (t *opsTracker) loadOps() OpsServices {
+42 -29
View File
@@ -38,7 +38,7 @@ import (
// exactly once. Unreserved records are published by the callback.
func TestOpsTrackerCallbackPublishesExpiry(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.register("sess-1", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", false, time.Now())
if got := len(tr.sessions); got != 1 {
@@ -60,14 +60,14 @@ func TestOpsTrackerCallbackPublishesExpiry(t *testing.T) {
}
func TestOpsTrackerReserveBlocksCallback(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.register("sess-2", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", true, time.Now())
// The READY path reserves before its completion call; the
// callback the call fires synchronously must skip the record.
emit := tr.reserve("sess-2")
if emit == nil {
rsv := tr.reserve("sess-2")
if rsv == nil {
t.Fatal("reserve returned nil for a live session")
}
tr.onTerminal(rcserver.TerminalEvent{SessionID: "sess-2"})
@@ -76,7 +76,7 @@ func TestOpsTrackerReserveBlocksCallback(t *testing.T) {
}
// The request path then publishes and drops the entry.
tr.publishReserved("sess-2", emit, nil, 4096)
tr.publishReserved("sess-2", rsv, nil, 4096)
if got := len(tr.sessions); got != 0 {
t.Fatalf("publishReserved left residue: %d", got)
}
@@ -88,7 +88,7 @@ func TestOpsTrackerReserveBlocksCallback(t *testing.T) {
}
func TestOpsTrackerReserveIsExclusive(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.register("sess-3", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", false, time.Now())
@@ -101,7 +101,7 @@ func TestOpsTrackerReserveIsExclusive(t *testing.T) {
}
func TestOpsTrackerFailOutcome(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.register("sess-4", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", false, time.Now())
@@ -116,7 +116,7 @@ func TestOpsTrackerFailOutcome(t *testing.T) {
}
func TestOpsTrackerUnregister(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.register("sess-5", auth.Account{Access: "ak"}, "us-east-1",
"bkt", "obj", false, time.Now())
tr.unregister("sess-5")
@@ -132,7 +132,7 @@ func TestOpsTrackerUnregister(t *testing.T) {
}
func TestOpsTrackerUnknownSession(t *testing.T) {
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
// Unknown sessions and the nil tracker are silent no-ops.
var nilTracker *opsTracker
nilTracker.reserve("ghost")
@@ -237,7 +237,7 @@ func (r *recordingLogger) Shutdown() error { return nil }
// publishes exactly one record with its own outcome and bytes.
func TestOpsTrackerPublishesExactlyOncePerSession(t *testing.T) {
rl := &recordingLogger{}
tr := newOpsTracker()
tr := newOpsTracker(pubQueueCapacity(0))
tr.SetOpsServices(OpsServices{Logger: rl})
// Expiry path: callback publishes a zero-byte error record.
@@ -247,69 +247,79 @@ func TestOpsTrackerPublishesExactlyOncePerSession(t *testing.T) {
// Reserved path: reserve, callback fires (skipped), the
// request path publishes success with bytes.
tr.register("s-res", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
emit := tr.reserve("s-res")
if emit == nil {
rsv := tr.reserve("s-res")
if rsv == nil {
t.Fatal("reserve failed")
}
tr.onTerminal(rcserver.TerminalEvent{SessionID: "s-res"})
tr.publishReserved("s-res", emit, nil, 128)
tr.publishReserved("s-res", rsv, nil, 128)
// Denial path while reserved: failOutcome must not steal the
// publication; the owner's success record is the only one.
tr.register("s-den", auth.Account{Access: "ak"}, "r", "b", "o", true, time.Now())
emit2 := tr.reserve("s-den")
if emit2 == nil {
rsv2 := tr.reserve("s-den")
if rsv2 == nil {
t.Fatal("reserve failed")
}
tr.failOutcome("s-den", errors.New("denied"))
tr.publishReserved("s-den", emit2, nil, 256)
tr.publishReserved("s-den", rsv2, nil, 256)
// Released reservation: the record returns to the pool and
// the reaper (or the next claimant) can still publish it.
tr.register("s-rel", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
emit3 := tr.reserve("s-rel")
if emit3 == nil {
rsv3 := tr.reserve("s-rel")
if rsv3 == nil {
t.Fatal("reserve failed")
}
tr.releaseReservation("s-rel", emit3)
tr.releaseReservation("s-rel", rsv3)
tr.onTerminal(rcserver.TerminalEvent{SessionID: "s-rel"})
// M1 regression: a terminal arriving while reserved is stashed,
// and a later claim-rollback release consumes it and publishes the
// expiry - the record is not orphaned.
tr.register("s-stash", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
emitS := tr.reserve("s-stash")
if emitS == nil {
rsvS := tr.reserve("s-stash")
if rsvS == nil {
t.Fatal("reserve failed")
}
tr.onTerminal(rcserver.TerminalEvent{SessionID: "s-stash"})
tr.releaseReservation("s-stash", emitS)
tr.releaseReservation("s-stash", rsvS)
// M2 regression: a second reservation of a live record is
// refused, so a duplicate READY cannot claim the transfer
// while another request owns the publication. The owner then
// completes normally.
tr.register("s-dbl", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
emitD := tr.reserve("s-dbl")
if emitD == nil {
rsvD := tr.reserve("s-dbl")
if rsvD == nil {
t.Fatal("first reserve failed")
}
if tr.reserve("s-dbl") != nil {
t.Fatal("double reserve succeeded")
}
tr.publishReserved("s-dbl", emitD, nil, 64)
tr.publishReserved("s-dbl", rsvD, nil, 64)
// Ownership: a stale emitter must not publish or consume the
// current record; the real owner still can, even after the
// callback fired (stashed) underneath it.
tr.register("s-own", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
emitO := tr.reserve("s-own")
if emitO == nil {
rsvO := tr.reserve("s-own")
if rsvO == nil {
t.Fatal("reserve failed")
}
tr.publishReserved("s-own", &opsEmitter{}, nil, 999)
// Stale reservation: released, re-reserved by another claim,
// then the stale token tries to publish. The generation check
// must reject the stale token while the current owner still
// publishes.
rsvO2 := tr.reserve("s-own")
tr.releaseReservation("s-own", rsvO)
rsvB := tr.reserve("s-own")
if rsvB == nil {
t.Fatal("re-reserve after release failed")
}
tr.publishReserved("s-own", rsvO2, nil, 999) // stale: no-op
tr.onTerminal(rcserver.TerminalEvent{SessionID: "s-own"})
tr.publishReserved("s-own", emitO, nil, 32)
tr.publishReserved("s-own", rsvB, nil, 32)
// Consume-or-noop denial of an unreserved session.
tr.register("s-fail", auth.Account{Access: "ak"}, "r", "b", "o", false, time.Now())
@@ -337,6 +347,9 @@ func TestOpsTrackerPublishesExactlyOncePerSession(t *testing.T) {
{false, 128}, // s-res success
{false, 256}, // s-den success (denial was skipped)
{true, 0}, // s-rel expiry after release
{true, 0}, // s-stash stashed terminal consumed by release
{false, 64}, // s-dbl owner publishes after refused double reserve
{false, 32}, // s-own current owner publishes; stale token no-op
{true, 0}, // s-fail denial
}
for i, w := range want {
+46 -20
View File
@@ -121,7 +121,7 @@ func New(svc *rcserver.RCSvc, be backend.Backend, iam auth.IAMService,
readonly, disableACL bool) *Handler {
return &Handler{svc: svc, be: be, iam: iam,
readonly: readonly, disableACL: disableACL,
ops: newOpsTracker()}
ops: newOpsTracker(pubQueueCapacity(svc.MaxSessions()))}
}
// principalID derives the session identity digest from the
@@ -139,6 +139,14 @@ func errNotAdmitted() error {
return errRouteUnavailable{}
}
// ErrNotAdmitted is the route error for requests that lost the
// race with shutdown: the RC service stopped admitting, so the
// request cannot be served. Exposed for the admission barrier in
// the route middleware, which runs before the handlers.
func ErrNotAdmitted() error {
return errNotAdmitted()
}
func invalidHeader(name, value string) error {
return fmt.Errorf("invalid %s header: %q: %w",
name, value, errRouteBadRequest{})
@@ -400,8 +408,8 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
// READY's denial (or the reaper) must not be able to consume
// the record in the window between the claim and the
// reservation. A reserved record is invisible to both.
emit := h.ops.reserve(sessionID)
if emit == nil {
rsv := h.ops.reserve(sessionID)
if rsv == nil {
// Another READY holds the publication reservation for
// this session. Publication ownership must track native
// transfer ownership: proceeding without the reservation
@@ -410,8 +418,15 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
// record on completion. Answer as a duplicate claim.
return fmt.Errorf("transfer in progress: %w", rcserver.ErrDouble)
}
// Panic safety starts at the reservation: an unwind anywhere
// below (authorization, backend I/O) must still retire the
// reservation so the record is not orphaned - the terminal
// callback can only stash under a reservation, and nobody
// else would ever release it. releaseReservation is a no-op
// once a later publish consumed the record.
defer h.ops.releaseReservation(sessionID, rsv)
publish := func(err error, bytes int64) {
h.ops.publishReserved(sessionID, emit, err, bytes)
h.ops.publishReserved(sessionID, rsv, err, bytes)
}
if err := h.authorize(ctx, acct, isRoot, bucket, key, info.Op == 1); err != nil {
// Permission revoked mid-session: publish the real
@@ -441,7 +456,7 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
// the record for this claim; release it without
// publishing so the surviving path (the other READY,
// or the eventual reaper) still owns it.
h.ops.releaseReservation(sessionID, emit)
h.ops.releaseReservation(sessionID, rsv)
return mapRcError(err)
}
@@ -453,7 +468,7 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
// released the same way: the session stays with the record
// the next READY (or the reaper) will claim.
if resp.Outcome == rcserver.ReadyBusy {
h.ops.releaseReservation(sessionID, emit)
h.ops.releaseReservation(sessionID, rsv)
return fmt.Errorf("peer busy: %w", rcserver.ErrDouble)
}
@@ -498,21 +513,26 @@ func (h *Handler) readyCore(ctx fiber.Ctx) error {
// finalizer retires exactly at that point; a failure
// *before* the borrow still falls back to the
// finalizer path below.
put, gd, err := h.commitPut(ctx, sessionID, bucket, key, sizeOf(resp))
if gd {
put, viewDone, committed, err := h.commitPut(ctx, sessionID, bucket, key, sizeOf(resp))
if viewDone {
finalized = true
}
if put != nil {
// The backend committed the object. Record the
// fact before anything else can fail: the audit
// record and creation event must reflect the
// commit even when the native finalizer below
// errors out.
rsv.emit.markCommitted(put.ETag, put.VersionID)
}
if err != nil {
doPublish(mapRcError(err), 0)
doPublish(mapRcError(err), committed)
return err
}
// The FINAL wire reply carries the stored object's
// metadata, which the backend assigned at commit time.
resp.Etag = put.ETag
resp.VersionID = put.VersionID
// The success publication's object-created event carries
// the same commit metadata.
emit.setCommitMeta(put.ETag, put.VersionID)
} else if err := h.svc.FinishFinal(sessionID); err != nil {
doPublish(mapRcError(err), 0)
return mapRcError(err)
@@ -553,18 +573,18 @@ func sizeOf(resp *rcserver.ReadyResponse) uint64 {
// panic-safe defer releases the view if the handler unwinds before
// FinishPut runs.
func (h *Handler) commitPut(ctx fiber.Ctx, sessionID, bucket, key string,
size uint64) (*s3response.PutObjectOutput, bool, error) {
size uint64) (put *s3response.PutObjectOutput, viewDone bool, committed int64, err error) {
view, err := h.svc.GetPutData(sessionID)
if err != nil {
return nil, false, mapRcError(err)
return nil, false, 0, mapRcError(err)
}
// Panic-safe ownership: if anything below unwinds, the view is
// still returned exactly once (the ABI consumes the handle a
// single time; a redundant FinishPut after a commit is a
// no-op STALE).
committed := false
putDone := false
defer func() {
if !committed {
if !putDone {
_ = h.svc.FinishPut(*view, false, "", "")
}
}()
@@ -579,13 +599,19 @@ func (h *Handler) commitPut(ctx fiber.Ctx, sessionID, bucket, key string,
Body: bytes.NewReader(view.Buf),
})
if err != nil {
return nil, true, err
// The object was not created; the audit records zero
// transferred bytes. The view was consumed above.
return nil, true, 0, err
}
// The object exists from here on. The value below is the
// committed byte count, reported even when the native
// finalizer fails, so the audit record reflects the commit.
committed = contentLength
if err := h.svc.FinishPut(*view, true, res.ETag, res.VersionID); err != nil {
return nil, true, mapRcError(err)
return &res, true, committed, mapRcError(err)
}
committed = true
return &res, true, nil
putDone = true
return &res, true, committed, nil
}
// Cancel handles CANCEL: authenticated owner tears the session down.
+7
View File
@@ -19,6 +19,8 @@
package rcroutes
import (
"errors"
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/auth"
@@ -65,5 +67,10 @@ func (h *Handler) SetOpsServices(ops OpsServices) {}
// PublishAuthFailure is a stub mirror of the linux handler.
func (h *Handler) PublishAuthFailure(ctx fiber.Ctx, err error) {}
// ErrNotAdmitted is a stub mirror of the linux helper.
func ErrNotAdmitted() error {
return errors.New("rc routes unavailable")
}
// Shutdown is a stub mirror of the linux handler.
func (h *Handler) Shutdown() {}
+16 -7
View File
@@ -174,12 +174,13 @@ type SessionInfo struct {
// unblock), waits for every entered call to leave, then tears the
// server down; it is idempotent and safe from any goroutine.
type RCSvc struct {
srv *C.rc_server
closing atomic.Bool
ops atomic.Int64
once sync.Once
ctx context.Context
cancel context.CancelFunc
srv *C.rc_server
closing atomic.Bool
ops atomic.Int64
once sync.Once
ctx context.Context
cancel context.CancelFunc
maxSessions uint32
}
// Context returns the service-lifetime context. Handlers bind
@@ -188,6 +189,14 @@ func (s *RCSvc) Context() context.Context {
return s.ctx
}
// MaxSessions reports the configured global session limit. The
// operational publication pipeline sizes its queue against it:
// each session publishes exactly one terminal record, so a queue
// this deep can never fill.
func (s *RCSvc) MaxSessions() uint32 {
return s.maxSessions
}
// rcLogSink receives every diagnostic line the C server emits.
// It is stateless and process-global on purpose: the sink must be
// valid from init through destroy, independent of any single
@@ -263,7 +272,7 @@ func Init(opts DeviceOpts) (*RCSvc, error) {
}
installLogSink(srv, opts.Debug)
ctx, cancel := context.WithCancel(context.Background())
return &RCSvc{srv: srv, ctx: ctx, cancel: cancel}, nil
return &RCSvc{srv: srv, ctx: ctx, cancel: cancel, maxSessions: opts.MaxSessions}, nil
}
// TryEnter admits a request into the service. It returns false once