seaweedfs/weed/storage/blockvol/nvme/controller.go

package nvme

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

// errDisconnect is a sentinel returned by handleDisconnect to signal
// the rxLoop to exit gracefully after enqueuing the disconnect response.
var errDisconnect = errors.New("disconnect")

// controllerState tracks the lifecycle of an NVMe controller session.
type controllerState int

const (
	stateConnected  controllerState = iota // TCP connected, no IC yet
	stateICComplete                        // IC exchange done
	stateAdminReady                        // Admin queue connected
	stateCtrlReady                         // CC.EN=1, CSTS.RDY=1
	stateIOActive                          // IO queues active
	stateClosed                            // Shut down
)

// Request represents an in-flight NVMe command being processed.
type Request struct {
	capsule CapsuleCommand
	payload []byte // inline data from host (Write commands)
	resp    CapsuleResponse
	c2hData []byte  // data to send to host (Read commands)
	status  StatusWord
}

// response represents a pending response to be written by the txLoop.
// Either a standard CapsuleResp (with optional C2H data), or an R2T PDU.
type response struct {
	// Standard response fields
	resp    CapsuleResponse
	c2hData []byte // non-nil for read responses

	// R2T variant (when r2t is non-nil, resp/c2hData are ignored)
	r2t     *R2THeader
	r2tDone chan struct{} // closed after R2T is flushed to wire
}

// Controller handles one NVMe/TCP connection (one queue per connection).
//
// After the IC handshake, the connection is split into two goroutines:
//   - rxLoop: reads PDUs from the wire, dispatches commands
//   - txLoop: drains the response channel, writes responses to the wire
//
// IO commands are dispatched to goroutines for concurrent processing.
// Admin commands run synchronously in the rxLoop (infrequent, state-changing).
// All responses flow through respCh — only txLoop writes to c.out.
type Controller struct {
	mu sync.Mutex

	// Session identity
	conn   net.Conn
	in     *Reader
	out    *Writer
	state  controllerState
	closed atomic.Bool

	// Queue state (one queue per TCP connection)
	queueID    uint16
	queueSize  uint16
	rxSQHD     uint16        // Submission Queue Head (updated by rxLoop only)
	sqhdVal    atomic.Uint32 // Latest SQHD for txLoop (lower 16 bits)
	flowCtlOff bool          // CATTR bit2: SQ flow control disabled

	// Controller identity
	cntlID uint16
	subNQN string

	// Controller registers
	regCAP  uint64 // Controller Capabilities
	regCC   uint32 // Controller Configuration (set by host via PropertySet)
	regCSTS uint32 // Controller Status (RDY bit)
	regVS   uint32 // Version

	// KeepAlive
	katoMs    uint32
	katoTimer *time.Timer
	katoMu    sync.Mutex

	// RX/TX split
	respCh chan *response // responses from handlers → txLoop
	done   chan struct{}  // closed when connection is shutting down
	cmdWg  sync.WaitGroup // tracks in-flight IO command goroutines

	// Backend
	subsystem *Subsystem
	server    *Server

	// Features
	maxIOQueues   uint16
	grantedQueues uint16
	isAdmin       bool   // true if this controller owns admin queue (QID=0)
	maxDataLen    uint32 // C2H/H2C data chunk size (from Config)

	// Command interleaving: capsules received during R2T H2CData collection.
	// Drained by rxLoop before reading the next PDU from the wire.
	pendingCapsules []*Request

	// Lifecycle
	wg        sync.WaitGroup
	closeOnce sync.Once
}

// newController creates a controller for the given connection.
func newController(conn net.Conn, server *Server) *Controller {
	maxData := server.cfg.MaxH2CDataLength
	if maxData == 0 {
		maxData = maxH2CDataLen // fallback to 32KB default
	}
	c := &Controller{
		conn:   conn,
		in:     NewReaderSize(conn, int(maxData)+maxHeaderSize),
		out:    NewWriterSize(conn, int(maxData)+maxHeaderSize),
		state:  stateConnected,
		server: server,
		regVS:  nvmeVersion14,
		// CAP register: MQES=63 (bits 15:0), CQR=1 (bit 16), TO=30 (bits 31:24, *500ms=15s), CSS bit37=1 (NVM command set)
		regCAP:      uint64(63) | (1 << 16) | (uint64(30) << 24) | (1 << 37),
		maxIOQueues: server.cfg.MaxIOQueues,
		maxDataLen:  maxData,
	}
	return c
}

// Serve is the main event loop for this controller connection.
//
// Phase 1: IC handshake (synchronous, direct c.out access).
// Phase 2: Start txLoop goroutine, enter rxLoop.
func (c *Controller) Serve() error {
	defer c.shutdown()

	// Phase 1: IC handshake with timeout (synchronous).
	if err := c.conn.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {
		return err
	}
	hdr, err := c.in.Dequeue()
	if err != nil {
		return fmt.Errorf("IC handshake: %w", err)
	}
	if hdr.Type != pduICReq {
		return fmt.Errorf("expected ICReq (0x%x), got 0x%x", pduICReq, hdr.Type)
	}
	if err := c.handleIC(); err != nil {
		return fmt.Errorf("IC handshake: %w", err)
	}
	if err := c.conn.SetReadDeadline(time.Time{}); err != nil {
		return err
	}

	// Phase 2: Start TX loop, enter RX loop.
	c.respCh = make(chan *response, 128)
	c.done = make(chan struct{})

	txErrCh := make(chan error, 1)
	go func() {
		txErrCh <- c.txLoop()
	}()

	rxErr := c.rxLoop()

	// Wait for in-flight IO command goroutines to finish and enqueue responses.
	c.cmdWg.Wait()

	// Signal txLoop to drain remaining responses and exit.
	close(c.done)
	txErr := <-txErrCh

	// errDisconnect is a graceful exit, not an error.
	if errors.Is(rxErr, errDisconnect) {
		rxErr = nil
	}
	if rxErr != nil {
		return rxErr
	}
	return txErr
}

// handleIC processes the IC handshake.
func (c *Controller) handleIC() error {
	var req ICRequest
	if err := c.in.Receive(&req); err != nil {
		return err
	}

	resp := ICResponse{
		PDUFormatVersion: 0,
		MaxH2CDataLength: c.maxDataLen,
	}
	if err := c.out.SendHeaderOnly(pduICResp, &resp, icBodySize); err != nil {
		return err
	}

	c.state = stateICComplete
	return nil
}

// rxLoop reads PDUs from the wire and dispatches commands.
// Admin commands (QID=0) run synchronously. IO commands dispatch to goroutines.
func (c *Controller) rxLoop() error {
	for {
		if c.closed.Load() {
			return nil
		}

		// Drain capsules that arrived during a prior R2T data collection.
		for len(c.pendingCapsules) > 0 {
			req := c.pendingCapsules[0]
			c.pendingCapsules = c.pendingCapsules[1:]
			c.advanceSQHD()
			if err := c.dispatchFromRx(req); err != nil {
				return err
			}
		}

		hdr, err := c.in.Dequeue()
		if err != nil {
			if err == io.EOF || c.closed.Load() {
				return nil
			}
			return fmt.Errorf("read header: %w", err)
		}

		switch hdr.Type {
		case pduCapsuleCmd:
			if c.state < stateICComplete {
				return fmt.Errorf("capsule command before IC handshake")
			}
			req, err := c.parseCapsule()
			if err != nil {
				return fmt.Errorf("capsule: %w", err)
			}
			c.advanceSQHD()
			if err := c.dispatchFromRx(req); err != nil {
				return err
			}

		case pduH2CData:
			return fmt.Errorf("unexpected H2CData PDU outside R2T flow")

		case pduH2CTermReq:
			return nil

		default:
			return fmt.Errorf("unexpected PDU type: 0x%x", hdr.Type)
		}
	}
}

// txLoop drains the response channel and writes responses to the wire.
// Only txLoop touches c.out after the IC handshake.
func (c *Controller) txLoop() error {
	var firstErr error
	for {
		select {
		case resp := <-c.respCh:
			if firstErr != nil {
				completeWaiters(resp)
				continue // discard — connection already failed
			}
			if err := c.writeResponse(resp); err != nil {
				firstErr = err
				c.conn.Close() // force rxLoop EOF
			}
		case <-c.done:
			// Drain remaining responses.
			for {
				select {
				case resp := <-c.respCh:
					if firstErr == nil {
						if err := c.writeResponse(resp); err != nil {
							firstErr = err
						}
					} else {
						completeWaiters(resp)
					}
				default:
					return firstErr
				}
			}
		}
	}
}

// parseCapsule reads a CapsuleCmd PDU (specific header + optional inline data).
func (c *Controller) parseCapsule() (*Request, error) {
	var capsule CapsuleCommand
	if err := c.in.Receive(&capsule); err != nil {
		return nil, err
	}

	var payload []byte
	if dataLen := c.in.Length(); dataLen > 0 {
		payload = getBuffer(int(dataLen))
		if err := c.in.ReceiveData(payload); err != nil {
			putBuffer(payload)
			return nil, err
		}
	}

	req := &Request{
		capsule: capsule,
		payload: payload,
	}
	req.resp.CID = capsule.CID
	req.resp.QueueID = c.queueID
	req.resp.Status = uint16(StatusSuccess)
	return req, nil
}

// advanceSQHD increments the submission queue head pointer (rxLoop only).
func (c *Controller) advanceSQHD() {
	sqhd := c.rxSQHD + 1
	if sqhd >= c.queueSize && c.queueSize > 0 {
		sqhd = 0
	}
	c.rxSQHD = sqhd
	c.sqhdVal.Store(uint32(sqhd))
}

// dispatchFromRx routes a parsed capsule to the appropriate handler.
// Admin queue: synchronous in rxLoop. IO queue: concurrent goroutine.
func (c *Controller) dispatchFromRx(req *Request) error {
	if c.queueID == 0 {
		return c.dispatchAdmin(req)
	}

	// For Write commands without inline data, collect R2T data in rxLoop
	// before dispatching to the goroutine.
	if req.capsule.OpCode == ioWrite && len(req.payload) == 0 {
		if err := c.collectR2TData(req); err != nil {
			return err
		}
	}

	c.cmdWg.Add(1)
	go func() {
		defer c.cmdWg.Done()
		c.dispatchIO(req)
	}()
	return nil
}

// collectR2TData handles the R2T/H2C flow in the rxLoop for Write commands
// that don't carry inline data. Sends R2T through the txLoop, then reads
// H2C Data PDUs inline. Sets req.payload with the collected data.
func (c *Controller) collectR2TData(req *Request) error {
	sub := c.subsystem
	if sub == nil {
		// Let handleWrite deal with the nil subsystem.
		return nil
	}

	nlb := req.capsule.LbaLength()
	expectedBytes := uint32(nlb) * sub.Dev.BlockSize()

	// Send R2T through txLoop and wait for it to be flushed.
	r2tDone := make(chan struct{})
	select {
	case c.respCh <- &response{
		r2t: &R2THeader{
			CCCID: req.capsule.CID,
			TAG:   0,
			DATAO: 0,
			DATAL: expectedBytes,
		},
		r2tDone: r2tDone,
	}:
	case <-c.done:
		return nil
	}

	select {
	case <-r2tDone:
	case <-c.done:
		return nil
	}

	// Read H2C Data PDUs from the wire.
	data, err := c.recvH2CData(expectedBytes)
	if err != nil {
		return err
	}
	req.payload = data
	return nil
}

// dispatchAdmin handles admin queue commands synchronously in the rxLoop.
func (c *Controller) dispatchAdmin(req *Request) error {
	defer func() {
		if req.payload != nil {
			putBuffer(req.payload)
			req.payload = nil
		}
	}()
	capsule := &req.capsule

	if capsule.OpCode == adminFabric {
		return c.handleFabricCommand(req)
	}

	switch capsule.OpCode {
	case adminIdentify:
		c.handleIdentify(req)
	case adminSetFeatures:
		c.handleSetFeatures(req)
	case adminGetFeatures:
		c.handleGetFeatures(req)
	case adminGetLogPage:
		c.handleGetLogPage(req)
	case adminKeepAlive:
		c.handleKeepAlive(req)
	case adminAsyncEvent:
		c.enqueueResponse(&response{resp: req.resp})
	default:
		req.resp.Status = uint16(StatusInvalidOpcode)
		c.enqueueResponse(&response{resp: req.resp})
	}
	return nil
}

// dispatchIO handles IO queue commands (runs in a goroutine).
func (c *Controller) dispatchIO(req *Request) {
	defer func() {
		if req.payload != nil {
			putBuffer(req.payload)
			req.payload = nil
		}
	}()
	capsule := &req.capsule

	switch capsule.OpCode {
	case ioRead:
		c.handleRead(req)
	case ioWrite:
		c.handleWrite(req)
	case ioFlush:
		c.handleFlush(req)
	case ioWriteZeros:
		c.handleWriteZeros(req)
	default:
		req.resp.Status = uint16(StatusInvalidOpcode)
		c.enqueueResponse(&response{resp: req.resp})
	}
}

// completeWaiters closes any synchronization channels on a discarded response.
// Must be called when txLoop drops a response after a write error, so that
// senders blocked on r2tDone (e.g. collectR2TData) are unblocked.
func completeWaiters(resp *response) {
	if resp.r2tDone != nil {
		close(resp.r2tDone)
	}
}

// enqueueResponse sends a response to the txLoop for writing.
// Safe to call from any goroutine. If the connection is shutting down,
// the response is silently discarded.
func (c *Controller) enqueueResponse(resp *response) {
	select {
	case c.respCh <- resp:
	case <-c.done:
	}
}

// writeResponse writes a single response to the wire (txLoop only).
func (c *Controller) writeResponse(resp *response) error {
	// R2T variant: write R2T header and signal caller.
	if resp.r2t != nil {
		err := c.out.SendHeaderOnly(pduR2T, resp.r2t, r2tHdrSize)
		if resp.r2tDone != nil {
			close(resp.r2tDone)
		}
		return err
	}

	// Standard response: set SQHD from latest value.
	if c.flowCtlOff {
		resp.resp.SQHD = 0xFFFF
	} else {
		resp.resp.SQHD = uint16(c.sqhdVal.Load())
	}
	c.resetKATO()

	if len(resp.c2hData) > 0 {
		return c.writeC2HAndResp(resp)
	}
	return c.out.SendHeaderOnly(pduCapsuleResp, &resp.resp, capsuleRespSize)
}

// writeC2HAndResp writes C2H data chunks followed by CapsuleResp, batched.
func (c *Controller) writeC2HAndResp(resp *response) error {
	data := resp.c2hData
	offset := uint32(0)
	chunkSize := c.maxDataLen

	for offset < uint32(len(data)) {
		end := offset + chunkSize
		if end > uint32(len(data)) {
			end = uint32(len(data))
		}
		chunk := data[offset:end]

		hdr := C2HDataHeader{
			CCCID: resp.resp.CID,
			DATAO: offset,
			DATAL: uint32(len(chunk)),
		}

		flags := uint8(0)
		if end >= uint32(len(data)) {
			flags = c2hFlagLast
		}

		if err := c.out.writeHeaderAndData(pduC2HData, flags, &hdr, c2hDataHdrSize, chunk); err != nil {
			return err
		}
		offset = end
	}

	if err := c.out.writeHeaderAndData(pduCapsuleResp, 0, &resp.resp, capsuleRespSize, nil); err != nil {
		return err
	}
	return c.out.FlushBuf()
}

// ---------- R2T / H2C Data ----------

// recvH2CData reads H2CData PDU(s) from the wire and returns the accumulated data.
// Reads exactly `totalBytes` of data, potentially across multiple H2C PDUs.
//
// At QD>1 the host may interleave CapsuleCmd PDUs on the same connection
// before the H2CData for a prior R2T arrives. Such capsules are fully read
// and buffered in c.pendingCapsules for dispatch after the current command
// completes (NVMe/TCP spec §3.5 — command pipelining).
func (c *Controller) recvH2CData(totalBytes uint32) ([]byte, error) {
	buf := getBuffer(int(totalBytes))
	received := uint32(0)

	for received < totalBytes {
		hdr, err := c.in.Dequeue()
		if err != nil {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: read header: %w", err)
		}

		// Interleaved CapsuleCmd: buffer it for later dispatch.
		if hdr.Type == pduCapsuleCmd {
			if err := c.bufferInterleaved(); err != nil {
				putBuffer(buf)
				return nil, fmt.Errorf("recvH2CData: buffer interleaved capsule: %w", err)
			}
			continue
		}

		if hdr.Type != pduH2CData {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: expected H2CData (0x6), got 0x%x", hdr.Type)
		}

		var h2c H2CDataHeader
		if err := c.in.Receive(&h2c); err != nil {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: receive header: %w", err)
		}

		dataLen := c.in.Length()
		if dataLen == 0 {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: H2CData PDU has no payload")
		}
		if h2c.DATAO+dataLen > totalBytes {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: data exceeds expected size (%d+%d > %d)",
				h2c.DATAO, dataLen, totalBytes)
		}

		if err := c.in.ReceiveData(buf[h2c.DATAO : h2c.DATAO+dataLen]); err != nil {
			putBuffer(buf)
			return nil, fmt.Errorf("recvH2CData: receive data: %w", err)
		}
		received += dataLen
	}

	return buf, nil
}

// bufferInterleaved reads a complete CapsuleCmd (header + optional inline
// data) that arrived during R2T data collection and appends it to
// c.pendingCapsules. Called from recvH2CData when hdr.Type == pduCapsuleCmd.
func (c *Controller) bufferInterleaved() error {
	var capsule CapsuleCommand
	if err := c.in.Receive(&capsule); err != nil {
		return err
	}

	var payload []byte
	if dataLen := c.in.Length(); dataLen > 0 {
		payload = getBuffer(int(dataLen))
		if err := c.in.ReceiveData(payload); err != nil {
			putBuffer(payload)
			return err
		}
	}

	req := &Request{
		capsule: capsule,
		payload: payload,
	}
	req.resp.CID = capsule.CID
	req.resp.QueueID = c.queueID
	req.resp.Status = uint16(StatusSuccess)

	c.pendingCapsules = append(c.pendingCapsules, req)
	return nil
}

// ---------- KATO management ----------

func (c *Controller) startKATO() {
	c.katoMu.Lock()
	defer c.katoMu.Unlock()
	if c.katoMs == 0 {
		return
	}
	d := time.Duration(c.katoMs) * time.Millisecond
	// Add 50% margin per spec recommendation
	d = d + d/2
	c.katoTimer = time.AfterFunc(d, func() {
		log.Printf("nvme: KATO expired for cntlid=%d, closing connection", c.cntlID)
		c.conn.Close()
	})
}

func (c *Controller) resetKATO() {
	c.katoMu.Lock()
	defer c.katoMu.Unlock()
	if c.katoTimer != nil {
		c.katoTimer.Reset(time.Duration(c.katoMs)*time.Millisecond + time.Duration(c.katoMs)*time.Millisecond/2)
	}
}

func (c *Controller) stopKATO() {
	c.katoMu.Lock()
	defer c.katoMu.Unlock()
	if c.katoTimer != nil {
		c.katoTimer.Stop()
		c.katoTimer = nil
	}
}

// ---------- Lifecycle ----------

func (c *Controller) shutdown() {
	c.closeOnce.Do(func() {
		c.closed.Store(true)
		c.stopKATO()
		c.state = stateClosed
		c.conn.Close()

		// Release pooled buffers from interleaved capsules that were never dispatched.
		for _, req := range c.pendingCapsules {
			if req.payload != nil {
				putBuffer(req.payload)
				req.payload = nil
			}
		}
		c.pendingCapsules = nil

		if c.server != nil {
			if c.isAdmin && c.cntlID != 0 {
				c.server.unregisterAdmin(c.cntlID)
			}
			c.server.removeSession(c)
		}
	})
}