seaweedfs/weed/storage/blockvol/batchio/iouring_giouring_linux.go

//go:build linux && iouring_giouring

package batchio

import (
	"fmt"
	"os"
	"unsafe"

	"github.com/pawelgaczynski/giouring"
)

func init() { IOUringImpl = "giouring" }

// giouringBatchIO implements BatchIO using giouring (direct liburing port).
// No goroutines or channels — direct SQE/CQE ring manipulation.
// Requires kernel 6.0+.
type giouringBatchIO struct {
	ring     *giouring.Ring
	ringSize int
}

// NewIOUring creates a BatchIO backed by giouring with the given ring size.
// Returns ErrIOUringUnavailable if io_uring cannot be initialized.
func NewIOUring(ringSize uint) (BatchIO, error) {
	ring, err := giouring.CreateRing(uint32(ringSize))
	if err != nil {
		return nil, fmt.Errorf("%w: %v", ErrIOUringUnavailable, err)
	}
	return &giouringBatchIO{ring: ring, ringSize: int(ringSize)}, nil
}

func (g *giouringBatchIO) PreadBatch(fd *os.File, ops []Op) error {
	if len(ops) == 0 {
		return nil
	}
	for start := 0; start < len(ops); start += g.ringSize {
		end := start + g.ringSize
		if end > len(ops) {
			end = len(ops)
		}
		if err := g.preadChunk(fd, ops[start:end]); err != nil {
			return err
		}
	}
	return nil
}

func (g *giouringBatchIO) preadChunk(fd *os.File, ops []Op) error {
	fdInt := int(fd.Fd())
	for i := range ops {
		sqe := g.ring.GetSQE()
		if sqe == nil {
			return fmt.Errorf("iouring PreadBatch: SQ full at op %d", i)
		}
		sqe.PrepareRead(fdInt, uintptr(unsafe.Pointer(&ops[i].Buf[0])), uint32(len(ops[i].Buf)), uint64(ops[i].Offset))
		sqe.SetData64(uint64(i))
	}

	_, err := g.ring.SubmitAndWait(uint32(len(ops)))
	if err != nil {
		return fmt.Errorf("iouring PreadBatch submit: %w", err)
	}

	for i := range ops {
		cqe, err := g.ring.WaitCQE()
		if err != nil {
			return fmt.Errorf("iouring PreadBatch wait op[%d]: %w", i, err)
		}
		if cqe.Res < 0 {
			g.ring.CQESeen(cqe)
			return fmt.Errorf("iouring PreadBatch op[%d]: %w", i, errFromRes(cqe.Res))
		}
		if int(cqe.Res) < len(ops[cqe.UserData].Buf) {
			g.ring.CQESeen(cqe)
			return fmt.Errorf("iouring PreadBatch op[%d]: short read %d/%d", cqe.UserData, cqe.Res, len(ops[cqe.UserData].Buf))
		}
		g.ring.CQESeen(cqe)
	}
	return nil
}

func (g *giouringBatchIO) PwriteBatch(fd *os.File, ops []Op) error {
	if len(ops) == 0 {
		return nil
	}
	for start := 0; start < len(ops); start += g.ringSize {
		end := start + g.ringSize
		if end > len(ops) {
			end = len(ops)
		}
		if err := g.pwriteChunk(fd, ops[start:end]); err != nil {
			return err
		}
	}
	return nil
}

func (g *giouringBatchIO) pwriteChunk(fd *os.File, ops []Op) error {
	fdInt := int(fd.Fd())
	for i := range ops {
		sqe := g.ring.GetSQE()
		if sqe == nil {
			return fmt.Errorf("iouring PwriteBatch: SQ full at op %d", i)
		}
		sqe.PrepareWrite(fdInt, uintptr(unsafe.Pointer(&ops[i].Buf[0])), uint32(len(ops[i].Buf)), uint64(ops[i].Offset))
		sqe.SetData64(uint64(i))
	}

	_, err := g.ring.SubmitAndWait(uint32(len(ops)))
	if err != nil {
		return fmt.Errorf("iouring PwriteBatch submit: %w", err)
	}

	for i := range ops {
		cqe, err := g.ring.WaitCQE()
		if err != nil {
			return fmt.Errorf("iouring PwriteBatch wait op[%d]: %w", i, err)
		}
		if cqe.Res < 0 {
			g.ring.CQESeen(cqe)
			return fmt.Errorf("iouring PwriteBatch op[%d]: %w", cqe.UserData, errFromRes(cqe.Res))
		}
		if int(cqe.Res) < len(ops[cqe.UserData].Buf) {
			g.ring.CQESeen(cqe)
			return fmt.Errorf("iouring PwriteBatch op[%d]: short write %d/%d", cqe.UserData, cqe.Res, len(ops[cqe.UserData].Buf))
		}
		g.ring.CQESeen(cqe)
	}
	return nil
}

// Fsync issues fdatasync via io_uring.
func (g *giouringBatchIO) Fsync(fd *os.File) error {
	sqe := g.ring.GetSQE()
	if sqe == nil {
		return fmt.Errorf("iouring Fsync: SQ full")
	}
	sqe.PrepareFsync(int(fd.Fd()), giouring.FsyncDatasync)

	_, err := g.ring.SubmitAndWait(1)
	if err != nil {
		return fmt.Errorf("iouring Fsync submit: %w", err)
	}

	cqe, err := g.ring.WaitCQE()
	if err != nil {
		return fmt.Errorf("iouring Fsync wait: %w", err)
	}
	defer g.ring.CQESeen(cqe)
	if cqe.Res < 0 {
		return fmt.Errorf("iouring Fsync: %w", errFromRes(cqe.Res))
	}
	return nil
}

// LinkedWriteFsync submits pwrite + fdatasync as a linked SQE chain.
func (g *giouringBatchIO) LinkedWriteFsync(fd *os.File, buf []byte, offset int64) error {
	fdInt := int(fd.Fd())

	// SQE 1: pwrite with IO_LINK flag
	sqe1 := g.ring.GetSQE()
	if sqe1 == nil {
		return fmt.Errorf("iouring LinkedWriteFsync: SQ full (write)")
	}
	sqe1.PrepareWrite(fdInt, uintptr(unsafe.Pointer(&buf[0])), uint32(len(buf)), uint64(offset))
	sqe1.SetFlags(uint32(giouring.SqeIOLink))

	// SQE 2: fdatasync (no link flag — last in chain)
	sqe2 := g.ring.GetSQE()
	if sqe2 == nil {
		return fmt.Errorf("iouring LinkedWriteFsync: SQ full (fsync)")
	}
	sqe2.PrepareFsync(fdInt, giouring.FsyncDatasync)

	_, err := g.ring.SubmitAndWait(2)
	if err != nil {
		// Fallback to sequential.
		if _, werr := fd.WriteAt(buf, offset); werr != nil {
			return werr
		}
		return fdatasync(fd)
	}

	// Collect both CQEs.
	for i := 0; i < 2; i++ {
		cqe, err := g.ring.WaitCQE()
		if err != nil {
			return fmt.Errorf("iouring LinkedWriteFsync wait op[%d]: %w", i, err)
		}
		if cqe.Res < 0 {
			g.ring.CQESeen(cqe)
			return fmt.Errorf("iouring LinkedWriteFsync op[%d]: %w", i, errFromRes(cqe.Res))
		}
		g.ring.CQESeen(cqe)
	}
	return nil
}

func (g *giouringBatchIO) Close() error {
	if g.ring != nil {
		g.ring.QueueExit()
	}
	return nil
}

// errFromRes converts a negative io_uring result code to a Go error.
func errFromRes(res int32) error {
	return fmt.Errorf("errno %d", -res)
}