mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-21 06:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Phase 3 — performance tuning, iSCSI session refactor, store integration

Browse Source 
Phase 3 delivers five checkpoints:

CP1 Engine Tuning: BlockVolConfig tunables, 256-shard DirtyMap, adaptive
group commit (low-watermark immediate flush), WAL pressure handling with
backpressure and ErrWALFull timeout.

CP2 iSCSI Session Refactor: RX/TX goroutine split with respCh (cap 64),
txLoop for serialized response writes, StatSN assignment modes. Login
phase stays single-goroutine; full-duplex after login.

CP3 Store Integration: BlockVolAdapter (iscsi.BlockDevice interface),
BlockVolumeStore management, BlockService in volume_server_block.go,
CLI flags (--block.listen/dir/iqn.prefix), sw-block-attach.sh helper.

CP5 Concurrency Hardening: WAL reuse guard (LSN validation in ReadLBA),
opsOutstanding counter with beginOp/endOp + Close drain, appendWithRetry
shared by WriteLBA and TrimLBA, flusher LSN guard in FlushOnce.

Bug fixes (P3-BUG-2–11): unbounded pending queue cap, Data-Out timeout,
flusher error logging, GroupCommitter panic recovery, Close vs concurrent
ops guard, target shutdown race, WAL-full retry vs Close, WRITE SAME(16)
for XFS, MODE SENSE(10) + VPD 0xB0/0xB2 for Linux kernel compatibility.

797 tests passing (517 engine + 280 iSCSI), go vet clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ping Qiu
2026-03-01 10:43:34 -08:00
co-authored by Claude Opus 4.6
parent 9b7be60b0c
commit 80801b0fac
37 changed files with 6569 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/command/volume.go
+12
View File
@@ -74,6 +74,10 @@ type VolumeServerOptions struct {
ldbTimeout *int64
debug *bool
debugPort *int
// Block volume (iSCSI) options
blockListen *string
blockDir *string
blockIQNPrefix *string
}
func init() {
@@ -114,6 +118,9 @@ func init() {
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging")
v.blockListen = cmdVolume.Flag.String("block.listen", "0.0.0.0:3260", "iSCSI target listen address for block volumes")
v.blockDir = cmdVolume.Flag.String("block.dir", "", "directory containing .blk block volume files. Empty disables iSCSI block service.")
v.blockIQNPrefix = cmdVolume.Flag.String("block.iqn.prefix", "iqn.2024-01.com.seaweedfs:vol.", "IQN prefix for block volume iSCSI targets")
}
var cmdVolume = &Command{
@@ -301,6 +308,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// starting the cluster http server
clusterHttpServer := v.startClusterHttpService(volumeMux)
// Start block volume iSCSI service (disabled if block.dir is empty).
blockService := weed_server.StartBlockService(*v.blockListen, *v.blockDir, *v.blockIQNPrefix)
grace.OnReload(volumeServer.LoadNewVolumes)
grace.OnReload(volumeServer.Reload)
@@ -315,6 +325,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
}
blockService.Shutdown()
shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
stopChan <- true
})
@@ -324,6 +335,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
select {
case <-stopChan:
case <-ctx.Done():
blockService.Shutdown()
shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
}
} else {
weed/server/volume_server_block.go
+114
View File
@@ -0,0 +1,114 @@
package weed_server
import (
"log"
"os"
"path/filepath"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)
// BlockService manages block volumes and the iSCSI target server.
type BlockService struct {
blockStore *storage.BlockVolumeStore
targetServer *iscsi.TargetServer
iqnPrefix string
}
// StartBlockService scans blockDir for .blk files, opens them as block volumes,
// registers them with an iSCSI target server, and starts listening.
// Returns nil if blockDir is empty (feature disabled).
func StartBlockService(listenAddr, blockDir, iqnPrefix string) *BlockService {
if blockDir == "" {
return nil
}
if iqnPrefix == "" {
iqnPrefix = "iqn.2024-01.com.seaweedfs:vol."
}
bs := &BlockService{
blockStore: storage.NewBlockVolumeStore(),
iqnPrefix: iqnPrefix,
}
logger := log.New(os.Stderr, "iscsi: ", log.LstdFlags)
config := iscsi.DefaultTargetConfig()
config.TargetName = iqnPrefix + "default"
bs.targetServer = iscsi.NewTargetServer(listenAddr, config, logger)
// Scan blockDir for .blk files.
entries, err := os.ReadDir(blockDir)
if err != nil {
glog.Warningf("block service: cannot read dir %s: %v", blockDir, err)
return bs
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".blk") {
continue
}
path := filepath.Join(blockDir, entry.Name())
vol, err := bs.blockStore.AddBlockVolume(path)
if err != nil {
// Auto-initialize raw files (e.g. created via truncate).
info, serr := entry.Info()
if serr == nil && info.Size() > 0 {
glog.V(0).Infof("block service: auto-creating blockvol %s (%d bytes)", path, info.Size())
os.Remove(path) // remove raw file so CreateBlockVol can use O_EXCL
created, cerr := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: uint64(info.Size()),
})
if cerr != nil {
glog.Warningf("block service: auto-create %s: %v", path, cerr)
continue
}
created.Close()
vol, err = bs.blockStore.AddBlockVolume(path)
if err != nil {
glog.Warningf("block service: skip %s after auto-create: %v", path, err)
continue
}
} else {
glog.Warningf("block service: skip %s: %v", path, err)
continue
}
}
// Derive IQN from filename: vol1.blk -> iqn.2024-01.com.seaweedfs:vol.vol1
name := strings.TrimSuffix(entry.Name(), ".blk")
iqn := iqnPrefix + name
adapter := blockvol.NewBlockVolAdapter(vol)
bs.targetServer.AddVolume(iqn, adapter)
glog.V(0).Infof("block service: registered %s as %s", path, iqn)
}
// Start iSCSI target in background.
go func() {
if err := bs.targetServer.ListenAndServe(); err != nil {
glog.Warningf("block service: iSCSI target stopped: %v", err)
}
}()
glog.V(0).Infof("block service: iSCSI target started on %s", listenAddr)
return bs
}
// Shutdown gracefully stops the iSCSI target and closes all block volumes.
func (bs *BlockService) Shutdown() {
if bs == nil {
return
}
glog.V(0).Infof("block service: shutting down...")
if bs.targetServer != nil {
bs.targetServer.Close()
}
bs.blockStore.Close()
glog.V(0).Infof("block service: shut down")
}
weed/server/volume_server_block_test.go
+59
View File
@@ -0,0 +1,59 @@
package weed_server
import (
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
func createTestBlockVolFile(t *testing.T, dir, name string) string {
t.Helper()
path := filepath.Join(dir, name)
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 1024 * 4096,
BlockSize: 4096,
ExtentSize: 65536,
WALSize: 1 << 20,
})
if err != nil {
t.Fatal(err)
}
vol.Close()
return path
}
func TestBlockServiceDisabledByDefault(t *testing.T) {
// Empty blockDir means feature is disabled.
bs := StartBlockService("0.0.0.0:3260", "", "")
if bs != nil {
bs.Shutdown()
t.Fatal("expected nil BlockService when blockDir is empty")
}
// Shutdown on nil should be safe (no panic).
var nilBS *BlockService
nilBS.Shutdown()
}
func TestBlockServiceStartAndShutdown(t *testing.T) {
dir := t.TempDir()
createTestBlockVolFile(t, dir, "testvol.blk")
bs := StartBlockService("127.0.0.1:0", dir, "iqn.2024-01.com.test:vol.")
if bs == nil {
t.Fatal("expected non-nil BlockService")
}
defer bs.Shutdown()
// Verify the volume was registered.
paths := bs.blockStore.ListBlockVolumes()
if len(paths) != 1 {
t.Fatalf("expected 1 volume, got %d", len(paths))
}
expected := filepath.Join(dir, "testvol.blk")
if paths[0] != expected {
t.Fatalf("expected path %s, got %s", expected, paths[0])
}
}
weed/storage/blockvol/adapter.go
+47
View File
@@ -0,0 +1,47 @@
package blockvol
import (
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)
// BlockVolAdapter wraps a *BlockVol to implement the iscsi.BlockDevice interface,
// bridging the BlockVol storage engine to iSCSI target sessions.
type BlockVolAdapter struct {
vol *BlockVol
}
// NewBlockVolAdapter creates a BlockDevice adapter for the given BlockVol.
func NewBlockVolAdapter(vol *BlockVol) *BlockVolAdapter {
return &BlockVolAdapter{vol: vol}
}
func (a *BlockVolAdapter) ReadAt(lba uint64, length uint32) ([]byte, error) {
return a.vol.ReadLBA(lba, length)
}
func (a *BlockVolAdapter) WriteAt(lba uint64, data []byte) error {
return a.vol.WriteLBA(lba, data)
}
func (a *BlockVolAdapter) Trim(lba uint64, length uint32) error {
return a.vol.Trim(lba, length)
}
func (a *BlockVolAdapter) SyncCache() error {
return a.vol.SyncCache()
}
func (a *BlockVolAdapter) BlockSize() uint32 {
return a.vol.Info().BlockSize
}
func (a *BlockVolAdapter) VolumeSize() uint64 {
return a.vol.Info().VolumeSize
}
func (a *BlockVolAdapter) IsHealthy() bool {
return a.vol.Info().Healthy
}
// Compile-time check that BlockVolAdapter implements iscsi.BlockDevice.
var _ iscsi.BlockDevice = (*BlockVolAdapter)(nil)
weed/storage/blockvol/adapter_test.go
+70
View File
@@ -0,0 +1,70 @@
package blockvol
import (
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)
func TestAdapterImplementsInterface(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "adapter_test.blk")
vol, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1024 * 4096, // 1024 blocks
BlockSize: 4096,
ExtentSize: 65536,
WALSize: 1 << 20,
})
if err != nil {
t.Fatal(err)
}
defer vol.Close()
adapter := NewBlockVolAdapter(vol)
// Verify it satisfies the interface.
var _ iscsi.BlockDevice = adapter
// Test basic operations through the adapter.
if adapter.BlockSize() != 4096 {
t.Fatalf("BlockSize: got %d, want 4096", adapter.BlockSize())
}
if adapter.VolumeSize() != 1024*4096 {
t.Fatalf("VolumeSize: got %d, want %d", adapter.VolumeSize(), 1024*4096)
}
if !adapter.IsHealthy() {
t.Fatal("expected healthy")
}
// Write and read back through adapter.
data := make([]byte, 4096)
for i := range data {
data[i] = 0xAB
}
if err := adapter.WriteAt(0, data); err != nil {
t.Fatal(err)
}
got, err := adapter.ReadAt(0, 4096)
if err != nil {
t.Fatal(err)
}
if len(got) != 4096 {
t.Fatalf("ReadAt length: got %d, want 4096", len(got))
}
if got[0] != 0xAB || got[4095] != 0xAB {
t.Fatal("data mismatch")
}
// SyncCache through adapter.
if err := adapter.SyncCache(); err != nil {
t.Fatal(err)
}
// Trim through adapter.
if err := adapter.Trim(0, 4096); err != nil {
t.Fatal(err)
}
}
weed/storage/blockvol/blockvol.go
+195 -46
View File
@@ -5,6 +5,7 @@ package blockvol
import (
"encoding/binary"
"errors"
"fmt"
"os"
"sync"
@@ -21,22 +22,38 @@ type CreateOptions struct {
Replication string // default "000"
}
// ErrVolumeClosed is returned when an operation is attempted on a closed BlockVol.
var ErrVolumeClosed = errors.New("blockvol: volume closed")
// BlockVol is the core block volume engine.
type BlockVol struct {
mu sync.RWMutex
fd *os.File
path string
super Superblock
wal *WALWriter
dirtyMap *DirtyMap
groupCommit *GroupCommitter
flusher *Flusher
nextLSN atomic.Uint64
healthy atomic.Bool
mu sync.RWMutex
fd *os.File
path string
super Superblock
config BlockVolConfig
wal *WALWriter
dirtyMap *DirtyMap
groupCommit *GroupCommitter
flusher *Flusher
nextLSN atomic.Uint64
healthy atomic.Bool
closed atomic.Bool
opsOutstanding atomic.Int64 // in-flight Read/Write/Trim/SyncCache ops
opsDrained chan struct{}
}
// CreateBlockVol creates a new block volume file at path.
func CreateBlockVol(path string, opts CreateOptions) (*BlockVol, error) {
func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*BlockVol, error) {
var cfg BlockVolConfig
if len(cfgs) > 0 {
cfg = cfgs[0]
}
cfg.applyDefaults()
if err := cfg.Validate(); err != nil {
return nil, err
}
if opts.VolumeSize == 0 {
return nil, ErrInvalidVolumeSize
}
@@ -74,20 +91,25 @@ func CreateBlockVol(path string, opts CreateOptions) (*BlockVol, error) {
return nil, fmt.Errorf("blockvol: sync: %w", err)
}
dm := NewDirtyMap()
dm := NewDirtyMap(cfg.DirtyMapShards)
wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, 0, 0)
v := &BlockVol{
fd: fd,
path: path,
super: sb,
wal: wal,
dirtyMap: dm,
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dm,
opsDrained: make(chan struct{}, 1),
}
v.nextLSN.Store(1)
v.healthy.Store(true)
v.groupCommit = NewGroupCommitter(GroupCommitterConfig{
SyncFunc: fd.Sync,
OnDegraded: func() { v.healthy.Store(false) },
SyncFunc: fd.Sync,
MaxDelay: cfg.GroupCommitMaxDelay,
MaxBatch: cfg.GroupCommitMaxBatch,
LowWatermark: cfg.GroupCommitLowWatermark,
OnDegraded: func() { v.healthy.Store(false) },
})
go v.groupCommit.Run()
v.flusher = NewFlusher(FlusherConfig{
@@ -95,13 +117,23 @@ func CreateBlockVol(path string, opts CreateOptions) (*BlockVol, error) {
Super: &v.super,
WAL: wal,
DirtyMap: dm,
Interval: cfg.FlushInterval,
})
go v.flusher.Run()
return v, nil
}
// OpenBlockVol opens an existing block volume file and runs crash recovery.
func OpenBlockVol(path string) (*BlockVol, error) {
func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) {
var cfg BlockVolConfig
if len(cfgs) > 0 {
cfg = cfgs[0]
}
cfg.applyDefaults()
if err := cfg.Validate(); err != nil {
return nil, err
}
fd, err := os.OpenFile(path, os.O_RDWR, 0644)
if err != nil {
return nil, fmt.Errorf("blockvol: open file: %w", err)
@@ -117,7 +149,7 @@ func OpenBlockVol(path string) (*BlockVol, error) {
return nil, fmt.Errorf("blockvol: validate superblock: %w", err)
}
dirtyMap := NewDirtyMap()
dirtyMap := NewDirtyMap(cfg.DirtyMapShards)
// Run WAL recovery: replay entries from tail to head.
result, err := RecoverWAL(fd, &sb, dirtyMap)
@@ -133,17 +165,22 @@ func OpenBlockVol(path string) (*BlockVol, error) {
wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, sb.WALHead, sb.WALTail)
v := &BlockVol{
fd: fd,
path: path,
super: sb,
wal: wal,
dirtyMap: dirtyMap,
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dirtyMap,
opsDrained: make(chan struct{}, 1),
}
v.nextLSN.Store(nextLSN)
v.healthy.Store(true)
v.groupCommit = NewGroupCommitter(GroupCommitterConfig{
SyncFunc: fd.Sync,
OnDegraded: func() { v.healthy.Store(false) },
SyncFunc: fd.Sync,
MaxDelay: cfg.GroupCommitMaxDelay,
MaxBatch: cfg.GroupCommitMaxBatch,
LowWatermark: cfg.GroupCommitLowWatermark,
OnDegraded: func() { v.healthy.Store(false) },
})
go v.groupCommit.Run()
v.flusher = NewFlusher(FlusherConfig{
@@ -151,14 +188,71 @@ func OpenBlockVol(path string) (*BlockVol, error) {
Super: &v.super,
WAL: wal,
DirtyMap: dirtyMap,
Interval: cfg.FlushInterval,
})
go v.flusher.Run()
return v, nil
}
// beginOp increments the in-flight ops counter. Returns ErrVolumeClosed if
// the volume is already closed, so callers must not proceed.
func (v *BlockVol) beginOp() error {
v.opsOutstanding.Add(1)
if v.closed.Load() {
v.endOp()
return ErrVolumeClosed
}
return nil
}
// endOp decrements the in-flight ops counter and signals the drain channel
// if this was the last op and the volume is closing.
func (v *BlockVol) endOp() {
if v.opsOutstanding.Add(-1) == 0 && v.closed.Load() {
select {
case v.opsDrained <- struct{}{}:
default:
}
}
}
// appendWithRetry appends a WAL entry, retrying on WAL-full by triggering
// the flusher until the timeout expires.
func (v *BlockVol) appendWithRetry(entry *WALEntry) (uint64, error) {
walOff, err := v.wal.Append(entry)
if errors.Is(err, ErrWALFull) {
deadline := time.After(v.config.WALFullTimeout)
for errors.Is(err, ErrWALFull) {
if v.closed.Load() {
return 0, ErrVolumeClosed
}
v.flusher.NotifyUrgent()
select {
case <-deadline:
return 0, fmt.Errorf("blockvol: WAL full timeout: %w", ErrWALFull)
case <-time.After(1 * time.Millisecond):
}
walOff, err = v.wal.Append(entry)
}
}
if err != nil {
return 0, fmt.Errorf("blockvol: WAL append: %w", err)
}
// Re-check closed after retry: Close() may have freed WAL space
// while we were retrying, allowing Append to succeed.
if v.closed.Load() {
return 0, ErrVolumeClosed
}
return walOff, nil
}
// WriteLBA writes data at the given logical block address.
// Data length must be a multiple of BlockSize.
func (v *BlockVol) WriteLBA(lba uint64, data []byte) error {
if err := v.beginOp(); err != nil {
return err
}
defer v.endOp()
if err := ValidateWrite(lba, uint32(len(data)), v.super.VolumeSize, v.super.BlockSize); err != nil {
return err
}
@@ -173,9 +267,14 @@ func (v *BlockVol) WriteLBA(lba uint64, data []byte) error {
Data: data,
}
walOff, err := v.wal.Append(entry)
walOff, err := v.appendWithRetry(entry)
if err != nil {
return fmt.Errorf("blockvol: WriteLBA: %w", err)
return err
}
// Check WAL pressure and notify flusher if threshold exceeded.
if v.wal.UsedFraction() >= v.config.WALPressureThreshold {
v.flusher.NotifyUrgent()
}
// Update dirty map: one entry per block written.
@@ -191,6 +290,10 @@ func (v *BlockVol) WriteLBA(lba uint64, data []byte) error {
// ReadLBA reads data at the given logical block address.
// length is in bytes and must be a multiple of BlockSize.
func (v *BlockVol) ReadLBA(lba uint64, length uint32) ([]byte, error) {
if err := v.beginOp(); err != nil {
return nil, err
}
defer v.endOp()
if err := ValidateWrite(lba, length, v.super.VolumeSize, v.super.BlockSize); err != nil {
return nil, err
}
@@ -212,9 +315,18 @@ func (v *BlockVol) ReadLBA(lba uint64, length uint32) ([]byte, error) {
// readOneBlock reads a single block, checking dirty map first, then extent.
func (v *BlockVol) readOneBlock(lba uint64) ([]byte, error) {
walOff, _, _, ok := v.dirtyMap.Get(lba)
walOff, lsn, _, ok := v.dirtyMap.Get(lba)
if ok {
return v.readBlockFromWAL(walOff, lba)
data, stale, err := v.readBlockFromWAL(walOff, lba, lsn)
if err != nil {
return nil, err
}
if !stale {
return data, nil
}
// WAL slot was reused (flusher reclaimed it between our
// dirty map read and WAL read). The data is already flushed
// to the extent region, so fall through to extent read.
}
return v.readBlockFromExtent(lba)
}
@@ -224,12 +336,23 @@ func (v *BlockVol) readOneBlock(lba uint64) ([]byte, error) {
const maxWALEntryDataLen = 256 * 1024 * 1024 // 256MB absolute ceiling
// readBlockFromWAL reads a block's data from its WAL entry.
func (v *BlockVol) readBlockFromWAL(walOff uint64, lba uint64) ([]byte, error) {
// Returns (data, stale, err). If stale is true, the WAL slot was reused
// by a newer entry (flusher reclaimed it) and the caller should fall back
// to the extent region.
func (v *BlockVol) readBlockFromWAL(walOff uint64, lba uint64, expectedLSN uint64) ([]byte, bool, error) {
// Read the WAL entry header to get the full entry size.
headerBuf := make([]byte, walEntryHeaderSize)
absOff := int64(v.super.WALOffset + walOff)
if _, err := v.fd.ReadAt(headerBuf, absOff); err != nil {
return nil, fmt.Errorf("readBlockFromWAL: read header at %d: %w", absOff, err)
return nil, false, fmt.Errorf("readBlockFromWAL: read header at %d: %w", absOff, err)
}
// WAL reuse guard: validate LSN before trusting the entry.
// If the flusher reclaimed this slot and a new write reused it,
// the LSN will differ — fall back to extent read.
entryLSN := binary.LittleEndian.Uint64(headerBuf[0:8])
if entryLSN != expectedLSN {
return nil, true, nil // stale — WAL slot reused
}
// Check entry type first — TRIM has no data payload, so Length is
@@ -237,38 +360,44 @@ func (v *BlockVol) readBlockFromWAL(walOff uint64, lba uint64) ([]byte, error) {
entryType := headerBuf[16] // Type is at offset LSN(8) + Epoch(8) = 16
if entryType == EntryTypeTrim {
// TRIM entry: return zeros regardless of Length field.
return make([]byte, v.super.BlockSize), nil
return make([]byte, v.super.BlockSize), false, nil
}
if entryType != EntryTypeWrite {
return nil, fmt.Errorf("readBlockFromWAL: expected WRITE or TRIM entry, got type 0x%02x", entryType)
// Unexpected type at a supposedly valid offset — treat as stale.
return nil, true, nil
}
// Parse and validate the data Length field before allocating (WRITE only).
dataLen := v.parseDataLength(headerBuf)
if uint64(dataLen) > v.super.WALSize || uint64(dataLen) > maxWALEntryDataLen {
return nil, fmt.Errorf("readBlockFromWAL: corrupt entry length %d exceeds WAL size %d", dataLen, v.super.WALSize)
// LSN matched but length is corrupt — real data integrity error.
return nil, false, fmt.Errorf("readBlockFromWAL: corrupt entry length %d exceeds WAL size %d", dataLen, v.super.WALSize)
}
entryLen := walEntryHeaderSize + int(dataLen)
fullBuf := make([]byte, entryLen)
if _, err := v.fd.ReadAt(fullBuf, absOff); err != nil {
return nil, fmt.Errorf("readBlockFromWAL: read entry at %d: %w", absOff, err)
return nil, false, fmt.Errorf("readBlockFromWAL: read entry at %d: %w", absOff, err)
}
entry, err := DecodeWALEntry(fullBuf)
if err != nil {
return nil, fmt.Errorf("readBlockFromWAL: decode: %w", err)
// LSN matched but CRC failed — real corruption.
return nil, false, fmt.Errorf("readBlockFromWAL: decode: %w", err)
}
// Find the block within the entry's data.
// Final guard: verify the entry actually covers this LBA.
if lba < entry.LBA {
return nil, true, nil // stale — different entry at same offset
}
blockOffset := (lba - entry.LBA) * uint64(v.super.BlockSize)
if blockOffset+uint64(v.super.BlockSize) > uint64(len(entry.Data)) {
return nil, fmt.Errorf("readBlockFromWAL: block offset %d out of range for entry data len %d", blockOffset, len(entry.Data))
return nil, true, nil // stale — LBA out of range
}
block := make([]byte, v.super.BlockSize)
copy(block, entry.Data[blockOffset:blockOffset+uint64(v.super.BlockSize)])
return block, nil
return block, false, nil
}
// parseDataLength extracts the Length field from a WAL entry header buffer.
@@ -293,6 +422,10 @@ func (v *BlockVol) readBlockFromExtent(lba uint64) ([]byte, error) {
// The trim is recorded in the WAL with a Length field so the flusher
// can zero the extent region and recovery can replay the trim.
func (v *BlockVol) Trim(lba uint64, length uint32) error {
if err := v.beginOp(); err != nil {
return err
}
defer v.endOp()
if err := ValidateWrite(lba, length, v.super.VolumeSize, v.super.BlockSize); err != nil {
return err
}
@@ -306,9 +439,9 @@ func (v *BlockVol) Trim(lba uint64, length uint32) error {
Length: length,
}
walOff, err := v.wal.Append(entry)
walOff, err := v.appendWithRetry(entry)
if err != nil {
return fmt.Errorf("blockvol: Trim: %w", err)
return err
}
// Update dirty map: mark each trimmed block so the flusher sees it.
@@ -349,18 +482,34 @@ type VolumeInfo struct {
// SyncCache ensures all previously written WAL entries are durable on disk.
// It submits a sync request to the group committer, which batches fsyncs.
func (v *BlockVol) SyncCache() error {
if err := v.beginOp(); err != nil {
return err
}
defer v.endOp()
return v.groupCommit.Submit()
}
// Close shuts down the block volume and closes the file.
// Shutdown order: group committer → stop flusher goroutine → final flush → close fd.
// Shutdown order: drain in-flight ops → group committer → flusher → final flush → close fd.
func (v *BlockVol) Close() error {
v.closed.Store(true)
// Drain in-flight ops: beginOp checks closed and returns ErrVolumeClosed,
// so no new ops can start. Wait for existing ones to finish (max 5s).
if v.opsOutstanding.Load() > 0 {
select {
case <-v.opsDrained:
case <-time.After(5 * time.Second):
// Proceed with shutdown even if ops are stuck.
}
}
if v.groupCommit != nil {
v.groupCommit.Stop()
}
var flushErr error
if v.flusher != nil {
v.flusher.Stop() // stop background goroutine first (no concurrent flush)
v.flusher.Stop() // stop background goroutine first (no concurrent flush)
flushErr = v.flusher.FlushOnce() // then do final flush safely
}
closeErr := v.fd.Close()
weed/storage/blockvol/blockvol_qa_test.go
+67 -154
View File
@@ -431,17 +431,22 @@ func testQAConcurrentWriteRead(t *testing.T) {
func testQAWALFillAdvanceRefill(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "wal_refill.blockvol")
// Small WAL: 128KB.
// Small WAL: 128KB. Short timeout so WAL-full returns quickly.
qaCfg := DefaultConfig()
qaCfg.WALFullTimeout = 10 * time.Millisecond
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 128 * 1024,
})
}, qaCfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
// Stop flusher so we can manually manage WAL tail.
v.flusher.Stop()
entrySize := uint64(walEntryHeaderSize + 4096) // ~4134 bytes per entry
maxEntries := 128 * 1024 / int(entrySize) // ~31 entries
@@ -617,7 +622,7 @@ func testQAWriteReadAllLBAs(t *testing.T) {
// testQADirtyMapRangeDuringDelete: Verify Range + Delete doesn't deadlock.
// This tests the reviewer fix (snapshot-then-iterate pattern).
func testQADirtyMapRangeDuringDelete(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
// Populate 100 entries.
for i := uint64(0); i < 100; i++ {
@@ -699,11 +704,13 @@ func testQAOracleRandomOps(t *testing.T) {
const numBlocks = 64 // small volume for fast test
const volSize = numBlocks * blockSize
qaCfg := DefaultConfig()
qaCfg.WALFullTimeout = 10 * time.Millisecond
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: volSize,
BlockSize: blockSize,
WALSize: 512 * 1024,
})
}, qaCfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
@@ -1512,13 +1519,7 @@ func testQAFlushCheckpointPersists(t *testing.T) {
}
// Update superblock and crash.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
// Reopen — recovery should skip LSN <= checkpoint and replay 5-9.
v2, err := OpenBlockVol(path)
@@ -1545,16 +1546,21 @@ func testQAFlushCheckpointPersists(t *testing.T) {
func testQAFlushWALReclaimThenWrite(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "reclaim.blockvol")
qaCfg := DefaultConfig()
qaCfg.WALFullTimeout = 10 * time.Millisecond
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 128 * 1024, // small WAL
})
}, qaCfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
// Stop background flusher so we can manually manage WAL.
v.flusher.Stop()
entrySize := uint64(walEntryHeaderSize + 4096)
maxEntries := int(128 * 1024 / entrySize)
@@ -1713,13 +1719,7 @@ func testQARecoverTrimEntry(t *testing.T) {
}
// Persist superblock.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -1778,13 +1778,7 @@ func testQARecoverMixedWriteTrimBarrier(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -1872,13 +1866,7 @@ func testQARecoverAfterFlushThenCrash(t *testing.T) {
}
// Persist superblock.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -1927,13 +1915,7 @@ func testQARecoverOverwriteSameLBA(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -1979,15 +1961,8 @@ func testQARecoverCrashLoop(t *testing.T) {
t.Fatalf("iter %d SyncCache: %v", iter, err)
}
// Persist superblock state.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
// Crash and recover.
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v, err = OpenBlockVol(path)
if err != nil {
t.Fatalf("iter %d OpenBlockVol: %v", iter, err)
@@ -2030,6 +2005,10 @@ func testQARecoverCorruptMiddleEntry(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Stop background goroutines before manual superblock/WAL manipulation.
v.groupCommit.Stop()
v.flusher.Stop()
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
@@ -2043,7 +2022,8 @@ func testQARecoverCorruptMiddleEntry(t *testing.T) {
v.fd.WriteAt([]byte{0xFF}, corruptOff)
v.fd.Sync()
path = simulateCrash(v)
v.fd.Close()
path = v.Path()
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2106,13 +2086,7 @@ func testQARecoverMultiBlockWrite(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2207,13 +2181,7 @@ func testQARecoverOracleWithCrash(t *testing.T) {
t.Fatalf("iter %d SyncCache: %v", iter, err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
// Recover.
v, err = OpenBlockVol(path)
@@ -2298,7 +2266,8 @@ func testQALifecycleReadAfterClose(t *testing.T) {
}
}
// testQALifecycleSyncAfterClose: SyncCache after Close must return shutdown error.
// testQALifecycleSyncAfterClose: SyncCache after Close must return an error
// (ErrVolumeClosed from the closed guard, or ErrGroupCommitShutdown).
func testQALifecycleSyncAfterClose(t *testing.T) {
v := createTestVol(t)
v.Close()
@@ -2307,8 +2276,8 @@ func testQALifecycleSyncAfterClose(t *testing.T) {
if err == nil {
t.Error("SyncCache after Close should fail")
}
if !errors.Is(err, ErrGroupCommitShutdown) {
t.Errorf("SyncCache after Close: got %v, want ErrGroupCommitShutdown", err)
if !errors.Is(err, ErrVolumeClosed) && !errors.Is(err, ErrGroupCommitShutdown) {
t.Errorf("SyncCache after Close: got %v, want ErrVolumeClosed or ErrGroupCommitShutdown", err)
}
}
@@ -2586,20 +2555,25 @@ func testQACrashNoSyncDataLossOK(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Persist superblock.
// Stop background goroutines before manual superblock manipulation.
v.groupCommit.Stop()
v.flusher.Stop()
// Persist superblock with current WAL state (before unsynced write).
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
// Write block 1 WITHOUT sync.
// Write block 1 WITHOUT sync — this write's WAL head is NOT in the superblock.
if err := v.WriteLBA(1, makeBlock('U')); err != nil {
t.Fatalf("WriteLBA(unsynced): %v", err)
}
// Hard crash (no sync, no superblock update for block 1).
path = simulateCrash(v)
v.fd.Close()
path = v.Path()
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2662,13 +2636,7 @@ func testQACrashWithFlushThenCrash(t *testing.T) {
}
// Persist superblock with latest WAL state.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2725,13 +2693,7 @@ func testQACrashWALNearFull(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2798,13 +2760,7 @@ func testQACrashConcurrentWriters(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -2877,13 +2833,7 @@ func testQACrashTrimHeavy(t *testing.T) {
t.Fatalf("iter %d SyncCache: %v", iter, err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v, err = OpenBlockVol(path)
if err != nil {
@@ -2955,13 +2905,7 @@ func testQACrashMultiBlockStress(t *testing.T) {
t.Fatalf("iter %d SyncCache: %v", iter, err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v, err = OpenBlockVol(path)
if err != nil {
@@ -3005,12 +2949,7 @@ func testQACrashOverwriteStorm(t *testing.T) {
if errors.Is(err, ErrWALFull) {
// Need to persist what we have and cycle.
v.SyncCache()
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v, err = OpenBlockVol(path)
if err != nil {
t.Fatalf("iter %d reopen: %v", iter, err)
@@ -3028,13 +2967,7 @@ func testQACrashOverwriteStorm(t *testing.T) {
t.Fatalf("iter %d SyncCache: %v", iter, err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v, err = OpenBlockVol(path)
if err != nil {
@@ -3103,6 +3036,10 @@ func testQARecoverEntrySizeMismatchAtTail(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Stop background goroutines before manual superblock/WAL manipulation.
v.groupCommit.Stop()
v.flusher.Stop()
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
@@ -3118,7 +3055,8 @@ func testQARecoverEntrySizeMismatchAtTail(t *testing.T) {
v.fd.WriteAt(badSize[:], entrySizeOff)
v.fd.Sync()
path = simulateCrash(v)
v.fd.Close()
path = v.Path()
v2, err := OpenBlockVol(path)
if err != nil {
@@ -3192,6 +3130,10 @@ func testQARecoverPartialPadding(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Stop background goroutines before manual superblock/WAL manipulation.
v.groupCommit.Stop()
v.flusher.Stop()
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
@@ -3205,7 +3147,8 @@ func testQARecoverPartialPadding(t *testing.T) {
v.fd.WriteAt(garbage, paddingOff)
v.fd.Sync()
path = simulateCrash(v)
v.fd.Close()
path = v.Path()
// Recovery should handle this — either skip corrupt padding and find
// entry 3, or stop at the corruption. Either way, no panic.
@@ -3258,13 +3201,7 @@ func testQARecoverTrimThenWriteSameLBA(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -3310,13 +3247,7 @@ func testQARecoverWriteThenTrimSameLBA(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -3371,13 +3302,7 @@ func testQARecoverBarrierOnlyFullWAL(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -3669,13 +3594,7 @@ func testQAConcurrentFlushAndWrite(t *testing.T) {
time.Sleep(150 * time.Millisecond)
// Persist superblock and crash.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -3841,13 +3760,7 @@ func testQABlockSize512WALSmall(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
weed/storage/blockvol/blockvol_test.go
+1003 -8
View File
File diff suppressed because it is too large Load Diff
weed/storage/blockvol/bug_gc_panic_test.go
+49
View File
@@ -0,0 +1,49 @@
package blockvol
import (
"testing"
"time"
)
// TestBugGCPanicWaitersHung demonstrates that a panic in syncFunc
// leaves all Submit() waiters permanently blocked.
//
// BUG: Run() has no panic recovery. When syncFunc panics, the batch
// of waiters (each blocked on <-ch in Submit) are never notified.
// They hang forever, leaking goroutines.
//
// FIX: Add defer/recover in Run() that drains pending waiters with
// an error before exiting.
//
// This test FAILS until the bug is fixed.
func TestBugGCPanicWaitersHung(t *testing.T) {
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
panic("simulated disk panic")
},
MaxDelay: 10 * time.Millisecond,
})
// Wrap Run() so the panic doesn't kill the test process.
go func() {
defer func() { recover() }()
gc.Run()
}()
// Submit should return an error (not hang forever).
result := make(chan error, 1)
go func() {
result <- gc.Submit()
}()
select {
case err := <-result:
// GOOD: Submit returned (with an error, presumably).
if err == nil {
t.Error("Submit returned nil; expected a panic-related error")
}
t.Logf("Submit returned: %v", err)
case <-time.After(3 * time.Second):
t.Fatal("BUG: Submit hung forever after syncFunc panic -- waiters not drained")
}
}
weed/storage/blockvol/config.go
+86
View File
@@ -0,0 +1,86 @@
package blockvol
import (
"errors"
"fmt"
"time"
)
// BlockVolConfig configures tunable parameters for a BlockVol engine.
// A zero-value config is valid and will use defaults via applyDefaults().
type BlockVolConfig struct {
GroupCommitMaxDelay time.Duration // max wait before flushing a partial batch (default 1ms)
GroupCommitMaxBatch int // flush immediately when this many waiters (default 64)
GroupCommitLowWatermark int // skip delay if fewer than this many pending (default 4)
WALPressureThreshold float64 // fraction of WAL used that triggers urgent flush (default 0.8)
WALFullTimeout time.Duration // max retry time when WAL is full (default 5s)
FlushInterval time.Duration // flusher periodic interval (default 100ms)
DirtyMapShards int // number of dirty map shards, must be power-of-2 (default 256)
}
// DefaultConfig returns a BlockVolConfig with production defaults.
func DefaultConfig() BlockVolConfig {
return BlockVolConfig{
GroupCommitMaxDelay: 1 * time.Millisecond,
GroupCommitMaxBatch: 64,
GroupCommitLowWatermark: 4,
WALPressureThreshold: 0.8,
WALFullTimeout: 5 * time.Second,
FlushInterval: 100 * time.Millisecond,
DirtyMapShards: 256,
}
}
// applyDefaults fills zero-value fields with production defaults.
func (c *BlockVolConfig) applyDefaults() {
d := DefaultConfig()
if c.GroupCommitMaxDelay == 0 {
c.GroupCommitMaxDelay = d.GroupCommitMaxDelay
}
if c.GroupCommitMaxBatch == 0 {
c.GroupCommitMaxBatch = d.GroupCommitMaxBatch
}
if c.GroupCommitLowWatermark == 0 {
c.GroupCommitLowWatermark = d.GroupCommitLowWatermark
}
if c.WALPressureThreshold == 0 {
c.WALPressureThreshold = d.WALPressureThreshold
}
if c.WALFullTimeout == 0 {
c.WALFullTimeout = d.WALFullTimeout
}
if c.FlushInterval == 0 {
c.FlushInterval = d.FlushInterval
}
if c.DirtyMapShards == 0 {
c.DirtyMapShards = d.DirtyMapShards
}
}
var errInvalidConfig = errors.New("blockvol: invalid config")
// Validate checks that config values are within acceptable ranges.
func (c *BlockVolConfig) Validate() error {
if c.DirtyMapShards <= 0 || (c.DirtyMapShards&(c.DirtyMapShards-1)) != 0 {
return fmt.Errorf("%w: DirtyMapShards must be a positive power-of-2, got %d", errInvalidConfig, c.DirtyMapShards)
}
if c.WALPressureThreshold <= 0 || c.WALPressureThreshold > 1 {
return fmt.Errorf("%w: WALPressureThreshold must be in (0,1], got %f", errInvalidConfig, c.WALPressureThreshold)
}
if c.GroupCommitMaxDelay <= 0 {
return fmt.Errorf("%w: GroupCommitMaxDelay must be positive, got %v", errInvalidConfig, c.GroupCommitMaxDelay)
}
if c.GroupCommitMaxBatch <= 0 {
return fmt.Errorf("%w: GroupCommitMaxBatch must be positive, got %d", errInvalidConfig, c.GroupCommitMaxBatch)
}
if c.GroupCommitLowWatermark < 0 {
return fmt.Errorf("%w: GroupCommitLowWatermark must be >= 0, got %d", errInvalidConfig, c.GroupCommitLowWatermark)
}
if c.WALFullTimeout <= 0 {
return fmt.Errorf("%w: WALFullTimeout must be positive, got %v", errInvalidConfig, c.WALFullTimeout)
}
if c.FlushInterval <= 0 {
return fmt.Errorf("%w: FlushInterval must be positive, got %v", errInvalidConfig, c.FlushInterval)
}
return nil
}
weed/storage/blockvol/config_test.go
+115
View File
@@ -0,0 +1,115 @@
package blockvol
import (
"errors"
"testing"
"time"
)
func TestBlockVolConfig(t *testing.T) {
tests := []struct {
name string
run func(t *testing.T)
}{
{name: "config_defaults", run: testConfigDefaults},
{name: "config_validate_good", run: testConfigValidateGood},
{name: "config_validate_bad_shards", run: testConfigValidateBadShards},
{name: "config_validate_bad_threshold", run: testConfigValidateBadThreshold},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.run(t)
})
}
}
func testConfigDefaults(t *testing.T) {
cfg := DefaultConfig()
if cfg.GroupCommitMaxDelay != 1*time.Millisecond {
t.Errorf("GroupCommitMaxDelay = %v, want 1ms", cfg.GroupCommitMaxDelay)
}
if cfg.GroupCommitMaxBatch != 64 {
t.Errorf("GroupCommitMaxBatch = %d, want 64", cfg.GroupCommitMaxBatch)
}
if cfg.GroupCommitLowWatermark != 4 {
t.Errorf("GroupCommitLowWatermark = %d, want 4", cfg.GroupCommitLowWatermark)
}
if cfg.WALPressureThreshold != 0.8 {
t.Errorf("WALPressureThreshold = %f, want 0.8", cfg.WALPressureThreshold)
}
if cfg.WALFullTimeout != 5*time.Second {
t.Errorf("WALFullTimeout = %v, want 5s", cfg.WALFullTimeout)
}
if cfg.FlushInterval != 100*time.Millisecond {
t.Errorf("FlushInterval = %v, want 100ms", cfg.FlushInterval)
}
if cfg.DirtyMapShards != 256 {
t.Errorf("DirtyMapShards = %d, want 256", cfg.DirtyMapShards)
}
if err := cfg.Validate(); err != nil {
t.Errorf("DefaultConfig().Validate() = %v, want nil", err)
}
}
func testConfigValidateGood(t *testing.T) {
cases := []BlockVolConfig{
DefaultConfig(),
{
GroupCommitMaxDelay: 5 * time.Millisecond,
GroupCommitMaxBatch: 128,
GroupCommitLowWatermark: 0,
WALPressureThreshold: 1.0,
WALFullTimeout: 10 * time.Second,
FlushInterval: 50 * time.Millisecond,
DirtyMapShards: 1,
},
{
GroupCommitMaxDelay: 1 * time.Microsecond,
GroupCommitMaxBatch: 1,
GroupCommitLowWatermark: 100,
WALPressureThreshold: 0.01,
WALFullTimeout: 1 * time.Millisecond,
FlushInterval: 1 * time.Millisecond,
DirtyMapShards: 1024,
},
}
for i, cfg := range cases {
if err := cfg.Validate(); err != nil {
t.Errorf("case %d: Validate() = %v, want nil", i, err)
}
}
}
func testConfigValidateBadShards(t *testing.T) {
cases := []int{0, 3, 5, 7, 10, 15, -1}
for _, shards := range cases {
cfg := DefaultConfig()
cfg.DirtyMapShards = shards
err := cfg.Validate()
if err == nil {
t.Errorf("DirtyMapShards=%d: expected error, got nil", shards)
continue
}
if !errors.Is(err, errInvalidConfig) {
t.Errorf("DirtyMapShards=%d: expected errInvalidConfig, got %v", shards, err)
}
}
}
func testConfigValidateBadThreshold(t *testing.T) {
cases := []float64{0, -0.1, -1, 1.01, 2.0}
for _, thresh := range cases {
cfg := DefaultConfig()
cfg.WALPressureThreshold = thresh
err := cfg.Validate()
if err == nil {
t.Errorf("WALPressureThreshold=%f: expected error, got nil", thresh)
continue
}
if !errors.Is(err, errInvalidConfig) {
t.Errorf("WALPressureThreshold=%f: expected errInvalidConfig, got %v", thresh, err)
}
}
}
weed/storage/blockvol/dirty_map.go
+74 -41
View File
@@ -9,30 +9,54 @@ type dirtyEntry struct {
length uint32
}
// DirtyMap is a concurrency-safe map from LBA to WAL offset.
// Phase 1 uses a single shard; Phase 3 upgrades to 256 shards.
type DirtyMap struct {
// dirtyShard is one shard of the sharded dirty map.
type dirtyShard struct {
mu sync.RWMutex
m map[uint64]dirtyEntry
}
// NewDirtyMap creates an empty dirty map.
func NewDirtyMap() *DirtyMap {
return &DirtyMap{m: make(map[uint64]dirtyEntry)}
// DirtyMap is a concurrency-safe map from LBA to WAL offset.
// It uses sharding to reduce lock contention on concurrent workloads.
type DirtyMap struct {
shards []dirtyShard
mask uint64 // numShards - 1, for fast modulo
}
// NewDirtyMap creates an empty dirty map with the given number of shards.
// numShards must be a power-of-2. Use 1 for a single-shard (unsharded) map.
func NewDirtyMap(numShards int) *DirtyMap {
if numShards <= 0 {
numShards = 1
}
shards := make([]dirtyShard, numShards)
for i := range shards {
shards[i].m = make(map[uint64]dirtyEntry)
}
return &DirtyMap{
shards: shards,
mask: uint64(numShards - 1),
}
}
// shard returns the shard for the given LBA.
func (d *DirtyMap) shard(lba uint64) *dirtyShard {
return &d.shards[lba&d.mask]
}
// Put records that the given LBA has dirty data at walOffset.
func (d *DirtyMap) Put(lba uint64, walOffset uint64, lsn uint64, length uint32) {
d.mu.Lock()
d.m[lba] = dirtyEntry{walOffset: walOffset, lsn: lsn, length: length}
d.mu.Unlock()
s := d.shard(lba)
s.mu.Lock()
s.m[lba] = dirtyEntry{walOffset: walOffset, lsn: lsn, length: length}
s.mu.Unlock()
}
// Get returns the dirty entry for lba. ok is false if lba is not dirty.
func (d *DirtyMap) Get(lba uint64) (walOffset uint64, lsn uint64, length uint32, ok bool) {
d.mu.RLock()
e, found := d.m[lba]
d.mu.RUnlock()
s := d.shard(lba)
s.mu.RLock()
e, found := s.m[lba]
s.mu.RUnlock()
if !found {
return 0, 0, 0, false
}
@@ -41,9 +65,10 @@ func (d *DirtyMap) Get(lba uint64) (walOffset uint64, lsn uint64, length uint32,
// Delete removes a single LBA from the dirty map.
func (d *DirtyMap) Delete(lba uint64) {
d.mu.Lock()
delete(d.m, lba)
d.mu.Unlock()
s := d.shard(lba)
s.mu.Lock()
delete(s.m, lba)
s.mu.Unlock()
}
// rangeEntry is a snapshot of one dirty entry for lock-free iteration.
@@ -55,32 +80,37 @@ type rangeEntry struct {
}
// Range calls fn for each dirty entry with LBA in [start, start+count).
// Entries are copied under lock, then fn is called without holding the lock,
// so fn may safely call back into DirtyMap (Put, Delete, etc.).
// Entries are copied under lock (one shard at a time), then fn is called
// without holding any lock, so fn may safely call back into DirtyMap.
func (d *DirtyMap) Range(start uint64, count uint32, fn func(lba, walOffset, lsn uint64, length uint32)) {
end := start + uint64(count)
// Snapshot matching entries under lock.
d.mu.RLock()
entries := make([]rangeEntry, 0, len(d.m))
for lba, e := range d.m {
if lba >= start && lba < end {
entries = append(entries, rangeEntry{lba, e.walOffset, e.lsn, e.length})
var entries []rangeEntry
for i := range d.shards {
s := &d.shards[i]
s.mu.RLock()
for lba, e := range s.m {
if lba >= start && lba < end {
entries = append(entries, rangeEntry{lba, e.walOffset, e.lsn, e.length})
}
}
s.mu.RUnlock()
}
d.mu.RUnlock()
// Call fn without lock.
for _, e := range entries {
fn(e.lba, e.walOffset, e.lsn, e.length)
}
}
// Len returns the number of dirty entries.
// Len returns the number of dirty entries across all shards.
func (d *DirtyMap) Len() int {
d.mu.RLock()
n := len(d.m)
d.mu.RUnlock()
n := 0
for i := range d.shards {
s := &d.shards[i]
s.mu.RLock()
n += len(s.m)
s.mu.RUnlock()
}
return n
}
@@ -92,19 +122,22 @@ type SnapshotEntry struct {
Length uint32
}
// Snapshot returns a copy of all dirty entries. The snapshot is taken under
// read lock but returned without holding the lock.
// Snapshot returns a copy of all dirty entries. Each shard is locked
// individually, and the result is returned without holding any lock.
func (d *DirtyMap) Snapshot() []SnapshotEntry {
d.mu.RLock()
entries := make([]SnapshotEntry, 0, len(d.m))
for lba, e := range d.m {
entries = append(entries, SnapshotEntry{
Lba: lba,
WalOffset: e.walOffset,
Lsn: e.lsn,
Length: e.length,
})
var entries []SnapshotEntry
for i := range d.shards {
s := &d.shards[i]
s.mu.RLock()
for lba, e := range s.m {
entries = append(entries, SnapshotEntry{
Lba: lba,
WalOffset: e.walOffset,
Lsn: e.lsn,
Length: e.length,
})
}
s.mu.RUnlock()
}
d.mu.RUnlock()
return entries
}
weed/storage/blockvol/dirty_map_test.go
+223 -7
View File
@@ -17,6 +17,16 @@ func TestDirtyMap(t *testing.T) {
{name: "dirty_empty", run: testDirtyEmpty},
{name: "dirty_concurrent_rw", run: testDirtyConcurrentRW},
{name: "dirty_range_modify", run: testDirtyRangeModify},
// Phase 3 Task 1.3: Sharded DirtyMap tests.
{name: "sharded_put_get", run: testShardedPutGet},
{name: "sharded_cross_shard", run: testShardedCrossShard},
{name: "sharded_concurrent_rw_256", run: testShardedConcurrentRW256},
{name: "sharded_snapshot_all_shards", run: testShardedSnapshotAllShards},
{name: "sharded_range_cross_shard", run: testShardedRangeCrossShard},
{name: "sharded_1_shard_compat", run: testSharded1ShardCompat},
{name: "sharded_delete", run: testShardedDelete},
{name: "sharded_overwrite", run: testShardedOverwrite},
{name: "sharded_concurrent_range_during_write", run: testShardedConcurrentRangeDuringWrite},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -26,7 +36,7 @@ func TestDirtyMap(t *testing.T) {
}
func testDirtyPutGet(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
dm.Put(100, 5000, 1, 4096)
off, lsn, length, ok := dm.Get(100)
@@ -45,7 +55,7 @@ func testDirtyPutGet(t *testing.T) {
}
func testDirtyOverwrite(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
dm.Put(100, 5000, 1, 4096)
dm.Put(100, 9000, 2, 4096)
@@ -62,7 +72,7 @@ func testDirtyOverwrite(t *testing.T) {
}
func testDirtyDelete(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
dm.Put(100, 5000, 1, 4096)
dm.Delete(100)
@@ -73,7 +83,7 @@ func testDirtyDelete(t *testing.T) {
}
func testDirtyRangeQuery(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
for i := uint64(100); i < 110; i++ {
dm.Put(i, i*1000, i, 4096)
}
@@ -94,7 +104,7 @@ func testDirtyRangeQuery(t *testing.T) {
}
func testDirtyEmpty(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
_, _, _, ok := dm.Get(0)
if ok {
@@ -110,7 +120,7 @@ func testDirtyEmpty(t *testing.T) {
}
func testDirtyConcurrentRW(t *testing.T) {
dm := NewDirtyMap()
dm := NewDirtyMap(1)
const goroutines = 16
const opsPerGoroutine = 1000
@@ -137,7 +147,7 @@ func testDirtyConcurrentRW(t *testing.T) {
func testDirtyRangeModify(t *testing.T) {
// Verify that Range callback can safely call Delete without deadlock.
dm := NewDirtyMap()
dm := NewDirtyMap(1)
for i := uint64(0); i < 10; i++ {
dm.Put(i, i*1000, i+1, 4096)
}
@@ -151,3 +161,209 @@ func testDirtyRangeModify(t *testing.T) {
t.Errorf("after Range+Delete: Len() = %d, want 0", dm.Len())
}
}
// --- Phase 3 Task 1.3: Sharded DirtyMap tests ---
func testShardedPutGet(t *testing.T) {
dm := NewDirtyMap(256)
dm.Put(100, 5000, 1, 4096)
off, lsn, length, ok := dm.Get(100)
if !ok {
t.Fatal("Get(100) returned not-found")
}
if off != 5000 || lsn != 1 || length != 4096 {
t.Errorf("got (%d, %d, %d), want (5000, 1, 4096)", off, lsn, length)
}
}
func testShardedCrossShard(t *testing.T) {
dm := NewDirtyMap(4) // 4 shards: mask = 3
// LBAs 0,1,2,3 each go to different shards.
for i := uint64(0); i < 4; i++ {
dm.Put(i, i*1000, i+1, 4096)
}
if dm.Len() != 4 {
t.Errorf("Len() = %d, want 4", dm.Len())
}
for i := uint64(0); i < 4; i++ {
off, lsn, _, ok := dm.Get(i)
if !ok {
t.Fatalf("Get(%d) not found", i)
}
if off != i*1000 || lsn != i+1 {
t.Errorf("LBA %d: got (%d, %d), want (%d, %d)", i, off, lsn, i*1000, i+1)
}
}
// Delete from specific shards.
dm.Delete(1)
dm.Delete(3)
if dm.Len() != 2 {
t.Errorf("after delete: Len() = %d, want 2", dm.Len())
}
}
func testShardedConcurrentRW256(t *testing.T) {
dm := NewDirtyMap(256)
const goroutines = 32
const opsPerGoroutine = 1000
var wg sync.WaitGroup
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
base := uint64(id * opsPerGoroutine)
for i := uint64(0); i < opsPerGoroutine; i++ {
lba := base + i
dm.Put(lba, lba*10, lba, 4096)
dm.Get(lba)
if i%3 == 0 {
dm.Delete(lba)
}
}
}(g)
}
wg.Wait()
// Test passes if no race/panic.
}
func testShardedSnapshotAllShards(t *testing.T) {
dm := NewDirtyMap(16)
// Insert entries that span all 16 shards.
for i := uint64(0); i < 100; i++ {
dm.Put(i, i*100, i+1, 4096)
}
snap := dm.Snapshot()
if len(snap) != 100 {
t.Errorf("Snapshot len = %d, want 100", len(snap))
}
// Verify all entries present.
seen := make(map[uint64]bool)
for _, e := range snap {
seen[e.Lba] = true
}
for i := uint64(0); i < 100; i++ {
if !seen[i] {
t.Errorf("Snapshot missing LBA %d", i)
}
}
}
func testShardedRangeCrossShard(t *testing.T) {
dm := NewDirtyMap(8)
// LBAs 10-19 spread across shards.
for i := uint64(10); i < 20; i++ {
dm.Put(i, i*100, i, 4096)
}
// Also put entries outside range.
dm.Put(0, 0, 1, 4096)
dm.Put(100, 100, 1, 4096)
found := make(map[uint64]bool)
dm.Range(10, 10, func(lba, walOffset, lsn uint64, length uint32) {
found[lba] = true
})
if len(found) != 10 {
t.Errorf("Range returned %d entries, want 10", len(found))
}
for i := uint64(10); i < 20; i++ {
if !found[i] {
t.Errorf("Range missing LBA %d", i)
}
}
}
func testShardedDelete(t *testing.T) {
dm := NewDirtyMap(256)
dm.Put(100, 5000, 1, 4096)
dm.Delete(100)
_, _, _, ok := dm.Get(100)
if ok {
t.Error("Get(100) should return not-found after delete with 256 shards")
}
if dm.Len() != 0 {
t.Errorf("Len() = %d, want 0", dm.Len())
}
}
func testShardedOverwrite(t *testing.T) {
dm := NewDirtyMap(256)
dm.Put(100, 5000, 1, 4096)
dm.Put(100, 9000, 2, 4096)
off, lsn, _, ok := dm.Get(100)
if !ok {
t.Fatal("Get(100) returned not-found after overwrite")
}
if off != 9000 {
t.Errorf("walOffset = %d, want 9000 (latest)", off)
}
if lsn != 2 {
t.Errorf("lsn = %d, want 2 (latest)", lsn)
}
if dm.Len() != 1 {
t.Errorf("Len() = %d, want 1", dm.Len())
}
}
func testShardedConcurrentRangeDuringWrite(t *testing.T) {
dm := NewDirtyMap(256)
// Pre-populate with entries.
for i := uint64(0); i < 100; i++ {
dm.Put(i, i*10, i+1, 4096)
}
var wg sync.WaitGroup
// Writers: continuously Put new entries.
wg.Add(1)
go func() {
defer wg.Done()
for i := uint64(100); i < 1100; i++ {
dm.Put(i, i*10, i+1, 4096)
}
}()
// Concurrent Range reader.
wg.Add(1)
go func() {
defer wg.Done()
for round := 0; round < 50; round++ {
count := 0
dm.Range(0, 200, func(lba, walOffset, lsn uint64, length uint32) {
count++
})
// Range should return some entries (at least the pre-populated ones).
if count == 0 {
// Shouldn't happen, but don't fatal from goroutine.
return
}
}
}()
wg.Wait()
// Test passes if no race/panic.
}
func testSharded1ShardCompat(t *testing.T) {
// 1-shard map should behave identically to old single-mutex map.
dm := NewDirtyMap(1)
for i := uint64(0); i < 50; i++ {
dm.Put(i, i*10, i+1, 4096)
}
if dm.Len() != 50 {
t.Errorf("Len() = %d, want 50", dm.Len())
}
snap := dm.Snapshot()
if len(snap) != 50 {
t.Errorf("Snapshot len = %d, want 50", len(snap))
}
dm.Range(10, 10, func(lba, walOffset, lsn uint64, length uint32) {
dm.Delete(lba)
})
if dm.Len() != 40 {
t.Errorf("after Range+Delete: Len() = %d, want 40", dm.Len())
}
}
weed/storage/blockvol/flusher.go
+59 -10
View File
@@ -3,6 +3,7 @@ package blockvol
import (
"encoding/binary"
"fmt"
"log"
"os"
"sync"
"time"
@@ -24,6 +25,9 @@ type Flusher struct {
checkpointLSN uint64 // last flushed LSN
checkpointTail uint64 // WAL physical tail after last flush
logger *log.Logger
lastErr bool // true if last FlushOnce returned error
interval time.Duration
notifyCh chan struct{}
stopCh chan struct{}
@@ -38,6 +42,7 @@ type FlusherConfig struct {
WAL *WALWriter
DirtyMap *DirtyMap
Interval time.Duration // default 100ms
Logger *log.Logger // optional; defaults to log.Default()
}
// NewFlusher creates a flusher. Call Run() in a goroutine.
@@ -45,6 +50,9 @@ func NewFlusher(cfg FlusherConfig) *Flusher {
if cfg.Interval == 0 {
cfg.Interval = 100 * time.Millisecond
}
if cfg.Logger == nil {
cfg.Logger = log.Default()
}
return &Flusher{
fd: cfg.FD,
super: cfg.Super,
@@ -54,6 +62,7 @@ func NewFlusher(cfg FlusherConfig) *Flusher {
walSize: cfg.Super.WALSize,
blockSize: cfg.Super.BlockSize,
extentStart: cfg.Super.WALOffset + cfg.Super.WALSize,
logger: cfg.Logger,
checkpointLSN: cfg.Super.WALCheckpointLSN,
checkpointTail: 0,
interval: cfg.Interval,
@@ -74,9 +83,23 @@ func (f *Flusher) Run() {
case <-f.stopCh:
return
case <-ticker.C:
f.FlushOnce()
if err := f.FlushOnce(); err != nil {
if !f.lastErr {
f.logger.Printf("flusher error: %v", err)
}
f.lastErr = true
} else {
f.lastErr = false
}
case <-f.notifyCh:
f.FlushOnce()
if err := f.FlushOnce(); err != nil {
if !f.lastErr {
f.logger.Printf("flusher error: %v", err)
}
f.lastErr = true
} else {
f.lastErr = false
}
}
}
}
@@ -89,6 +112,12 @@ func (f *Flusher) Notify() {
}
}
// NotifyUrgent wakes the flusher for an urgent flush (WAL pressure).
// Phase 3 MVP: delegates to Notify(). Future: may use a priority channel.
func (f *Flusher) NotifyUrgent() {
f.Notify()
}
// Stop shuts down the flusher. Safe to call multiple times.
func (f *Flusher) Stop() {
f.stopOnce.Do(func() {
@@ -126,6 +155,14 @@ func (f *Flusher) FlushOnce() error {
return fmt.Errorf("flusher: read WAL header at %d: %w", absWALOff, err)
}
// WAL reuse guard: validate LSN before trusting the entry.
// Between Snapshot() and this read, the WAL slot could have been
// reused (by a previous flush cycle advancing the tail + new writes).
entryLSN := binary.LittleEndian.Uint64(headerBuf[0:8])
if entryLSN != e.Lsn {
continue // stale — WAL slot reused, skip this entry
}
// Parse entry type and length.
entryType := headerBuf[16] // Type at LSN(8)+Epoch(8)=16
dataLen := parseLength(headerBuf)
@@ -140,17 +177,22 @@ func (f *Flusher) FlushOnce() error {
entry, err := DecodeWALEntry(fullBuf)
if err != nil {
return fmt.Errorf("flusher: decode WAL entry: %w", err)
continue // corrupt or partially overwritten — skip
}
// Write data to extent region.
blocks := entry.Length / f.blockSize
for i := uint32(0); i < blocks; i++ {
blockLBA := entry.LBA + uint64(i)
extentOff := int64(f.extentStart + blockLBA*uint64(f.blockSize))
blockData := entry.Data[i*f.blockSize : (i+1)*f.blockSize]
// Write only this block's data to extent (not all blocks in the
// WAL entry). Other blocks may have been overwritten by newer
// writes and their dirty map entries point elsewhere.
if e.Lba < entry.LBA {
continue // LBA mismatch — stale entry
}
blockIdx := e.Lba - entry.LBA
dataStart := blockIdx * uint64(f.blockSize)
if dataStart+uint64(f.blockSize) <= uint64(len(entry.Data)) {
extentOff := int64(f.extentStart + e.Lba*uint64(f.blockSize))
blockData := entry.Data[dataStart : dataStart+uint64(f.blockSize)]
if _, err := f.fd.WriteAt(blockData, extentOff); err != nil {
return fmt.Errorf("flusher: write extent at LBA %d: %w", blockLBA, err)
return fmt.Errorf("flusher: write extent at LBA %d: %w", e.Lba, err)
}
}
@@ -232,6 +274,13 @@ func (f *Flusher) CheckpointLSN() uint64 {
return f.checkpointLSN
}
// SetFD replaces the file descriptor used for extent writes. Test-only.
func (f *Flusher) SetFD(fd *os.File) {
f.mu.Lock()
defer f.mu.Unlock()
f.fd = fd
}
// parseLength extracts the Length field from a WAL entry header buffer.
func parseLength(headerBuf []byte) uint32 {
// Length at LSN(8)+Epoch(8)+Type(1)+Flags(1)+LBA(8) = 26
weed/storage/blockvol/flusher_test.go
+123
View File
@@ -2,7 +2,10 @@ package blockvol
import (
"bytes"
"log"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
@@ -17,6 +20,10 @@ func TestFlusher(t *testing.T) {
{name: "flush_concurrent_writes", run: testFlushConcurrentWrites},
{name: "flush_frees_wal_space", run: testFlushFreesWALSpace},
{name: "flush_partial", run: testFlushPartial},
// Phase 3 Task 1.6: NotifyUrgent.
{name: "flusher_notify_urgent_triggers_flush", run: testFlusherNotifyUrgentTriggersFlush},
// Phase 3 bug fix: P3-BUG-4 error logging.
{name: "flusher_error_logged", run: testFlusherErrorLogged},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -259,3 +266,119 @@ func testFlushPartial(t *testing.T) {
}
}
}
// --- Phase 3 Task 1.6: NotifyUrgent test ---
func testFlusherNotifyUrgentTriggersFlush(t *testing.T) {
v, f := createTestVolWithFlusher(t)
defer v.Close()
data := makeBlock('U')
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
if v.dirtyMap.Len() != 1 {
t.Fatalf("dirty map len = %d, want 1", v.dirtyMap.Len())
}
// Start flusher in background with long interval.
go f.Run()
defer f.Stop()
// NotifyUrgent should trigger a flush.
f.NotifyUrgent()
// Wait for flush to complete.
deadline := time.After(2 * time.Second)
for {
if v.dirtyMap.Len() == 0 {
break
}
select {
case <-deadline:
t.Fatal("NotifyUrgent did not trigger flush within 2s")
default:
time.Sleep(5 * time.Millisecond)
}
}
got, err := v.ReadLBA(0, 4096)
if err != nil {
t.Fatalf("ReadLBA after urgent flush: %v", err)
}
if !bytes.Equal(got, data) {
t.Error("data mismatch after urgent flush")
}
}
// testFlusherErrorLogged verifies that flusher I/O errors are logged
// and that consecutive errors are deduplicated.
func testFlusherErrorLogged(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "errlog.blockvol")
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 256 * 1024,
})
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
// Write a block so dirty map has an entry.
if err := v.WriteLBA(0, makeBlock('E')); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
// Create a flusher with a captured logger and a CLOSED fd to force error.
var logBuf strings.Builder
logger := log.New(&logBuf, "", 0)
closedFD, err := openAndClose(path)
if err != nil {
t.Fatalf("openAndClose: %v", err)
}
f := NewFlusher(FlusherConfig{
FD: closedFD,
Super: &v.super,
WAL: v.wal,
DirtyMap: v.dirtyMap,
Interval: 1 * time.Hour,
Logger: logger,
})
// Run flusher briefly — FlushOnce should error, and Run should log it.
go f.Run()
f.Notify()
time.Sleep(50 * time.Millisecond)
// Send another notify to test dedup.
f.Notify()
time.Sleep(50 * time.Millisecond)
f.Stop()
logged := logBuf.String()
if !strings.Contains(logged, "flusher error:") {
t.Fatalf("expected 'flusher error:' in log, got: %q", logged)
}
// Should only be logged once (dedup of consecutive errors).
count := strings.Count(logged, "flusher error:")
if count != 1 {
t.Errorf("expected 1 log line (dedup), got %d: %q", count, logged)
}
v.Close()
}
// openAndClose opens a file and immediately closes the fd, returning
// the now-invalid *os.File for injection into flusher tests.
func openAndClose(path string) (*os.File, error) {
fd, err := os.Open(path)
if err != nil {
return nil, err
}
fd.Close()
return fd, nil
}
weed/storage/blockvol/group_commit.go
+74 -34
View File
@@ -2,20 +2,25 @@ package blockvol
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
)
var ErrGroupCommitShutdown = errors.New("blockvol: group committer shut down")
var (
ErrGroupCommitShutdown = errors.New("blockvol: group committer shut down")
ErrGroupCommitPanic = errors.New("blockvol: group committer panic in syncFunc")
)
// GroupCommitter batches SyncCache requests and performs a single fsync
// for the entire batch. This amortizes the cost of fsync across many callers.
type GroupCommitter struct {
syncFunc func() error // called to fsync (injectable for testing)
maxDelay time.Duration // max wait before flushing a partial batch
maxBatch int // flush immediately when this many waiters accumulate
onDegraded func() // called when fsync fails
syncFunc func() error // called to fsync (injectable for testing)
maxDelay time.Duration // max wait before flushing a partial batch
maxBatch int // flush immediately when this many waiters accumulate
lowWatermark int // skip delay if fewer pending (0 = always wait)
onDegraded func() // called when fsync fails
mu sync.Mutex
pending []chan error
@@ -30,10 +35,11 @@ type GroupCommitter struct {
// GroupCommitterConfig configures the group committer.
type GroupCommitterConfig struct {
SyncFunc func() error // required: the fsync function
MaxDelay time.Duration // default 1ms
MaxBatch int // default 64
OnDegraded func() // optional: called on fsync error
SyncFunc func() error // required: the fsync function
MaxDelay time.Duration // default 1ms
MaxBatch int // default 64
LowWatermark int // skip delay if fewer pending (0 = always wait)
OnDegraded func() // optional: called on fsync error
}
// NewGroupCommitter creates a new group committer. Call Run() to start it.
@@ -48,13 +54,14 @@ func NewGroupCommitter(cfg GroupCommitterConfig) *GroupCommitter {
cfg.OnDegraded = func() {}
}
return &GroupCommitter{
syncFunc: cfg.SyncFunc,
maxDelay: cfg.MaxDelay,
maxBatch: cfg.MaxBatch,
onDegraded: cfg.OnDegraded,
notifyCh: make(chan struct{}, 1),
stopCh: make(chan struct{}),
done: make(chan struct{}),
syncFunc: cfg.SyncFunc,
maxDelay: cfg.MaxDelay,
maxBatch: cfg.MaxBatch,
lowWatermark: cfg.LowWatermark,
onDegraded: cfg.OnDegraded,
notifyCh: make(chan struct{}, 1),
stopCh: make(chan struct{}),
done: make(chan struct{}),
}
}
@@ -70,29 +77,44 @@ func (gc *GroupCommitter) Run() {
case <-gc.notifyCh:
}
// Collect batch: wait up to maxDelay for more waiters, or until maxBatch reached.
deadline := time.NewTimer(gc.maxDelay)
for {
// Adaptive: if pending count is below low watermark, flush immediately
// without waiting for more waiters.
skipDelay := false
if gc.lowWatermark > 0 {
gc.mu.Lock()
n := len(gc.pending)
gc.mu.Unlock()
if n >= gc.maxBatch {
deadline.Stop()
break
if n < gc.lowWatermark {
skipDelay = true
}
select {
case <-gc.stopCh:
deadline.Stop()
gc.markStoppedAndDrain()
return
case <-deadline.C:
goto flush
case <-gc.notifyCh:
continue
}
if !skipDelay {
// Collect batch: wait up to maxDelay for more waiters, or until maxBatch reached.
deadline := time.NewTimer(gc.maxDelay)
collectDone := false
for !collectDone {
gc.mu.Lock()
n := len(gc.pending)
gc.mu.Unlock()
if n >= gc.maxBatch {
deadline.Stop()
collectDone = true
break
}
select {
case <-gc.stopCh:
deadline.Stop()
gc.markStoppedAndDrain()
return
case <-deadline.C:
collectDone = true
case <-gc.notifyCh:
continue
}
}
}
flush:
// Take all pending waiters.
gc.mu.Lock()
batch := gc.pending
@@ -103,8 +125,9 @@ func (gc *GroupCommitter) Run() {
continue
}
// Perform fsync.
err := gc.syncFunc()
// Perform fsync with panic recovery. A panic in syncFunc must
// not leave waiters hung — notify them and drain stragglers.
err := gc.callSyncFunc()
gc.syncCount.Add(1)
if err != nil {
gc.onDegraded()
@@ -114,6 +137,12 @@ func (gc *GroupCommitter) Run() {
for _, ch := range batch {
ch <- err
}
// If syncFunc panicked, stop accepting new work.
if errors.Is(err, ErrGroupCommitPanic) {
gc.markStoppedAndDrain()
return
}
}
}
@@ -152,6 +181,17 @@ func (gc *GroupCommitter) SyncCount() uint64 {
return gc.syncCount.Load()
}
// callSyncFunc calls syncFunc with panic recovery. Returns ErrGroupCommitPanic
// wrapping the panic value if syncFunc panics.
func (gc *GroupCommitter) callSyncFunc() (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("%w: %v", ErrGroupCommitPanic, r)
}
}()
return gc.syncFunc()
}
// markStoppedAndDrain sets the stopped flag under mu and drains all pending
// waiters with ErrGroupCommitShutdown. This ensures no new Submit() can
// enqueue after we drain, closing the race window from QA-002.
weed/storage/blockvol/group_commit_test.go
+297
View File
@@ -21,6 +21,15 @@ func TestGroupCommitter(t *testing.T) {
{name: "group_sequential", run: testGroupSequential},
{name: "group_shutdown", run: testGroupShutdown},
{name: "group_submit_after_stop", run: testGroupSubmitAfterStop},
// Phase 3 Task 1.4: Adaptive group commit.
{name: "adaptive_single_immediate", run: testAdaptiveSingleImmediate},
{name: "adaptive_batch_above_watermark", run: testAdaptiveBatchAboveWatermark},
{name: "adaptive_max_batch_triggers_early", run: testAdaptiveMaxBatchTriggersEarly},
{name: "adaptive_watermark_zero_compat", run: testAdaptiveWatermarkZeroCompat},
{name: "adaptive_threshold_boundary", run: testAdaptiveThresholdBoundary},
{name: "adaptive_ramp_up", run: testAdaptiveRampUp},
{name: "adaptive_max_delay_honored", run: testAdaptiveMaxDelayHonored},
{name: "adaptive_fsync_error_all_waiters", run: testAdaptiveFsyncErrorAllWaiters},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -262,6 +271,294 @@ func testGroupShutdown(t *testing.T) {
}
}
// --- Phase 3 Task 1.4: Adaptive group commit tests ---
func testAdaptiveSingleImmediate(t *testing.T) {
// With lowWatermark=4, a single Submit should flush immediately (no delay).
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 5 * time.Second, // very long delay
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
start := time.Now()
if err := gc.Submit(); err != nil {
t.Fatalf("Submit: %v", err)
}
elapsed := time.Since(start)
if syncCalls.Load() != 1 {
t.Errorf("syncCalls = %d, want 1", syncCalls.Load())
}
// Should complete much faster than 5s maxDelay.
if elapsed > 500*time.Millisecond {
t.Errorf("Submit took %v, expected immediate (low watermark skip)", elapsed)
}
}
func testAdaptiveBatchAboveWatermark(t *testing.T) {
// With lowWatermark=4 and a slow sync, 10 concurrent submits should batch.
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
time.Sleep(5 * time.Millisecond) // slow sync so requests queue up
return nil
},
MaxDelay: 50 * time.Millisecond,
MaxBatch: 64,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
const n = 10
var wg sync.WaitGroup
// Pre-enqueue all requests, then start Run.
wg.Add(n)
for i := 0; i < n; i++ {
go func() {
defer wg.Done()
gc.Submit()
}()
}
wg.Wait()
// With slow sync, many should batch. At most n fsyncs (no batching), at least 1.
c := syncCalls.Load()
if c == 0 {
t.Error("syncCalls = 0, want > 0")
}
t.Logf("adaptive batch: %d submits, %d fsyncs", n, c)
}
func testAdaptiveMaxBatchTriggersEarly(t *testing.T) {
// maxBatch should still trigger immediate flush even with adaptive watermark.
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error { return nil },
MaxDelay: 5 * time.Second,
MaxBatch: 8,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
var wg sync.WaitGroup
wg.Add(8)
for i := 0; i < 8; i++ {
go func() {
defer wg.Done()
gc.Submit()
}()
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("maxBatch=8 did not trigger flush with adaptive watermark")
}
}
func testAdaptiveWatermarkZeroCompat(t *testing.T) {
// LowWatermark=0 should preserve Phase 2 behavior (always wait for delay).
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 10 * time.Millisecond,
LowWatermark: 0,
})
go gc.Run()
defer gc.Stop()
// Two sequential submits should each trigger their own fsync.
if err := gc.Submit(); err != nil {
t.Fatalf("Submit 1: %v", err)
}
if err := gc.Submit(); err != nil {
t.Fatalf("Submit 2: %v", err)
}
if c := syncCalls.Load(); c != 2 {
t.Errorf("syncCalls = %d, want 2 (no watermark optimization)", c)
}
}
func testAdaptiveThresholdBoundary(t *testing.T) {
// Exactly lowWatermark concurrent submits: should batch and flush.
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 50 * time.Millisecond,
MaxBatch: 64,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
const n = 4 // exactly lowWatermark
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func() {
defer wg.Done()
gc.Submit()
}()
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
// All 4 should complete. SyncCount should be >= 1.
if c := syncCalls.Load(); c == 0 {
t.Error("syncCalls = 0, want >= 1")
}
case <-time.After(2 * time.Second):
t.Fatal("threshold boundary: timed out waiting for 4 submits")
}
}
func testAdaptiveRampUp(t *testing.T) {
// Start with single callers (below watermark -> immediate), then ramp up.
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 50 * time.Millisecond,
MaxBatch: 64,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
// Phase 1: single serial submits (below watermark, immediate).
for i := 0; i < 5; i++ {
if err := gc.Submit(); err != nil {
t.Fatalf("serial Submit %d: %v", i, err)
}
}
serialSyncs := syncCalls.Load()
if serialSyncs != 5 {
t.Errorf("serial phase: syncCalls = %d, want 5 (1 per call)", serialSyncs)
}
// Phase 2: burst 64 concurrent submits (above watermark, batched).
var wg sync.WaitGroup
wg.Add(64)
for i := 0; i < 64; i++ {
go func() {
defer wg.Done()
gc.Submit()
}()
}
wg.Wait()
totalSyncs := syncCalls.Load()
batchSyncs := totalSyncs - serialSyncs
// 64 submits should complete without deadlock. Some batching expected
// but fast syncFunc means each call may flush individually.
if batchSyncs > 64 {
t.Errorf("batch phase: %d fsyncs for 64 submits, want <= 64", batchSyncs)
}
t.Logf("ramp_up: serial=%d, batch=%d fsyncs for 64 submits", serialSyncs, batchSyncs)
}
func testAdaptiveMaxDelayHonored(t *testing.T) {
// 2 concurrent submits (below watermark) should still complete promptly.
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error { return nil },
MaxDelay: 5 * time.Second,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
var wg sync.WaitGroup
wg.Add(2)
for i := 0; i < 2; i++ {
go func() {
defer wg.Done()
gc.Submit()
}()
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
// Good: completed without waiting 5s maxDelay (adaptive skip).
case <-time.After(2 * time.Second):
t.Fatal("2 submits should complete quickly (below watermark), took >2s")
}
}
func testAdaptiveFsyncErrorAllWaiters(t *testing.T) {
// Inject fsync error during batched flush: all waiters must receive the error.
errIO := errors.New("simulated disk error")
var degradedCalled atomic.Bool
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
return errIO
},
MaxDelay: 50 * time.Millisecond,
MaxBatch: 64,
LowWatermark: 4,
OnDegraded: func() { degradedCalled.Store(true) },
})
go gc.Run()
defer gc.Stop()
const n = 8
var wg sync.WaitGroup
errs := make([]error, n)
wg.Add(n)
for i := 0; i < n; i++ {
go func(idx int) {
defer wg.Done()
errs[idx] = gc.Submit()
}(i)
}
wg.Wait()
for i, err := range errs {
if !errors.Is(err, errIO) {
t.Errorf("Submit[%d]: got %v, want %v", i, err, errIO)
}
}
if !degradedCalled.Load() {
t.Error("OnDegraded was not called")
}
}
func testGroupSubmitAfterStop(t *testing.T) {
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error { return nil },
weed/storage/blockvol/iscsi/bug_dataout_timeout_test.go
+69
View File
@@ -0,0 +1,69 @@
package iscsi
import (
"encoding/binary"
"testing"
"time"
)
// TestBugCollectDataOutNoTimeout demonstrates that collectDataOut
// blocks forever if the initiator stops sending Data-Out PDUs.
//
// BUG: collectDataOut calls ReadPDU(s.conn) in a loop with no
// read deadline. If the initiator sends a WRITE requiring R2T,
// receives the R2T, but never sends Data-Out, the session is stuck
// forever (until TCP keepalive kills it, which could be minutes).
//
// FIX: Set a read deadline on s.conn before entering the Data-Out
// collection loop (e.g., 30 seconds), and return a timeout error
// if the deadline fires.
//
// This test FAILS until the bug is fixed.
func TestBugCollectDataOutNoTimeout(t *testing.T) {
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T (no immediate data accepted)
cfg.DataOutTimeout = 1 * time.Second // short timeout for test
})
doLogin(t, env.clientConn)
// Send WRITE command that requires R2T (no immediate data).
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0xBEEF)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0) // LBA 0
binary.BigEndian.PutUint16(cdb[7:9], 1) // 1 block
cmd.SetCDB(cdb)
// No DataSegment — forces R2T.
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatalf("WritePDU: %v", err)
}
// Read R2T from server.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("ReadPDU (R2T): %v", err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// DO NOT send Data-Out. The session should time out and close.
// Currently it blocks forever in collectDataOut → ReadPDU(s.conn).
select {
case err := <-env.done:
// GOOD: session exited (timed out or errored out).
t.Logf("session exited: %v", err)
case <-time.After(5 * time.Second):
env.clientConn.Close()
t.Fatal("collectDataOut did not time out within 5s")
}
}
weed/storage/blockvol/iscsi/bug_pending_unbounded_test.go
+114
View File
@@ -0,0 +1,114 @@
package iscsi
import (
"encoding/binary"
"runtime"
"testing"
"time"
)
// TestBugPendingQueueUnbounded demonstrates that the pending queue
// in collectDataOut grows without bound.
//
// BUG: During Data-Out collection, any non-DataOut PDU is appended
// to s.pending (session.go line 428) with no limit. A misbehaving
// initiator can send thousands of PDUs during a Data-Out exchange,
// causing unbounded memory growth (OOM risk).
//
// FIX: Cap s.pending at a reasonable limit (e.g., 64 entries).
// Drop or reject excess PDUs with a REJECT response.
//
// This test FAILS until the bug is fixed.
func TestBugPendingQueueUnbounded(t *testing.T) {
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T
})
doLogin(t, env.clientConn)
// Start WRITE requiring R2T.
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0xAAAA)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0)
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Read R2T.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// Flood with 200 NOP-Out PDUs during Data-Out collection.
// These all get queued in s.pending with no limit.
memBefore := getAlloc()
for i := 0; i < 200; i++ {
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(uint32(0xB000 + i))
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
// If session closed or rejected, that's OK.
t.Logf("NOP %d: write failed (session may have rejected): %v", i, err)
break
}
}
// Complete the Data-Out to let the session process the pending queue.
dataOut := &PDU{}
dataOut.SetOpcode(OpSCSIDataOut)
dataOut.SetOpSpecific1(FlagF)
dataOut.SetInitiatorTaskTag(0xAAAA)
dataOut.SetTargetTransferTag(r2t.TargetTransferTag())
dataOut.SetBufferOffset(0)
dataOut.DataSegment = make([]byte, 4096)
if err := WritePDU(env.clientConn, dataOut); err != nil {
// Session may have closed if it correctly rejected the flood.
t.Logf("Data-Out write: %v (may be expected if session rejected flood)", err)
}
// Read responses. Count how many NOP-In responses we get.
env.clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
nopResponses := 0
for {
resp, err := ReadPDU(env.clientConn)
if err != nil {
break
}
if resp.Opcode() == OpNOPIn {
nopResponses++
}
}
memAfter := getAlloc()
t.Logf("pending flood: 200 NOPs sent, %d NOP-In responses, mem delta ~%d KB",
nopResponses, (memAfter-memBefore)/1024)
// BUG: All 200 NOPs were queued in pending and processed.
// A well-behaved server should cap the pending queue.
// With a cap of 64, at most 64 NOP responses should arrive.
const maxAcceptable = 64
if nopResponses > maxAcceptable {
t.Fatalf("BUG: received %d NOP-In responses (all 200 queued in pending) -- "+
"pending queue is unbounded, should be capped at %d", nopResponses, maxAcceptable)
}
}
func getAlloc() uint64 {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return m.Alloc
}
weed/storage/blockvol/iscsi/dataio.go
+70 -6
View File
@@ -172,13 +172,13 @@ func (c *DataOutCollector) Remaining() uint32 {
}
// BuildR2T creates an R2T PDU requesting more data from the initiator.
func BuildR2T(itt, ttt uint32, r2tSN uint32, bufferOffset, desiredLen uint32, statSN, expCmdSN, maxCmdSN uint32) *PDU {
// StatSN is NOT set here — txLoop assigns it (statSNCopy mode, no increment).
func BuildR2T(itt, ttt uint32, r2tSN uint32, bufferOffset, desiredLen uint32, expCmdSN, maxCmdSN uint32) *PDU {
pdu := &PDU{}
pdu.SetOpcode(OpR2T)
pdu.SetOpSpecific1(FlagF) // always Final for R2T
pdu.SetInitiatorTaskTag(itt)
pdu.SetTargetTransferTag(ttt)
pdu.SetStatSN(statSN)
pdu.SetExpCmdSN(expCmdSN)
pdu.SetMaxCmdSN(maxCmdSN)
pdu.SetR2TSN(r2tSN)
@@ -189,25 +189,89 @@ func BuildR2T(itt, ttt uint32, r2tSN uint32, bufferOffset, desiredLen uint32, st
// SendSCSIResponse sends a SCSI Response PDU with optional sense data.
func SendSCSIResponse(w io.Writer, result SCSIResult, itt uint32, statSN *uint32, expCmdSN, maxCmdSN uint32) error {
pdu := BuildSCSIResponse(result, itt, expCmdSN, maxCmdSN)
pdu.SetStatSN(*statSN)
*statSN++
return WritePDU(w, pdu)
}
// BuildSCSIResponse creates a SCSI Response PDU without writing it.
// StatSN is NOT set — the caller (or txLoop) assigns it.
func BuildSCSIResponse(result SCSIResult, itt uint32, expCmdSN, maxCmdSN uint32) *PDU {
pdu := &PDU{}
pdu.SetOpcode(OpSCSIResp)
pdu.SetOpSpecific1(FlagF) // Final
pdu.SetSCSIResponse(ISCSIRespCompleted)
pdu.SetSCSIStatus(result.Status)
pdu.SetInitiatorTaskTag(itt)
pdu.SetStatSN(*statSN)
*statSN++
pdu.SetExpCmdSN(expCmdSN)
pdu.SetMaxCmdSN(maxCmdSN)
if result.Status == SCSIStatusCheckCond {
senseData := BuildSenseData(result.SenseKey, result.SenseASC, result.SenseASCQ)
// Sense data is wrapped in a 2-byte length prefix
pdu.DataSegment = make([]byte, 2+len(senseData))
pdu.DataSegment[0] = byte(len(senseData) >> 8)
pdu.DataSegment[1] = byte(len(senseData))
copy(pdu.DataSegment[2:], senseData)
}
return WritePDU(w, pdu)
return pdu
}
// BuildDataInPDUs splits data into Data-In PDUs and returns them.
// StatSN is NOT set on the final PDU — the txLoop assigns it.
// Intermediate PDUs (without S-bit) never carry StatSN.
func (d *DataInWriter) BuildDataInPDUs(data []byte, itt uint32, expCmdSN, maxCmdSN uint32) []*PDU {
totalLen := uint32(len(data))
if totalLen == 0 {
pdu := &PDU{}
pdu.SetOpcode(OpSCSIDataIn)
pdu.SetOpSpecific1(FlagF | FlagS)
pdu.SetInitiatorTaskTag(itt)
pdu.SetTargetTransferTag(0xFFFFFFFF)
pdu.SetExpCmdSN(expCmdSN)
pdu.SetMaxCmdSN(maxCmdSN)
pdu.SetDataSN(0)
pdu.SetSCSIStatus(SCSIStatusGood)
return []*PDU{pdu}
}
var pdus []*PDU
var offset uint32
var dataSN uint32
for offset < totalLen {
segLen := d.maxSegLen
if offset+segLen > totalLen {
segLen = totalLen - offset
}
isFinal := (offset + segLen) >= totalLen
pdu := &PDU{}
pdu.SetOpcode(OpSCSIDataIn)
pdu.SetInitiatorTaskTag(itt)
pdu.SetTargetTransferTag(0xFFFFFFFF)
pdu.SetExpCmdSN(expCmdSN)
pdu.SetMaxCmdSN(maxCmdSN)
pdu.SetDataSN(dataSN)
pdu.SetBufferOffset(offset)
// Copy data segment (don't share slice with caller).
seg := make([]byte, segLen)
copy(seg, data[offset:offset+segLen])
pdu.DataSegment = seg
if isFinal {
pdu.SetOpSpecific1(FlagF | FlagS)
pdu.SetSCSIStatus(SCSIStatusGood)
} else {
pdu.SetOpSpecific1(0)
}
pdus = append(pdus, pdu)
dataSN++
offset += segLen
}
return pdus
}
weed/storage/blockvol/iscsi/dataio_test.go
+1 -4
View File
@@ -314,7 +314,7 @@ func testDataOutOverflow(t *testing.T) {
}
func testR2TBuild(t *testing.T) {
pdu := BuildR2T(0x100, 0x200, 0, 4096, 4096, 5, 10, 20)
pdu := BuildR2T(0x100, 0x200, 0, 4096, 4096, 10, 20)
if pdu.Opcode() != OpR2T {
t.Fatal("wrong opcode")
}
@@ -333,9 +333,6 @@ func testR2TBuild(t *testing.T) {
if pdu.DesiredDataLength() != 4096 {
t.Fatal("desired length wrong")
}
if pdu.StatSN() != 5 {
t.Fatal("StatSN wrong")
}
}
func testSCSIResponseGood(t *testing.T) {
weed/storage/blockvol/iscsi/login.go
+3
View File
@@ -3,6 +3,7 @@ package iscsi
import (
"errors"
"strconv"
"time"
)
// Login status classes (RFC 7143, Section 11.13.5)
@@ -69,6 +70,7 @@ type TargetConfig struct {
InitialR2T bool
ImmediateData bool
ErrorRecoveryLevel int
DataOutTimeout time.Duration // read deadline for Data-Out collection (default 30s)
}
// DefaultTargetConfig returns sensible defaults for a target.
@@ -86,6 +88,7 @@ func DefaultTargetConfig() TargetConfig {
InitialR2T: true,
ImmediateData: true,
ErrorRecoveryLevel: 0,
DataOutTimeout: 30 * time.Second,
}
}
weed/storage/blockvol/iscsi/qa_rxtx_test.go
+634
View File
@@ -0,0 +1,634 @@
package iscsi
import (
"encoding/binary"
"io"
"log"
"net"
"runtime"
"sync"
"testing"
"time"
)
// TestQAPhase3RXTX tests Phase 3 iSCSI RX/TX split adversarial scenarios.
func TestQAPhase3RXTX(t *testing.T) {
tests := []struct {
name string
run func(t *testing.T)
}{
// QA-2.1: Channel & Goroutine Safety
{name: "rxtx_respch_full_backpressure", run: testQARXTXRespChFullBackpressure},
{name: "rxtx_double_close_session", run: testQARXTXDoubleCloseSession},
{name: "rxtx_txloop_goroutine_leak", run: testQARXTXTxLoopGoroutineLeak},
{name: "rxtx_concurrent_session_50", run: testQARXTXConcurrentSession50},
// QA-2.2: Pending Queue & Data-Out
{name: "rxtx_scsi_cmd_during_dataout", run: testQARXTXSCSICmdDuringDataOut},
{name: "rxtx_nop_response_timing", run: testQARXTXNOPResponseTiming},
// QA-2.3: StatSN & Sequence Edge Cases
{name: "rxtx_statsn_wrap", run: testQARXTXStatSNWrap},
{name: "rxtx_expstsn_mismatch", run: testQARXTXExpStatSNMismatch},
{name: "rxtx_statsn_after_error_response", run: testQARXTXStatSNAfterErrorResponse},
// QA-2.4: Shutdown Ordering
{name: "rxtx_shutdown_writer_queued", run: testQARXTXShutdownWriterQueued},
{name: "rxtx_target_close_active_io", run: testQARXTXTargetCloseActiveIO},
{name: "rxtx_session_after_target_close", run: testQARXTXSessionAfterTargetClose},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.run(t)
})
}
}
// --- QA-2.1: Channel & Goroutine Safety ---
func testQARXTXRespChFullBackpressure(t *testing.T) {
// With net.Pipe, writes block when reader isn't consuming, so we can't
// truly fill respCh without reading. Instead, test that closing the
// connection while enqueue is potentially blocked works cleanly.
env := setupSession(t)
doLogin(t, env.clientConn)
// Start reading responses in background so writes don't block on net.Pipe.
readDone := make(chan int, 1)
go func() {
count := 0
for {
_, err := ReadPDU(env.clientConn)
if err != nil {
break
}
count++
}
readDone <- count
}()
// Send 100 NOPs rapidly.
sent := 0
for i := 0; i < 100; i++ {
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(uint32(0x1000 + i))
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
break
}
sent++
}
// Close conn to trigger cleanup.
env.clientConn.Close()
select {
case count := <-readDone:
t.Logf("backpressure: sent %d NOPs, received %d responses", sent, count)
case <-time.After(3 * time.Second):
t.Fatal("reader did not exit after conn close")
}
select {
case <-env.done:
case <-time.After(3 * time.Second):
t.Fatal("session did not exit after conn close during backpressure")
}
}
func testQARXTXDoubleCloseSession(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Close session twice -- should not panic.
env.session.Close()
env.session.Close()
select {
case <-env.done:
case <-time.After(2 * time.Second):
t.Fatal("session did not exit after double close")
}
}
func testQARXTXTxLoopGoroutineLeak(t *testing.T) {
// Create and destroy 50 sessions rapidly. Goroutine count should not grow.
baseGoroutines := runtime.NumGoroutine()
for i := 0; i < 50; i++ {
client, server := net.Pipe()
dev := newMockDevice(256 * 4096)
config := DefaultTargetConfig()
config.TargetName = testTargetName
resolver := newTestResolverWithDevice(dev)
logger := log.New(io.Discard, "", 0)
sess := NewSession(server, config, resolver, resolver, logger)
done := make(chan error, 1)
go func() {
done <- sess.HandleConnection()
}()
// Login then immediately close.
doLogin(t, client)
client.Close()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatalf("session %d did not exit", i)
}
server.Close()
}
// Allow goroutines to settle.
time.Sleep(100 * time.Millisecond)
runtime.GC()
time.Sleep(50 * time.Millisecond)
finalGoroutines := runtime.NumGoroutine()
// Allow some slack for GC/runtime goroutines.
if finalGoroutines > baseGoroutines+10 {
t.Errorf("goroutine leak: started with %d, ended with %d after 50 sessions",
baseGoroutines, finalGoroutines)
}
t.Logf("goroutine leak check: base=%d, final=%d", baseGoroutines, finalGoroutines)
}
func testQARXTXConcurrentSession50(t *testing.T) {
// 50 concurrent sessions on same TargetServer, each doing I/O.
ts, addr := qaSetupTarget(t)
_ = ts
var wg sync.WaitGroup
wg.Add(50)
for i := 0; i < 50; i++ {
go func(id int) {
defer wg.Done()
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Logf("session %d: dial failed: %v", id, err)
return
}
defer conn.Close()
doLogin(t, conn)
// Write 1 block.
data := make([]byte, 4096)
data[0] = byte(id)
lba := uint32(id % 256)
sendSCSIWriteImmediate(t, conn, lba, data, uint32(id+100), 2)
resp, err := ReadPDU(conn)
if err != nil {
t.Logf("session %d: write response read failed: %v", id, err)
return
}
if resp.Opcode() != OpSCSIResp {
t.Logf("session %d: expected SCSI-Response, got %s", id, OpcodeName(resp.Opcode()))
}
}(i)
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("50 concurrent sessions did not complete in 10s")
}
}
// --- QA-2.2: Pending Queue & Data-Out ---
func testQARXTXSCSICmdDuringDataOut(t *testing.T) {
// Start WRITE requiring R2T, send a READ_10 during Data-Out exchange.
// The READ should be queued in pending and executed after Data-Out completes.
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T
})
doLogin(t, env.clientConn)
// Pre-write data to LBA 10 so we can read it back.
preData := make([]byte, 4096)
for i := range preData {
preData[i] = 0xBB
}
sendSCSIWriteImmediate(t, env.clientConn, 10, preData, 0x50, 2)
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("pre-write read response: %v", err)
}
if resp.Opcode() != OpSCSIResp {
t.Fatalf("pre-write: expected SCSI-Response, got %s", OpcodeName(resp.Opcode()))
}
// Start WRITE to LBA 0 with no immediate data (requires R2T).
writeData := make([]byte, 4096)
for i := range writeData {
writeData[i] = 0xAA
}
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0x100)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(3)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0) // LBA 0
binary.BigEndian.PutUint16(cdb[7:9], 1) // 1 block
cmd.SetCDB(cdb)
// No DataSegment (no immediate data).
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatalf("write cmd: %v", err)
}
// Read R2T.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("read R2T: %v", err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// While Data-Out is expected, send a READ_10 for LBA 10.
sendSCSIRead(t, env.clientConn, 10, 1, 0x200, 4)
// Now send Data-Out to complete the WRITE.
dataOut := &PDU{}
dataOut.SetOpcode(OpSCSIDataOut)
dataOut.SetOpSpecific1(FlagF)
dataOut.SetInitiatorTaskTag(0x100)
dataOut.SetTargetTransferTag(r2t.TargetTransferTag())
dataOut.SetBufferOffset(0)
dataOut.DataSegment = writeData
if err := WritePDU(env.clientConn, dataOut); err != nil {
t.Fatalf("data-out: %v", err)
}
// Should receive WRITE response first, then READ response.
resp1, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("resp1: %v", err)
}
resp2, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("resp2: %v", err)
}
// First response should be WRITE (ITT=0x100).
if resp1.InitiatorTaskTag() != 0x100 {
t.Errorf("first response ITT=0x%x, want 0x100 (WRITE)", resp1.InitiatorTaskTag())
}
// Second response should be READ Data-In or SCSI-Response (ITT=0x200).
if resp2.InitiatorTaskTag() != 0x200 {
t.Errorf("second response ITT=0x%x, want 0x200 (READ)", resp2.InitiatorTaskTag())
}
}
func testQARXTXNOPResponseTiming(t *testing.T) {
// Send NOP-Out during R2T exchange. NOP-In should arrive after WRITE response.
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T
})
doLogin(t, env.clientConn)
// Start WRITE requiring R2T.
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0x300)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0) // LBA 0
binary.BigEndian.PutUint16(cdb[7:9], 1) // 1 block
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Read R2T.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// Send NOP-Out during Data-Out collection.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(0x400)
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatal(err)
}
// Send Data-Out to complete WRITE.
dataOut := &PDU{}
dataOut.SetOpcode(OpSCSIDataOut)
dataOut.SetOpSpecific1(FlagF)
dataOut.SetInitiatorTaskTag(0x300)
dataOut.SetTargetTransferTag(r2t.TargetTransferTag())
dataOut.SetBufferOffset(0)
dataOut.DataSegment = make([]byte, 4096)
if err := WritePDU(env.clientConn, dataOut); err != nil {
t.Fatal(err)
}
// Collect responses: WRITE response (0x300) and NOP-In (0x400).
var responses []*PDU
env.clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
for i := 0; i < 2; i++ {
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("response %d: %v", i, err)
}
responses = append(responses, resp)
}
// WRITE response should come first (Data-Out completed, then pending NOP processed).
if responses[0].InitiatorTaskTag() != 0x300 {
t.Errorf("first response ITT=0x%x, want 0x300 (WRITE)", responses[0].InitiatorTaskTag())
}
if responses[1].InitiatorTaskTag() != 0x400 {
t.Errorf("second response ITT=0x%x, want 0x400 (NOP)", responses[1].InitiatorTaskTag())
}
if responses[1].Opcode() != OpNOPIn {
t.Errorf("second response opcode=%s, want NOP-In", OpcodeName(responses[1].Opcode()))
}
}
// --- QA-2.3: StatSN & Sequence Edge Cases ---
func testQARXTXStatSNWrap(t *testing.T) {
// Drive StatSN close to 0xFFFFFFFF and verify it wraps to 0.
env := setupSession(t)
doLogin(t, env.clientConn)
// We can't easily set the initial StatSN, but we can observe the
// current StatSN from a response and then verify monotonic increment.
// Send 5 NOP-Outs and verify StatSN increases by 1 each time.
var statSNs []uint32
for i := 0; i < 5; i++ {
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(uint32(0x1000 + i))
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatal(err)
}
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
statSNs = append(statSNs, resp.StatSN())
}
// Verify monotonic increment.
for i := 1; i < len(statSNs); i++ {
expected := statSNs[i-1] + 1
if statSNs[i] != expected {
t.Errorf("StatSN[%d] = %d, want %d (monotonic)", i, statSNs[i], expected)
}
}
t.Logf("StatSN sequence: %v", statSNs)
}
func testQARXTXExpStatSNMismatch(t *testing.T) {
// Send command with ExpStatSN != server's StatSN. Per RFC 7143,
// ExpStatSN is advisory and should not cause rejection.
env := setupSession(t)
doLogin(t, env.clientConn)
// Send NOP with wrong ExpStatSN.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(0x5000)
nop.SetImmediate(true)
// Set ExpStatSN to a wrong value.
binary.BigEndian.PutUint32(nop.BHS[28:32], 0xDEADBEEF)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatal(err)
}
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
// Should still get NOP-In response (ExpStatSN mismatch is not an error).
if resp.Opcode() != OpNOPIn {
t.Errorf("expected NOP-In, got %s", OpcodeName(resp.Opcode()))
}
if resp.InitiatorTaskTag() != 0x5000 {
t.Errorf("ITT = 0x%x, want 0x5000", resp.InitiatorTaskTag())
}
}
func testQARXTXStatSNAfterErrorResponse(t *testing.T) {
// SCSI error response should still increment StatSN.
env := setupSession(t)
doLogin(t, env.clientConn)
// First: send a NOP to get current StatSN.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(0x6000)
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatal(err)
}
nopResp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
baseSN := nopResp.StatSN()
// Second: send READ to out-of-range LBA (causes CHECK_CONDITION).
var cdb [16]byte
cdb[0] = ScsiRead10
binary.BigEndian.PutUint32(cdb[2:6], 0xFFFFFFF0) // huge LBA
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagR)
cmd.SetInitiatorTaskTag(0x6001)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
errResp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if errResp.Opcode() != OpSCSIResp {
t.Fatalf("expected SCSI-Response, got %s", OpcodeName(errResp.Opcode()))
}
errSN := errResp.StatSN()
// Third: send another NOP.
nop2 := &PDU{}
nop2.SetOpcode(OpNOPOut)
nop2.SetOpSpecific1(FlagF)
nop2.SetInitiatorTaskTag(0x6002)
nop2.SetImmediate(true)
if err := WritePDU(env.clientConn, nop2); err != nil {
t.Fatal(err)
}
nop2Resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
afterSN := nop2Resp.StatSN()
// Error response should have incremented StatSN.
if errSN != baseSN+1 {
t.Errorf("error StatSN = %d, want %d (baseSN+1)", errSN, baseSN+1)
}
if afterSN != baseSN+2 {
t.Errorf("after error StatSN = %d, want %d (baseSN+2)", afterSN, baseSN+2)
}
t.Logf("StatSN: base=%d, error=%d, after=%d", baseSN, errSN, afterSN)
}
// --- QA-2.4: Shutdown Ordering ---
func testQARXTXShutdownWriterQueued(t *testing.T) {
// Enqueue multiple responses, then disconnect.
// Writer should drain all queued responses before exiting.
env := setupSession(t)
doLogin(t, env.clientConn)
// Send 10 NOP-Outs rapidly.
for i := 0; i < 10; i++ {
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(uint32(0x7000 + i))
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatalf("NOP %d: %v", i, err)
}
}
// Read all 10 responses.
env.clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
count := 0
for i := 0; i < 10; i++ {
_, err := ReadPDU(env.clientConn)
if err != nil {
break
}
count++
}
if count != 10 {
t.Errorf("received %d NOP-In responses, want 10", count)
}
}
func testQARXTXTargetCloseActiveIO(t *testing.T) {
// Heavy I/O on multiple sessions, target.Close() mid-flight.
ts, addr := qaSetupTarget(t)
var wg sync.WaitGroup
wg.Add(4)
// Start 4 sessions doing I/O.
for i := 0; i < 4; i++ {
go func(id int) {
defer wg.Done()
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
return
}
defer conn.Close()
doLogin(t, conn)
// Write loop until connection closes.
for j := 0; j < 100; j++ {
data := make([]byte, 4096)
data[0] = byte(id)
sendSCSIWriteImmediate(t, conn, uint32(id), data, uint32(j+100), uint32(j+2))
_, err := ReadPDU(conn)
if err != nil {
return // expected: target closed
}
}
}(i)
}
// Give sessions time to start I/O.
time.Sleep(50 * time.Millisecond)
// Close target while I/O is in progress.
ts.Close()
// All goroutines should exit cleanly.
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("sessions did not exit after target.Close()")
}
}
func testQARXTXSessionAfterTargetClose(t *testing.T) {
// Close target, then try to connect -- should fail or close immediately.
ts, addr := qaSetupTarget(t)
// Verify one session works.
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Fatalf("initial dial: %v", err)
}
doLogin(t, conn)
conn.Close()
// Close target.
ts.Close()
// New connection should fail.
conn2, err := net.DialTimeout("tcp", addr, 500*time.Millisecond)
if err != nil {
// Expected: listener closed, dial fails.
return
}
defer conn2.Close()
// If dial succeeded (unlikely), the connection should be closed by server.
conn2.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
_, err = ReadPDU(conn2)
if err == nil {
t.Error("expected error reading from connection after target close")
}
}
weed/storage/blockvol/iscsi/qa_stability_test.go
+536
View File
@@ -0,0 +1,536 @@
package iscsi_test
import (
"bytes"
"encoding/binary"
"io"
"log"
"net"
"path/filepath"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)
// TestQAPhase3Stability tests Phase 3 integration stability scenarios:
// WAL pressure through iSCSI, crash recovery with sessions, lifecycle stress.
func TestQAPhase3Stability(t *testing.T) {
tests := []struct {
name string
run func(t *testing.T)
}{
// QA-3.1: WAL Pressure + iSCSI E2E
{name: "e2e_sustained_write_1000", run: testE2ESustainedWrite1000},
{name: "e2e_write_read_under_pressure", run: testE2EWriteReadUnderPressure},
{name: "e2e_synccache_under_pressure", run: testE2ESyncCacheUnderPressure},
{name: "e2e_multiple_sessions_shared_vol", run: testE2EMultipleSessionsSharedVol},
// QA-3.2: Crash Recovery + Session Reconnect
{name: "e2e_crash_recovery_via_iscsi", run: testE2ECrashRecoveryViaISCSI},
{name: "e2e_session_reconnect", run: testE2ESessionReconnect},
// QA-3.3: Lifecycle & Resource Stress
{name: "e2e_rapid_open_close_target", run: testE2ERapidOpenCloseTarget},
{name: "e2e_config_extreme_values", run: testE2EConfigExtremeValues},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.run(t)
})
}
}
// --- Helpers ---
func setupStabilityTarget(t *testing.T, opts blockvol.CreateOptions, cfgs ...blockvol.BlockVolConfig) (*blockvol.BlockVol, net.Conn, *iscsi.TargetServer, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "stability.blk")
var vol *blockvol.BlockVol
var err error
if len(cfgs) > 0 {
vol, err = blockvol.CreateBlockVol(path, opts, cfgs[0])
} else {
vol, err = blockvol.CreateBlockVol(path, opts)
}
if err != nil {
t.Fatal(err)
}
adapter := &blockVolAdapter{vol: vol}
config := iscsi.DefaultTargetConfig()
config.TargetName = intTargetName
logger := log.New(io.Discard, "", 0)
ts := iscsi.NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(intTargetName, adapter)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
go ts.Serve(ln)
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Fatal(err)
}
// Login
stabilityDoLogin(t, conn)
t.Cleanup(func() {
conn.Close()
ts.Close()
vol.Close()
})
return vol, conn, ts, addr
}
func stabilityDoLogin(t *testing.T, conn net.Conn) {
t.Helper()
params := iscsi.NewParams()
params.Set("InitiatorName", intInitiatorName)
params.Set("TargetName", intTargetName)
params.Set("SessionType", "Normal")
loginReq := &iscsi.PDU{}
loginReq.SetOpcode(iscsi.OpLoginReq)
loginReq.SetLoginStages(iscsi.StageSecurityNeg, iscsi.StageFullFeature)
loginReq.SetLoginTransit(true)
loginReq.SetISID([6]byte{0x00, 0x02, 0x3D, 0x00, 0x00, 0x01})
loginReq.SetCmdSN(1)
loginReq.DataSegment = params.Encode()
if err := iscsi.WritePDU(conn, loginReq); err != nil {
t.Fatal(err)
}
resp, err := iscsi.ReadPDU(conn)
if err != nil {
t.Fatal(err)
}
if resp.LoginStatusClass() != iscsi.LoginStatusSuccess {
t.Fatalf("login failed: %d/%d", resp.LoginStatusClass(), resp.LoginStatusDetail())
}
}
func stabilityWrite(t *testing.T, conn net.Conn, lba uint32, data []byte, cmdSN uint32) {
t.Helper()
var cdb [16]byte
cdb[0] = iscsi.ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], lba)
blocks := uint16(len(data) / 4096)
binary.BigEndian.PutUint16(cdb[7:9], blocks)
resp := sendSCSICmd(t, conn, cdb, cmdSN, false, true, data, uint32(len(data)))
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("write LBA %d failed: status %d", lba, resp.SCSIStatus())
}
}
func stabilityRead(t *testing.T, conn net.Conn, lba uint32, cmdSN uint32) []byte {
t.Helper()
var cdb [16]byte
cdb[0] = iscsi.ScsiRead10
binary.BigEndian.PutUint32(cdb[2:6], lba)
binary.BigEndian.PutUint16(cdb[7:9], 1)
resp := sendSCSICmd(t, conn, cdb, cmdSN, true, false, nil, 4096)
if resp.Opcode() != iscsi.OpSCSIDataIn {
t.Fatalf("read LBA %d: expected Data-In, got %s", lba, iscsi.OpcodeName(resp.Opcode()))
}
return resp.DataSegment
}
// --- QA-3.1: WAL Pressure + iSCSI E2E ---
func testE2ESustainedWrite1000(t *testing.T) {
// 1000 sequential WRITEs via iSCSI, flusher running.
_, conn, _, _ := setupStabilityTarget(t, blockvol.CreateOptions{
VolumeSize: 1024 * 4096,
BlockSize: 4096,
WALSize: 256 * 1024, // 256KB WAL
})
cmdSN := uint32(2)
for i := 0; i < 1000; i++ {
lba := uint32(i % 1024)
data := make([]byte, 4096)
data[0] = byte(i % 256)
data[1] = byte(i / 256)
var cdb [16]byte
cdb[0] = iscsi.ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], lba)
binary.BigEndian.PutUint16(cdb[7:9], 1)
resp := sendSCSICmd(t, conn, cdb, cmdSN, false, true, data, 4096)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("write %d (LBA %d) failed: status %d", i, lba, resp.SCSIStatus())
}
cmdSN++
}
t.Log("sustained: 1000 WRITEs completed successfully")
}
func testE2EWriteReadUnderPressure(t *testing.T) {
// Heavy writes to create WAL pressure, concurrent reads should still work.
cfg := blockvol.DefaultConfig()
cfg.WALPressureThreshold = 0.3
cfg.FlushInterval = 5 * time.Millisecond
_, conn, _, _ := setupStabilityTarget(t, blockvol.CreateOptions{
VolumeSize: 256 * 4096,
BlockSize: 4096,
WALSize: 64 * 1024, // small WAL for pressure
}, cfg)
// Write to first 50 LBAs.
cmdSN := uint32(2)
for i := 0; i < 50; i++ {
data := bytes.Repeat([]byte{byte(i)}, 4096)
stabilityWrite(t, conn, uint32(i), data, cmdSN)
cmdSN++
}
// Read them back under potential WAL pressure.
for i := 0; i < 50; i++ {
readData := stabilityRead(t, conn, uint32(i), cmdSN)
cmdSN++
expected := bytes.Repeat([]byte{byte(i)}, 4096)
if !bytes.Equal(readData, expected) {
t.Fatalf("LBA %d: data mismatch under pressure", i)
}
}
}
func testE2ESyncCacheUnderPressure(t *testing.T) {
// SYNC_CACHE while WAL is under pressure.
cfg := blockvol.DefaultConfig()
cfg.WALPressureThreshold = 0.3
cfg.FlushInterval = 5 * time.Millisecond
_, conn, _, _ := setupStabilityTarget(t, blockvol.CreateOptions{
VolumeSize: 256 * 4096,
BlockSize: 4096,
WALSize: 64 * 1024,
}, cfg)
// Write enough to create pressure.
cmdSN := uint32(2)
for i := 0; i < 30; i++ {
data := bytes.Repeat([]byte{byte(i)}, 4096)
stabilityWrite(t, conn, uint32(i%256), data, cmdSN)
cmdSN++
}
// SYNC_CACHE should succeed even under pressure.
var syncCDB [16]byte
syncCDB[0] = iscsi.ScsiSyncCache10
resp := sendSCSICmd(t, conn, syncCDB, cmdSN, false, false, nil, 0)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("SYNC_CACHE under pressure failed: status %d", resp.SCSIStatus())
}
}
func testE2EMultipleSessionsSharedVol(t *testing.T) {
// 4 sessions on same BlockVol, each writing to different LBA ranges.
path := filepath.Join(t.TempDir(), "shared.blk")
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 1024 * 4096,
BlockSize: 4096,
WALSize: 512 * 1024,
})
if err != nil {
t.Fatal(err)
}
defer vol.Close()
adapter := &blockVolAdapter{vol: vol}
config := iscsi.DefaultTargetConfig()
config.TargetName = intTargetName
logger := log.New(io.Discard, "", 0)
ts := iscsi.NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(intTargetName, adapter)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
go ts.Serve(ln)
defer ts.Close()
var wg sync.WaitGroup
wg.Add(4)
for sess := 0; sess < 4; sess++ {
go func(id int) {
defer wg.Done()
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Logf("session %d: dial failed: %v", id, err)
return
}
defer conn.Close()
stabilityDoLogin(t, conn)
// Each session writes to its own LBA range (id*10 to id*10+9).
cmdSN := uint32(2)
baseLBA := uint32(id * 10)
for i := 0; i < 10; i++ {
data := make([]byte, 4096)
data[0] = byte(id)
data[1] = byte(i)
var cdb [16]byte
cdb[0] = iscsi.ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], baseLBA+uint32(i))
binary.BigEndian.PutUint16(cdb[7:9], 1)
resp := sendSCSICmd(t, conn, cdb, cmdSN, false, true, data, 4096)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Logf("session %d: write %d failed", id, i)
return
}
cmdSN++
}
// Read back and verify.
for i := 0; i < 10; i++ {
var cdb [16]byte
cdb[0] = iscsi.ScsiRead10
binary.BigEndian.PutUint32(cdb[2:6], baseLBA+uint32(i))
binary.BigEndian.PutUint16(cdb[7:9], 1)
resp := sendSCSICmd(t, conn, cdb, cmdSN, true, false, nil, 4096)
if resp.Opcode() != iscsi.OpSCSIDataIn {
t.Logf("session %d: read %d: expected Data-In", id, i)
return
}
if resp.DataSegment[0] != byte(id) || resp.DataSegment[1] != byte(i) {
t.Errorf("session %d: LBA %d data mismatch: got [%d,%d], want [%d,%d]",
id, baseLBA+uint32(i), resp.DataSegment[0], resp.DataSegment[1], id, i)
}
cmdSN++
}
}(sess)
}
wg.Wait()
}
// --- QA-3.2: Crash Recovery + Session Reconnect ---
func testE2ECrashRecoveryViaISCSI(t *testing.T) {
// Write via iSCSI, close vol (simulating crash), reopen and verify data via direct read.
dir := t.TempDir()
path := filepath.Join(dir, "crash.blk")
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 256 * 4096,
BlockSize: 4096,
WALSize: 128 * 1024,
})
if err != nil {
t.Fatal(err)
}
adapter := &blockVolAdapter{vol: vol}
config := iscsi.DefaultTargetConfig()
config.TargetName = intTargetName
logger := log.New(io.Discard, "", 0)
ts := iscsi.NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(intTargetName, adapter)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
go ts.Serve(ln)
conn, err := net.DialTimeout("tcp", ln.Addr().String(), 2*time.Second)
if err != nil {
t.Fatal(err)
}
stabilityDoLogin(t, conn)
// Write 10 blocks via iSCSI.
cmdSN := uint32(2)
for i := 0; i < 10; i++ {
data := bytes.Repeat([]byte{byte(i + 0xA0)}, 4096)
stabilityWrite(t, conn, uint32(i), data, cmdSN)
cmdSN++
}
// SYNC_CACHE to ensure WAL is durable.
var syncCDB [16]byte
syncCDB[0] = iscsi.ScsiSyncCache10
sendSCSICmd(t, conn, syncCDB, cmdSN, false, false, nil, 0)
// Close everything (simulate crash).
conn.Close()
ts.Close()
vol.Close()
// Reopen and verify data.
vol2, err := blockvol.OpenBlockVol(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer vol2.Close()
for i := 0; i < 10; i++ {
data, err := vol2.ReadLBA(uint64(i), 4096)
if err != nil {
t.Fatalf("ReadLBA(%d) after recovery: %v", i, err)
}
expected := bytes.Repeat([]byte{byte(i + 0xA0)}, 4096)
if !bytes.Equal(data, expected) {
t.Fatalf("LBA %d: data mismatch after crash recovery", i)
}
}
}
func testE2ESessionReconnect(t *testing.T) {
// Write via session 1, disconnect, reconnect as session 2, read back.
dir := t.TempDir()
path := filepath.Join(dir, "reconnect.blk")
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 256 * 4096,
BlockSize: 4096,
WALSize: 128 * 1024,
})
if err != nil {
t.Fatal(err)
}
defer vol.Close()
adapter := &blockVolAdapter{vol: vol}
config := iscsi.DefaultTargetConfig()
config.TargetName = intTargetName
logger := log.New(io.Discard, "", 0)
ts := iscsi.NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(intTargetName, adapter)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
go ts.Serve(ln)
defer ts.Close()
// Session 1: write data.
conn1, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Fatal(err)
}
stabilityDoLogin(t, conn1)
writeData := bytes.Repeat([]byte{0xDE}, 4096)
stabilityWrite(t, conn1, 5, writeData, 2)
conn1.Close()
// Session 2: read back.
conn2, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err != nil {
t.Fatal(err)
}
defer conn2.Close()
stabilityDoLogin(t, conn2)
readData := stabilityRead(t, conn2, 5, 2)
if !bytes.Equal(readData, writeData) {
t.Fatal("data mismatch after session reconnect")
}
}
// --- QA-3.3: Lifecycle & Resource Stress ---
func testE2ERapidOpenCloseTarget(t *testing.T) {
// Open TargetServer, do I/O, close. Repeat 10 times. No fd/goroutine leak.
dir := t.TempDir()
path := filepath.Join(dir, "rapid.blk")
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 256 * 4096,
BlockSize: 4096,
WALSize: 128 * 1024,
})
if err != nil {
t.Fatal(err)
}
defer vol.Close()
for cycle := 0; cycle < 10; cycle++ {
adapter := &blockVolAdapter{vol: vol}
config := iscsi.DefaultTargetConfig()
config.TargetName = intTargetName
logger := log.New(io.Discard, "", 0)
ts := iscsi.NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(intTargetName, adapter)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("cycle %d: listen: %v", cycle, err)
}
go ts.Serve(ln)
conn, err := net.DialTimeout("tcp", ln.Addr().String(), 2*time.Second)
if err != nil {
t.Fatalf("cycle %d: dial: %v", cycle, err)
}
stabilityDoLogin(t, conn)
// Write one block.
data := bytes.Repeat([]byte{byte(cycle)}, 4096)
stabilityWrite(t, conn, uint32(cycle), data, 2)
conn.Close()
ts.Close()
}
// Verify all 10 LBAs have correct data.
for i := 0; i < 10; i++ {
data, err := vol.ReadLBA(uint64(i), 4096)
if err != nil {
t.Fatalf("ReadLBA(%d) after 10 cycles: %v", i, err)
}
if data[0] != byte(i) {
t.Errorf("LBA %d: byte[0] = %d, want %d", i, data[0], i)
}
}
}
func testE2EConfigExtremeValues(t *testing.T) {
// Create BlockVol with extreme but valid config, use via iSCSI.
cfg := blockvol.DefaultConfig()
cfg.GroupCommitMaxDelay = 100 * time.Microsecond // very fast
cfg.DirtyMapShards = 1 // minimum
cfg.FlushInterval = 1 * time.Millisecond
_, conn, _, _ := setupStabilityTarget(t, blockvol.CreateOptions{
VolumeSize: 64 * 4096,
BlockSize: 4096,
WALSize: 32 * 1024, // small WAL
}, cfg)
// Write + read should work.
cmdSN := uint32(2)
for i := 0; i < 20; i++ {
data := bytes.Repeat([]byte{byte(i)}, 4096)
stabilityWrite(t, conn, uint32(i%64), data, cmdSN)
cmdSN++
}
// SYNC_CACHE.
var syncCDB [16]byte
syncCDB[0] = iscsi.ScsiSyncCache10
resp := sendSCSICmd(t, conn, syncCDB, cmdSN, false, false, nil, 0)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("SYNC_CACHE with extreme config failed: %d", resp.SCSIStatus())
}
}
weed/storage/blockvol/iscsi/scsi.go
+187 -1
View File
@@ -9,6 +9,7 @@ const (
ScsiTestUnitReady uint8 = 0x00
ScsiInquiry uint8 = 0x12
ScsiModeSense6 uint8 = 0x1a
ScsiModeSense10 uint8 = 0x5a
ScsiReadCapacity10 uint8 = 0x25
ScsiRead10 uint8 = 0x28
ScsiWrite10 uint8 = 0x2a
@@ -19,6 +20,7 @@ const (
ScsiWrite16 uint8 = 0x8a
ScsiReadCapacity16 uint8 = 0x9e // SERVICE ACTION IN (16), SA=0x10
ScsiSyncCache16 uint8 = 0x91
ScsiWriteSame16 uint8 = 0x93
)
// Service action for READ CAPACITY (16)
@@ -95,6 +97,8 @@ func (h *SCSIHandler) HandleCommand(cdb [16]byte, dataOut []byte) SCSIResult {
return h.inquiry(cdb)
case ScsiModeSense6:
return h.modeSense6(cdb)
case ScsiModeSense10:
return h.modeSense10(cdb)
case ScsiReadCapacity10:
return h.readCapacity10()
case ScsiReadCapacity16:
@@ -119,6 +123,8 @@ func (h *SCSIHandler) HandleCommand(cdb [16]byte, dataOut []byte) SCSIResult {
return h.syncCache()
case ScsiUnmap:
return h.unmap(cdb, dataOut)
case ScsiWriteSame16:
return h.writeSame16(cdb, dataOut)
default:
return illegalRequest(ASCInvalidOpcode, ASCQLuk)
}
@@ -180,10 +186,12 @@ func (h *SCSIHandler) inquiryVPD(pageCode uint8, allocLen uint16) SCSIResult {
data := []byte{
0x00, // device type
0x00, // page code
0x00, 0x03, // page length
0x00, 0x05, // page length
0x00, // supported pages: 0x00
0x80, // 0x80 (serial)
0x83, // 0x83 (device identification)
0xb0, // 0xB0 (block limits)
0xb2, // 0xB2 (logical block provisioning)
}
if int(allocLen) < len(data) {
data = data[:allocLen]
@@ -221,11 +229,92 @@ func (h *SCSIHandler) inquiryVPD(pageCode uint8, allocLen uint16) SCSIResult {
}
return SCSIResult{Status: SCSIStatusGood, Data: data}
case 0xb0: // Block Limits (SBC-4, Section 6.6.4)
return h.inquiryVPDB0(allocLen)
case 0xb2: // Logical Block Provisioning (SBC-4, Section 6.6.6)
return h.inquiryVPDB2(allocLen)
default:
return illegalRequest(ASCInvalidFieldInCDB, ASCQLuk)
}
}
// inquiryVPDB0 returns the Block Limits VPD page (0xB0).
// Tells the kernel our maximum WRITE SAME, UNMAP, and transfer lengths.
func (h *SCSIHandler) inquiryVPDB0(allocLen uint16) SCSIResult {
blockSize := h.dev.BlockSize()
totalBlocks := h.dev.VolumeSize() / uint64(blockSize)
// Cap WRITE SAME to the whole volume (no practical limit).
maxWS := totalBlocks
if maxWS > 0xFFFFFFFF {
maxWS = 0xFFFFFFFF
}
data := make([]byte, 64)
data[0] = 0x00 // device type
data[1] = 0xb0 // page code
binary.BigEndian.PutUint16(data[2:4], 0x003c) // page length = 60
// Byte 5: WSNZ (Write Same No Zero) = 0 — we accept zero-length WRITE SAME
// Bytes 6-7: Maximum compare and write length = 0 (not supported)
// Bytes 8-9: Optimal transfer length granularity = 1 block
binary.BigEndian.PutUint16(data[6:8], 1)
// Bytes 8-11: Maximum transfer length = 0 (no limit from our side)
// Bytes 12-15: Optimal transfer length = 128 blocks (512KB)
binary.BigEndian.PutUint32(data[12:16], 128)
// Bytes 20-23: Maximum prefetch length = 0
// Bytes 24-27: Maximum UNMAP LBA count
binary.BigEndian.PutUint32(data[24:28], uint32(maxWS))
// Bytes 28-31: Maximum UNMAP block descriptor count
binary.BigEndian.PutUint32(data[28:32], 256)
// Bytes 32-35: Optimal UNMAP granularity = 1 block
binary.BigEndian.PutUint32(data[32:36], 1)
// Bytes 36-39: UNMAP granularity alignment (bit 31 = UGAVALID)
// Not set — no alignment requirement.
// Bytes 40-47: Maximum WRITE SAME length (uint64)
binary.BigEndian.PutUint64(data[40:48], maxWS)
if int(allocLen) < len(data) {
data = data[:allocLen]
}
return SCSIResult{Status: SCSIStatusGood, Data: data}
}
// inquiryVPDB2 returns the Logical Block Provisioning VPD page (0xB2).
// Tells the kernel what thin provisioning features we support.
func (h *SCSIHandler) inquiryVPDB2(allocLen uint16) SCSIResult {
data := make([]byte, 8)
data[0] = 0x00 // device type
data[1] = 0xb2 // page code
binary.BigEndian.PutUint16(data[2:4], 0x0004) // page length = 4
// Byte 5: Provisioning type and flags
// Bits 7: Threshold exponent = 0
data[4] = 0x00
// Byte 5 (offset 5 in page):
// Bit 7: DP (descriptor present) = 0
// Bits 2-0: Provisioning type = 0x02 (thin provisioned)
data[5] = 0x02
// Byte 6: Provisioning group descriptor flags
// Bit 7: LBPU (Logical Block Provisioning UNMAP) = 1 — we support UNMAP
// Bit 6: LBPWS (LBP Write Same) = 1 — we support WRITE SAME with UNMAP bit
// Bit 5: LBPWS10 = 0 — we don't support WRITE SAME(10)
data[6] = 0xC0 // LBPU=1, LBPWS=1
if int(allocLen) < len(data) {
data = data[:allocLen]
}
return SCSIResult{Status: SCSIStatusGood, Data: data}
}
func (h *SCSIHandler) readCapacity10() SCSIResult {
blockSize := h.dev.BlockSize()
totalBlocks := h.dev.VolumeSize() / uint64(blockSize)
@@ -282,6 +371,29 @@ func (h *SCSIHandler) modeSense6(cdb [16]byte) SCSIResult {
return SCSIResult{Status: SCSIStatusGood, Data: data}
}
func (h *SCSIHandler) modeSense10(cdb [16]byte) SCSIResult {
// MODE SENSE(10) — 8-byte header, no mode pages
allocLen := binary.BigEndian.Uint16(cdb[7:9])
if allocLen == 0 {
allocLen = 8
}
data := make([]byte, 8)
// Mode data length = 6 (8-byte header minus the 2-byte length field)
binary.BigEndian.PutUint16(data[0:2], 6)
data[2] = 0x00 // Medium type: default
data[3] = 0x00 // Device-specific parameter (no write protect)
data[4] = 0x00 // Reserved (LONGLBA=0)
data[5] = 0x00 // Reserved
// Block descriptor length = 0 (bytes 6-7)
binary.BigEndian.PutUint16(data[6:8], 0)
if int(allocLen) < len(data) {
data = data[:allocLen]
}
return SCSIResult{Status: SCSIStatusGood, Data: data}
}
func (h *SCSIHandler) reportLuns(cdb [16]byte) SCSIResult {
allocLen := binary.BigEndian.Uint32(cdb[6:10])
if allocLen < 16 {
@@ -433,6 +545,80 @@ func (h *SCSIHandler) unmap(cdb [16]byte, dataOut []byte) SCSIResult {
return SCSIResult{Status: SCSIStatusGood}
}
// writeSame16 handles WRITE SAME(16) — SBC-4, Section 5.42.
// If UNMAP bit is set, the range is trimmed (zeroed). Otherwise the single
// logical block from dataOut is written repeatedly across the range.
// NDOB (No Data-Out Buffer) means zero the range with no data payload.
func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult {
lba := binary.BigEndian.Uint64(cdb[2:10])
numBlocks := binary.BigEndian.Uint32(cdb[10:14])
unmap := cdb[1]&0x08 != 0
ndob := cdb[1]&0x01 != 0
if numBlocks == 0 {
return SCSIResult{Status: SCSIStatusGood}
}
blockSize := h.dev.BlockSize()
totalBlocks := h.dev.VolumeSize() / uint64(blockSize)
if lba+uint64(numBlocks) > totalBlocks {
return illegalRequest(ASCLBAOutOfRange, ASCQLuk)
}
// UNMAP or NDOB: zero/trim the range.
if unmap || ndob {
if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil {
return SCSIResult{
Status: SCSIStatusCheckCond,
SenseKey: SenseMediumError,
SenseASC: 0x0C,
SenseASCQ: 0x00,
}
}
return SCSIResult{Status: SCSIStatusGood}
}
// Normal WRITE SAME: replicate the single block across the range.
if uint32(len(dataOut)) < blockSize {
return illegalRequest(ASCInvalidFieldInCDB, ASCQLuk)
}
pattern := dataOut[:blockSize]
// Check if the pattern is all zeros — use Trim for efficiency.
allZero := true
for _, b := range pattern {
if b != 0 {
allZero = false
break
}
}
if allZero {
if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil {
return SCSIResult{
Status: SCSIStatusCheckCond,
SenseKey: SenseMediumError,
SenseASC: 0x0C,
SenseASCQ: 0x00,
}
}
return SCSIResult{Status: SCSIStatusGood}
}
// Non-zero pattern: write each block individually.
for i := uint32(0); i < numBlocks; i++ {
if err := h.dev.WriteAt(lba+uint64(i), pattern); err != nil {
return SCSIResult{
Status: SCSIStatusCheckCond,
SenseKey: SenseMediumError,
SenseASC: 0x0C,
SenseASCQ: 0x00,
}
}
}
return SCSIResult{Status: SCSIStatusGood}
}
// BuildSenseData constructs a fixed-format sense data buffer (18 bytes).
func BuildSenseData(key, asc, ascq uint8) []byte {
data := make([]byte, 18)
weed/storage/blockvol/iscsi/scsi_test.go
+245 -3
View File
@@ -108,6 +108,15 @@ func TestSCSI(t *testing.T) {
{"read_error", testReadError},
{"write_error", testWriteError},
{"read_capacity_16_invalid_sa", testReadCapacity16InvalidSA},
{"write_same_16_unmap", testWriteSame16Unmap},
{"write_same_16_ndob", testWriteSame16NDOB},
{"write_same_16_pattern", testWriteSame16Pattern},
{"write_same_16_zeros", testWriteSame16Zeros},
{"write_same_16_oob", testWriteSame16OOB},
{"write_same_16_zero_blocks", testWriteSame16ZeroBlocks},
{"mode_sense_10", testModeSense10},
{"vpd_b0_block_limits", testVPDB0BlockLimits},
{"vpd_b2_logical_block_prov", testVPDB2LogicalBlockProv},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -185,9 +194,15 @@ func testInquiryVPDSupportedPages(t *testing.T) {
if r.Data[1] != 0x00 {
t.Fatal("wrong page code")
}
// Should list pages 0x00, 0x80, 0x83
if len(r.Data) < 7 || r.Data[4] != 0x00 || r.Data[5] != 0x80 || r.Data[6] != 0x83 {
t.Fatalf("supported pages: %v", r.Data)
// Should list pages 0x00, 0x80, 0x83, 0xB0, 0xB2
want := []byte{0x00, 0x80, 0x83, 0xb0, 0xb2}
if len(r.Data) < 4+len(want) {
t.Fatalf("supported pages too short: %v", r.Data)
}
for i, p := range want {
if r.Data[4+i] != p {
t.Fatalf("page[%d]: got %02x, want %02x (data=%v)", i, r.Data[4+i], p, r.Data)
}
}
}
@@ -679,6 +694,233 @@ func testWriteError(t *testing.T) {
}
}
// --- WRITE SAME(16) tests ---
func testWriteSame16Unmap(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
// Write data at LBA 10-14
for i := uint64(10); i < 15; i++ {
block := make([]byte, 4096)
block[0] = byte(i)
dev.blocks[i] = block
}
// WRITE SAME(16) with UNMAP bit: should trim LBA 10-14
var cdb [16]byte
cdb[0] = ScsiWriteSame16
cdb[1] = 0x08 // UNMAP bit
binary.BigEndian.PutUint64(cdb[2:10], 10)
binary.BigEndian.PutUint32(cdb[10:14], 5)
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
for i := uint64(10); i < 15; i++ {
if _, ok := dev.blocks[i]; ok {
t.Fatalf("block %d should be trimmed", i)
}
}
}
func testWriteSame16NDOB(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
dev.blocks[20] = make([]byte, 4096)
dev.blocks[20][0] = 0xFF
// WRITE SAME(16) with NDOB bit (no data-out buffer): zero the range
var cdb [16]byte
cdb[0] = ScsiWriteSame16
cdb[1] = 0x01 // NDOB bit
binary.BigEndian.PutUint64(cdb[2:10], 20)
binary.BigEndian.PutUint32(cdb[10:14], 1)
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
if _, ok := dev.blocks[20]; ok {
t.Fatal("block 20 should be trimmed")
}
}
func testWriteSame16Pattern(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
// Write a non-zero pattern across 3 blocks
pattern := make([]byte, 4096)
pattern[0] = 0xAB
pattern[4095] = 0xCD
var cdb [16]byte
cdb[0] = ScsiWriteSame16
// No UNMAP, no NDOB — normal write same
binary.BigEndian.PutUint64(cdb[2:10], 30)
binary.BigEndian.PutUint32(cdb[10:14], 3)
r := h.HandleCommand(cdb, pattern)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
for i := uint64(30); i < 33; i++ {
if dev.blocks[i][0] != 0xAB {
t.Fatalf("block %d[0]: got %02x, want AB", i, dev.blocks[i][0])
}
if dev.blocks[i][4095] != 0xCD {
t.Fatalf("block %d[4095]: got %02x, want CD", i, dev.blocks[i][4095])
}
}
}
func testWriteSame16Zeros(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
// Write data at LBA 40
dev.blocks[40] = make([]byte, 4096)
dev.blocks[40][0] = 0xFF
// WRITE SAME with all-zero pattern — should use Trim for efficiency
pattern := make([]byte, 4096)
var cdb [16]byte
cdb[0] = ScsiWriteSame16
binary.BigEndian.PutUint64(cdb[2:10], 40)
binary.BigEndian.PutUint32(cdb[10:14], 1)
r := h.HandleCommand(cdb, pattern)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
if _, ok := dev.blocks[40]; ok {
t.Fatal("block 40 should be trimmed (zero pattern)")
}
}
func testWriteSame16OOB(t *testing.T) {
dev := newMockDevice(10 * 4096) // 10 blocks
h := NewSCSIHandler(dev)
var cdb [16]byte
cdb[0] = ScsiWriteSame16
cdb[1] = 0x08 // UNMAP
binary.BigEndian.PutUint64(cdb[2:10], 8)
binary.BigEndian.PutUint32(cdb[10:14], 5) // 8+5 > 10
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusCheckCond {
t.Fatal("should fail for OOB")
}
if r.SenseASC != ASCLBAOutOfRange {
t.Fatalf("expected LBA_OUT_OF_RANGE, got ASC=%02x", r.SenseASC)
}
}
func testWriteSame16ZeroBlocks(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
var cdb [16]byte
cdb[0] = ScsiWriteSame16
binary.BigEndian.PutUint64(cdb[2:10], 0)
binary.BigEndian.PutUint32(cdb[10:14], 0) // 0 blocks = no-op
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatal("zero-block write same should succeed")
}
}
func testModeSense10(t *testing.T) {
dev := newMockDevice(1024 * 1024)
h := NewSCSIHandler(dev)
var cdb [16]byte
cdb[0] = ScsiModeSense10
binary.BigEndian.PutUint16(cdb[7:9], 255) // alloc length
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
if len(r.Data) != 8 {
t.Fatalf("mode sense(10) data: %d bytes, want 8", len(r.Data))
}
// Mode data length field (bytes 0-1) = 6
mdl := binary.BigEndian.Uint16(r.Data[0:2])
if mdl != 6 {
t.Fatalf("mode data length: %d, want 6", mdl)
}
// No write protect (byte 3, bit 7)
if r.Data[3]&0x80 != 0 {
t.Fatal("write protect set")
}
// Block descriptor length = 0 (bytes 6-7)
bdl := binary.BigEndian.Uint16(r.Data[6:8])
if bdl != 0 {
t.Fatalf("block descriptor length: %d, want 0", bdl)
}
}
func testVPDB0BlockLimits(t *testing.T) {
dev := newMockDevice(100 * 4096) // 100 blocks
h := NewSCSIHandler(dev)
var cdb [16]byte
cdb[0] = ScsiInquiry
cdb[1] = 0x01 // EVPD
cdb[2] = 0xb0 // Block Limits
binary.BigEndian.PutUint16(cdb[3:5], 255)
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
if r.Data[1] != 0xb0 {
t.Fatalf("page code: %02x, want B0", r.Data[1])
}
// Page length (bytes 2-3) = 60
pgLen := binary.BigEndian.Uint16(r.Data[2:4])
if pgLen != 60 {
t.Fatalf("page length: %d, want 60", pgLen)
}
// Max WRITE SAME length (bytes 40-47) should be 100 (total blocks)
maxWS := binary.BigEndian.Uint64(r.Data[40:48])
if maxWS != 100 {
t.Fatalf("max WRITE SAME length: %d, want 100", maxWS)
}
// Max UNMAP LBA count (bytes 24-27) should be 100
maxUnmap := binary.BigEndian.Uint32(r.Data[24:28])
if maxUnmap != 100 {
t.Fatalf("max UNMAP LBA count: %d, want 100", maxUnmap)
}
}
func testVPDB2LogicalBlockProv(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
var cdb [16]byte
cdb[0] = ScsiInquiry
cdb[1] = 0x01 // EVPD
cdb[2] = 0xb2 // Logical Block Provisioning
binary.BigEndian.PutUint16(cdb[3:5], 255)
r := h.HandleCommand(cdb, nil)
if r.Status != SCSIStatusGood {
t.Fatalf("status: %d", r.Status)
}
if r.Data[1] != 0xb2 {
t.Fatalf("page code: %02x, want B2", r.Data[1])
}
// Provisioning type (byte 5, bits 2:0) = 0x02 (thin)
if r.Data[5]&0x07 != 0x02 {
t.Fatalf("provisioning type: %02x, want 02", r.Data[5]&0x07)
}
// LBPU=1 (byte 6, bit 7), LBPWS=1 (byte 6, bit 6)
if r.Data[6]&0xC0 != 0xC0 {
t.Fatalf("LBPU/LBPWS flags: %02x, want C0", r.Data[6]&0xC0)
}
}
func testReadCapacity16InvalidSA(t *testing.T) {
dev := newMockDevice(100 * 4096)
h := NewSCSIHandler(dev)
weed/storage/blockvol/iscsi/session.go
+210 -102
View File
@@ -8,13 +8,18 @@ import (
"net"
"sync"
"sync/atomic"
"time"
)
var (
ErrSessionClosed = errors.New("iscsi: session closed")
ErrSessionClosed = errors.New("iscsi: session closed")
ErrCmdSNOutOfWindow = errors.New("iscsi: CmdSN out of window")
)
// maxPendingQueue limits the number of non-Data-Out PDUs that can be
// queued during a Data-Out collection phase. 2× the CmdSN window (32).
const maxPendingQueue = 64
// SessionState tracks the lifecycle of an iSCSI session.
type SessionState int
@@ -39,7 +44,7 @@ type Session struct {
// Sequence numbers
expCmdSN atomic.Uint32 // expected CmdSN from initiator
maxCmdSN atomic.Uint32 // max CmdSN we allow
statSN uint32 // target status sequence number
statSN uint32 // target status sequence number (txLoop only after login)
// Login state
negotiator *LoginNegotiator
@@ -53,10 +58,13 @@ type Session struct {
dataInWriter *DataInWriter
// PDU queue for commands received during Data-Out collection.
// The initiator pipelines commands; we may read a SCSI Command
// while waiting for Data-Out PDUs.
pending []*PDU
// TX goroutine channel for response PDUs.
// Buffered; txLoop reads and writes to conn.
respCh chan *PDU
txDone chan struct{}
// Shutdown
closed atomic.Bool
closeErr error
@@ -77,6 +85,8 @@ func NewSession(conn net.Conn, config TargetConfig, resolver TargetResolver, dev
resolver: resolver,
devices: devices,
negotiator: NewLoginNegotiator(config),
respCh: make(chan *PDU, 64),
txDone: make(chan struct{}),
logger: logger,
}
s.expCmdSN.Store(1)
@@ -85,21 +95,44 @@ func NewSession(conn net.Conn, config TargetConfig, resolver TargetResolver, dev
}
// HandleConnection processes PDUs until the connection is closed or an error occurs.
// Login phase runs inline (single goroutine). After login completes, a txLoop
// goroutine is started for pipelined response writing.
func (s *Session) HandleConnection() error {
defer s.close()
for !s.closed.Load() {
pdu, err := s.nextPDU()
// Login phase: handle login PDUs inline (no txLoop yet).
if err := s.loginPhase(); err != nil {
return err
}
if !s.loginDone {
// Connection closed during login phase.
return nil
}
// Start TX goroutine for full-feature phase.
go s.txLoop()
// RX loop: read PDUs and dispatch serially.
err := s.rxLoop()
// Shutdown: close respCh so txLoop exits, then wait for it.
close(s.respCh)
<-s.txDone
return err
}
// loginPhase handles login PDUs inline until login is complete or connection closes.
func (s *Session) loginPhase() error {
for !s.closed.Load() && !s.loginDone {
pdu, err := ReadPDU(s.conn)
if err != nil {
if s.closed.Load() {
return nil
}
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
if s.closed.Load() || errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return nil
}
return fmt.Errorf("read PDU: %w", err)
}
if err := s.dispatch(pdu); err != nil {
if s.closed.Load() {
return nil
@@ -110,6 +143,102 @@ func (s *Session) HandleConnection() error {
return nil
}
// rxLoop reads PDUs from the connection and dispatches them serially.
// Response PDUs are enqueued on respCh by handlers.
func (s *Session) rxLoop() error {
for !s.closed.Load() {
pdu, err := s.nextPDU()
if err != nil {
if s.closed.Load() || errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return nil
}
return fmt.Errorf("read PDU: %w", err)
}
if err := s.dispatch(pdu); err != nil {
if s.closed.Load() {
return nil
}
return fmt.Errorf("dispatch %s: %w", OpcodeName(pdu.Opcode()), err)
}
}
return nil
}
// txLoop reads response PDUs from respCh, assigns StatSN, and writes to conn.
// Runs as a goroutine during full-feature phase.
func (s *Session) txLoop() {
defer close(s.txDone)
for pdu := range s.respCh {
if pdu == nil {
continue
}
// Assign StatSN based on PDU type.
switch s.pduStatSNMode(pdu) {
case statSNAssign:
s.mu.Lock()
pdu.SetStatSN(s.statSN)
s.statSN++
pdu.SetExpCmdSN(s.expCmdSN.Load())
pdu.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
case statSNCopy:
s.mu.Lock()
pdu.SetStatSN(s.statSN)
pdu.SetExpCmdSN(s.expCmdSN.Load())
pdu.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
}
if err := WritePDU(s.conn, pdu); err != nil {
if !s.closed.Load() {
s.logger.Printf("txLoop write error: %v", err)
}
// Close connection so rxLoop exits, which lets HandleConnection
// close respCh, which lets the drain loop below finish.
s.closed.Store(true)
s.conn.Close()
for range s.respCh {
}
return
}
}
}
// statSNMode controls how txLoop handles StatSN for a PDU.
type statSNMode int
const (
statSNNone statSNMode = iota // don't touch (intermediate Data-In)
statSNAssign // assign current StatSN, increment
statSNCopy // assign current StatSN, do NOT increment (R2T)
)
// pduStatSNMode returns the StatSN handling mode for the given PDU.
func (s *Session) pduStatSNMode(pdu *PDU) statSNMode {
op := pdu.Opcode()
switch op {
case OpSCSIDataIn:
// Only the final Data-In PDU (with S-bit) gets StatSN.
if pdu.OpSpecific1()&FlagS != 0 {
return statSNAssign
}
return statSNNone
case OpR2T:
// R2T carries StatSN but does NOT increment it (RFC 7143).
return statSNCopy
default:
return statSNAssign
}
}
// enqueue sends a response PDU to the TX goroutine.
func (s *Session) enqueue(pdu *PDU) {
select {
case s.respCh <- pdu:
case <-s.txDone:
// TX loop exited, discard response.
}
}
// nextPDU returns the next PDU to process, draining the pending queue
// (populated during Data-Out collection) before reading from the connection.
func (s *Session) nextPDU() (*PDU, error) {
@@ -166,7 +295,7 @@ func (s *Session) handleLogin(pdu *PDU) error {
resp := s.negotiator.HandleLoginPDU(pdu, s.resolver)
// Set sequence numbers
// During login phase, StatSN is assigned inline (no txLoop yet).
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
@@ -184,9 +313,6 @@ func (s *Session) handleLogin(pdu *PDU) error {
s.negImmediateData = result.ImmediateData
s.negInitialR2T = result.InitialR2T
// Bind SCSI handler to the device for the target the initiator logged into.
// Discovery sessions have no TargetName — s.scsi stays nil, which is
// checked in handleSCSICmd.
if s.devices != nil && result.TargetName != "" {
dev := s.devices.LookupDevice(result.TargetName)
s.scsi = NewSCSIHandler(dev)
@@ -200,22 +326,19 @@ func (s *Session) handleLogin(pdu *PDU) error {
}
func (s *Session) handleText(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.loginDone {
return s.sendReject(pdu, 0x0b) // protocol error
}
// Gather discovery targets from resolver if it supports listing
var targets []DiscoveryTarget
if lister, ok := s.resolver.(TargetLister); ok {
targets = lister.ListTargets()
}
resp := HandleTextRequest(pdu, targets)
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
return WritePDU(s.conn, resp)
// ExpCmdSN/MaxCmdSN are set by txLoop via pduNeedsStatSN.
s.enqueue(resp)
return nil
}
func (s *Session) handleSCSICmd(pdu *PDU) error {
@@ -223,7 +346,6 @@ func (s *Session) handleSCSICmd(pdu *PDU) error {
return s.sendReject(pdu, 0x0b) // protocol error
}
// Discovery sessions have no SCSI handler — reject commands
if s.scsi == nil {
return s.sendReject(pdu, 0x04) // command not supported
}
@@ -232,15 +354,14 @@ func (s *Session) handleSCSICmd(pdu *PDU) error {
itt := pdu.InitiatorTaskTag()
flags := pdu.OpSpecific1()
// CmdSN validation for non-immediate commands (RFC 7143 section 4.2.2.1)
// CmdSN validation for non-immediate commands
if !pdu.Immediate() {
cmdSN := pdu.CmdSN()
expCmdSN := s.expCmdSN.Load()
maxCmdSN := s.maxCmdSN.Load()
// CmdSN is within window if ExpCmdSN <= CmdSN <= MaxCmdSN (serial arithmetic)
if !cmdSNInWindow(cmdSN, expCmdSN, maxCmdSN) {
s.logger.Printf("CmdSN %d out of window [%d, %d], dropping", cmdSN, expCmdSN, maxCmdSN)
return nil // silently drop per RFC 7143
return nil
}
s.advanceCmdSN()
}
@@ -254,10 +375,8 @@ func (s *Session) handleSCSICmd(pdu *PDU) error {
if isWrite && expectedLen > 0 {
collector := NewDataOutCollector(expectedLen)
// Immediate data — enforce negotiated ImmediateData flag
if len(pdu.DataSegment) > 0 {
if !s.negImmediateData {
// ImmediateData=No but initiator sent data — reject
return s.sendCheckCondition(itt, SenseIllegalRequest, ASCInvalidFieldInCDB, ASCQLuk)
}
if err := collector.AddImmediateData(pdu.DataSegment); err != nil {
@@ -265,7 +384,6 @@ func (s *Session) handleSCSICmd(pdu *PDU) error {
}
}
// If more data needed, send R2T and collect Data-Out PDUs
if !collector.Done() {
if err := s.collectDataOut(collector, itt); err != nil {
return err
@@ -278,52 +396,51 @@ func (s *Session) handleSCSICmd(pdu *PDU) error {
// Execute SCSI command
result := s.scsi.HandleCommand(cdb, dataOut)
// Send response
s.mu.Lock()
expCmdSN := s.expCmdSN.Load()
maxCmdSN := s.maxCmdSN.Load()
s.mu.Unlock()
if isRead && result.Status == SCSIStatusGood && len(result.Data) > 0 {
// Send Data-In PDUs
s.mu.Lock()
_, err := s.dataInWriter.WriteDataIn(s.conn, result.Data, itt, expCmdSN, maxCmdSN, &s.statSN)
s.mu.Unlock()
return err
// Build Data-In PDUs and enqueue them all.
pdus := s.dataInWriter.BuildDataInPDUs(result.Data, itt, s.expCmdSN.Load(), s.maxCmdSN.Load())
for _, p := range pdus {
s.enqueue(p)
}
return nil
}
// Send SCSI Response
s.mu.Lock()
err := SendSCSIResponse(s.conn, result, itt, &s.statSN, expCmdSN, maxCmdSN)
s.mu.Unlock()
return err
// Build SCSI Response PDU and enqueue.
resp := BuildSCSIResponse(result, itt, s.expCmdSN.Load(), s.maxCmdSN.Load())
s.enqueue(resp)
return nil
}
func (s *Session) collectDataOut(collector *DataOutCollector, itt uint32) error {
var r2tSN uint32
ttt := itt // use ITT as TTT for simplicity
ttt := itt
// Clear any read deadline on exit (success or error).
defer s.conn.SetReadDeadline(time.Time{})
for !collector.Done() {
// Send R2T
s.mu.Lock()
// Build R2T and enqueue (txLoop assigns StatSN before writing).
r2t := BuildR2T(itt, ttt, r2tSN, s.totalReceived(collector), collector.Remaining(),
s.statSN, s.expCmdSN.Load(), s.maxCmdSN.Load())
s.mu.Unlock()
s.expCmdSN.Load(), s.maxCmdSN.Load())
if err := WritePDU(s.conn, r2t); err != nil {
return err
}
s.enqueue(r2t)
r2tSN++
// Set read deadline for Data-Out collection.
if s.config.DataOutTimeout > 0 {
s.conn.SetReadDeadline(time.Now().Add(s.config.DataOutTimeout))
}
// Read Data-Out PDUs until F-bit.
// The initiator may pipeline other commands; queue them for later.
for {
doPDU, err := ReadPDU(s.conn)
if err != nil {
return err
}
if doPDU.Opcode() != OpSCSIDataOut {
// Not our Data-Out — queue for later dispatch
if len(s.pending) >= maxPendingQueue {
return fmt.Errorf("pending queue overflow (%d PDUs)", maxPendingQueue)
}
s.pending = append(s.pending, doPDU)
continue
}
@@ -343,69 +460,59 @@ func (s *Session) totalReceived(c *DataOutCollector) uint32 {
}
func (s *Session) handleNOPOut(pdu *PDU) error {
if !s.loginDone {
return s.sendReject(pdu, 0x0b) // protocol error
}
resp := &PDU{}
resp.SetOpcode(OpNOPIn)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.SetTargetTransferTag(0xFFFFFFFF)
s.mu.Lock()
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
// Echo back data if present
if len(pdu.DataSegment) > 0 {
resp.DataSegment = pdu.DataSegment
}
return WritePDU(s.conn, resp)
s.enqueue(resp)
return nil
}
func (s *Session) handleLogout(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
s.state = SessionLogout
if !s.loginDone {
return s.sendReject(pdu, 0x0b) // protocol error
}
resp := &PDU{}
resp.SetOpcode(OpLogoutResp)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.BHS[2] = 0x00 // response: connection/session closed successfully
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
if err := WritePDU(s.conn, resp); err != nil {
return err
}
s.enqueue(resp)
// Signal HandleConnection to exit
// Give txLoop a moment to write the response before closing.
// We signal close after enqueue so the logout response is sent.
s.mu.Lock()
s.state = SessionLogout
s.mu.Unlock()
s.closed.Store(true)
s.conn.Close()
return nil
}
func (s *Session) handleTaskMgmt(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.loginDone {
return s.sendReject(pdu, 0x0b) // protocol error
}
// Simplified: always respond with "function complete"
resp := &PDU{}
resp.SetOpcode(OpSCSITaskResp)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.BHS[2] = 0x00 // function complete
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
return WritePDU(s.conn, resp)
s.enqueue(resp)
return nil
}
func (s *Session) advanceCmdSN() {
@@ -416,7 +523,6 @@ func (s *Session) advanceCmdSN() {
// cmdSNInWindow checks if cmdSN is within [expCmdSN, maxCmdSN] using
// serial number arithmetic (RFC 7143 section 4.2.2.1). Handles uint32 wrap.
func cmdSNInWindow(cmdSN, expCmdSN, maxCmdSN uint32) bool {
// Serial comparison: a <= b means (b - a) < 2^31
return serialLE(expCmdSN, cmdSN) && serialLE(cmdSN, maxCmdSN)
}
@@ -430,18 +536,21 @@ func (s *Session) sendReject(origPDU *PDU, reason uint8) error {
resp.SetOpSpecific1(FlagF)
resp.BHS[2] = reason
resp.SetInitiatorTaskTag(0xFFFFFFFF)
s.mu.Lock()
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
// Include the rejected BHS in the data segment
resp.DataSegment = origPDU.BHS[:]
return WritePDU(s.conn, resp)
if s.loginDone {
s.enqueue(resp)
} else {
// During login phase, write inline (no txLoop).
s.mu.Lock()
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
return WritePDU(s.conn, resp)
}
return nil
}
func (s *Session) sendCheckCondition(itt uint32, senseKey, asc, ascq uint8) error {
@@ -451,9 +560,9 @@ func (s *Session) sendCheckCondition(itt uint32, senseKey, asc, ascq uint8) erro
SenseASC: asc,
SenseASCQ: ascq,
}
s.mu.Lock()
defer s.mu.Unlock()
return SendSCSIResponse(s.conn, result, itt, &s.statSN, s.expCmdSN.Load(), s.maxCmdSN.Load())
resp := BuildSCSIResponse(result, itt, s.expCmdSN.Load(), s.maxCmdSN.Load())
s.enqueue(resp)
return nil
}
// TargetLister is an optional interface that TargetResolver can implement
@@ -463,7 +572,6 @@ type TargetLister interface {
}
// DeviceLookup resolves a target IQN to a BlockDevice.
// Used after login to bind the session to the correct volume.
type DeviceLookup interface {
LookupDevice(iqn string) BlockDevice
}
weed/storage/blockvol/iscsi/session_test.go
+775
View File
@@ -62,6 +62,26 @@ func TestSession(t *testing.T) {
{"task_mgmt", testTaskMgmt},
{"reject_scsi_before_login", testRejectSCSIBeforeLogin},
{"connection_close", testConnectionClose},
// Phase 3 CP2: RX/TX split tests.
{"rxtx_read_write_basic", testRXTXReadWriteBasic},
{"rxtx_pipelined_reads", testRXTXPipelinedReads},
{"rxtx_pipelined_writes", testRXTXPipelinedWrites},
{"rxtx_write_with_r2t", testRXTXWriteWithR2T},
{"rxtx_mixed_ops", testRXTXMixedOps},
{"rxtx_large_read_datain", testRXTXLargeReadDataIn},
{"rxtx_statsn_monotonic", testRXTXStatSNMonotonic},
{"rxtx_statsn_datain_no_increment", testRXTXStatSNDataInNoIncrement},
{"rxtx_statsn_mixed_types", testRXTXStatSNMixedTypes},
{"rxtx_shutdown_clean", testRXTXShutdownClean},
{"rxtx_conn_drop_reader", testRXTXConnDropReader},
{"rxtx_conn_drop_writer", testRXTXConnDropWriter},
// Phase 3 code review fixes.
{"rxtx_r2t_statsn_fresh", testRXTXR2TStatSNFresh},
{"rxtx_tx_error_exits_clean", testRXTXTxErrorExitsClean},
{"rxtx_login_phase_reject", testRXTXLoginPhaseReject},
// Phase 3 bug fixes (P3-BUG-2, P3-BUG-3).
{"rxtx_pending_queue_overflow", testRXTXPendingQueueOverflow},
{"rxtx_dataout_timeout", testRXTXDataOutTimeout},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -77,12 +97,19 @@ type sessionTestEnv struct {
}
func setupSession(t *testing.T) *sessionTestEnv {
return setupSessionWithConfig(t, nil)
}
func setupSessionWithConfig(t *testing.T, cfgFn func(*TargetConfig)) *sessionTestEnv {
t.Helper()
client, server := net.Pipe()
dev := newMockDevice(1024 * 4096) // 1024 blocks
config := DefaultTargetConfig()
config.TargetName = testTargetName
if cfgFn != nil {
cfgFn(&config)
}
resolver := newTestResolverWithDevice(dev)
logger := log.New(io.Discard, "", 0)
@@ -362,3 +389,751 @@ func testConnectionClose(t *testing.T) {
t.Fatal("session did not detect close")
}
}
// --- Phase 3 CP2: RX/TX split tests ---
func sendSCSIRead(t *testing.T, conn net.Conn, lba uint32, blocks uint16, itt uint32, cmdSN uint32) {
t.Helper()
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagR)
cmd.SetInitiatorTaskTag(itt)
cmd.SetExpectedDataTransferLength(uint32(blocks) * 4096)
cmd.SetCmdSN(cmdSN)
var cdb [16]byte
cdb[0] = ScsiRead10
binary.BigEndian.PutUint32(cdb[2:6], lba)
binary.BigEndian.PutUint16(cdb[7:9], blocks)
cmd.SetCDB(cdb)
if err := WritePDU(conn, cmd); err != nil {
t.Fatalf("sendSCSIRead: %v", err)
}
}
func sendSCSIWriteImmediate(t *testing.T, conn net.Conn, lba uint32, data []byte, itt uint32, cmdSN uint32) {
t.Helper()
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(itt)
cmd.SetExpectedDataTransferLength(uint32(len(data)))
cmd.SetCmdSN(cmdSN)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], lba)
binary.BigEndian.PutUint16(cdb[7:9], uint16(len(data)/4096))
cmd.SetCDB(cdb)
cmd.DataSegment = data
if err := WritePDU(conn, cmd); err != nil {
t.Fatalf("sendSCSIWriteImmediate: %v", err)
}
}
func testRXTXReadWriteBasic(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Write then read via the RX/TX split path.
data := bytes.Repeat([]byte{0xBB}, 4096)
sendSCSIWriteImmediate(t, env.clientConn, 0, data, 1, 2)
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpSCSIResp || resp.SCSIStatus() != SCSIStatusGood {
t.Fatalf("write resp: opcode=%s status=%d", OpcodeName(resp.Opcode()), resp.SCSIStatus())
}
sendSCSIRead(t, env.clientConn, 0, 1, 2, 3)
resp, err = ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpSCSIDataIn {
t.Fatalf("expected DataIn, got %s", OpcodeName(resp.Opcode()))
}
if !bytes.Equal(resp.DataSegment, data) {
t.Error("read data mismatch")
}
}
func testRXTXPipelinedReads(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Send 4 reads back-to-back.
for i := uint32(0); i < 4; i++ {
sendSCSIRead(t, env.clientConn, i, 1, i+1, i+2)
}
// Receive all 4 responses.
for i := 0; i < 4; i++ {
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("read %d: %v", i, err)
}
if resp.Opcode() != OpSCSIDataIn {
t.Fatalf("read %d: expected DataIn, got %s", i, OpcodeName(resp.Opcode()))
}
}
}
func testRXTXPipelinedWrites(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Send 8 writes back-to-back with immediate data.
for i := uint32(0); i < 8; i++ {
data := bytes.Repeat([]byte{byte(0xA0 + i)}, 4096)
sendSCSIWriteImmediate(t, env.clientConn, i, data, i+1, i+2)
}
// Receive all 8 responses.
for i := 0; i < 8; i++ {
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatalf("write %d: %v", i, err)
}
if resp.Opcode() != OpSCSIResp {
t.Fatalf("write %d: expected SCSIResp, got %s", i, OpcodeName(resp.Opcode()))
}
if resp.SCSIStatus() != SCSIStatusGood {
t.Fatalf("write %d: status=%d", i, resp.SCSIStatus())
}
}
}
func testRXTXWriteWithR2T(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Write without immediate data — should trigger R2T + Data-Out.
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW) // no immediate data
cmd.SetInitiatorTaskTag(1)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 10)
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd.SetCDB(cdb)
// No DataSegment = no immediate data
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Expect R2T from target.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// Send Data-Out.
dataOut := &PDU{}
dataOut.SetOpcode(OpSCSIDataOut)
dataOut.SetOpSpecific1(FlagF) // Final
dataOut.SetInitiatorTaskTag(1)
dataOut.SetTargetTransferTag(r2t.TargetTransferTag())
dataOut.SetDataSN(0)
dataOut.SetBufferOffset(0)
dataOut.DataSegment = bytes.Repeat([]byte{0xCC}, 4096)
if err := WritePDU(env.clientConn, dataOut); err != nil {
t.Fatal(err)
}
// Expect SCSI Response.
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpSCSIResp {
t.Fatalf("expected SCSIResp, got %s", OpcodeName(resp.Opcode()))
}
if resp.SCSIStatus() != SCSIStatusGood {
t.Fatalf("status=%d", resp.SCSIStatus())
}
}
func testRXTXMixedOps(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Interleave: WRITE, READ, NOP, READ, WRITE
cmdSN := uint32(2)
// WRITE
sendSCSIWriteImmediate(t, env.clientConn, 0, bytes.Repeat([]byte{0x11}, 4096), 1, cmdSN)
cmdSN++
resp, _ := ReadPDU(env.clientConn)
if resp.Opcode() != OpSCSIResp {
t.Fatalf("write: got %s", OpcodeName(resp.Opcode()))
}
// READ
sendSCSIRead(t, env.clientConn, 0, 1, 2, cmdSN)
cmdSN++
resp, _ = ReadPDU(env.clientConn)
if resp.Opcode() != OpSCSIDataIn {
t.Fatalf("read: got %s", OpcodeName(resp.Opcode()))
}
// NOP
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(3)
nop.SetImmediate(true)
WritePDU(env.clientConn, nop)
resp, _ = ReadPDU(env.clientConn)
if resp.Opcode() != OpNOPIn {
t.Fatalf("nop: got %s", OpcodeName(resp.Opcode()))
}
// TEST_UNIT_READY (CDB[0] = 0x00)
tuCmd := &PDU{}
tuCmd.SetOpcode(OpSCSICmd)
tuCmd.SetOpSpecific1(FlagF)
tuCmd.SetInitiatorTaskTag(4)
tuCmd.SetCmdSN(cmdSN)
var tuCDB [16]byte
tuCmd.SetCDB(tuCDB) // all zeros = TEST_UNIT_READY
WritePDU(env.clientConn, tuCmd)
resp, _ = ReadPDU(env.clientConn)
if resp.Opcode() != OpSCSIResp {
t.Fatalf("tur: got %s", OpcodeName(resp.Opcode()))
}
}
func testRXTXLargeReadDataIn(t *testing.T) {
env := setupSessionWithConfig(t, func(c *TargetConfig) {
c.MaxRecvDataSegmentLength = 4096 // small: force multi-PDU
})
doLogin(t, env.clientConn)
// Read 4 blocks = 16384 bytes. With MaxRecvDataSegmentLength=4096,
// this should produce 4 Data-In PDUs.
sendSCSIRead(t, env.clientConn, 0, 4, 1, 2)
var pdus []*PDU
for {
p, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
pdus = append(pdus, p)
if p.OpSpecific1()&FlagS != 0 {
break // final PDU with S-bit
}
}
if len(pdus) < 2 {
t.Fatalf("expected >= 2 Data-In PDUs, got %d", len(pdus))
}
// Reconstruct data.
totalData := make([]byte, 0)
for _, p := range pdus {
totalData = append(totalData, p.DataSegment...)
}
if len(totalData) != 4*4096 {
t.Fatalf("total data length: %d, want %d", len(totalData), 4*4096)
}
}
func testRXTXStatSNMonotonic(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
var lastStatSN uint32
cmdSN := uint32(2)
for i := 0; i < 10; i++ {
sendSCSIRead(t, env.clientConn, 0, 1, uint32(i+1), cmdSN)
cmdSN++
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
statSN := resp.StatSN()
if i > 0 && statSN != lastStatSN+1 {
t.Fatalf("StatSN not monotonic: prev=%d, curr=%d", lastStatSN, statSN)
}
lastStatSN = statSN
}
}
func testRXTXStatSNDataInNoIncrement(t *testing.T) {
env := setupSessionWithConfig(t, func(c *TargetConfig) {
c.MaxRecvDataSegmentLength = 4096 // small: force multi-PDU
})
doLogin(t, env.clientConn)
// Read 4 blocks to get multi-PDU Data-In.
sendSCSIRead(t, env.clientConn, 0, 4, 1, 2)
var pdus []*PDU
for {
p, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
pdus = append(pdus, p)
if p.OpSpecific1()&FlagS != 0 {
break
}
}
if len(pdus) < 2 {
t.Fatalf("expected >= 2 Data-In PDUs, got %d", len(pdus))
}
// Intermediate PDUs should NOT have StatSN incremented.
// Only the final PDU (with S-bit) should have StatSN.
finalPDU := pdus[len(pdus)-1]
if finalPDU.OpSpecific1()&FlagS == 0 {
t.Fatal("last PDU missing S-bit")
}
// StatSN on final PDU should be set.
statSN := finalPDU.StatSN()
if statSN == 0 {
// It's actually valid for StatSN to be any value (depends on login StatSN).
// Just verify it's set on the final PDU.
}
_ = statSN
}
func testRXTXStatSNMixedTypes(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
cmdSN := uint32(2)
var statSNs []uint32
// NOP
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(1)
nop.SetImmediate(true)
WritePDU(env.clientConn, nop)
resp, _ := ReadPDU(env.clientConn)
statSNs = append(statSNs, resp.StatSN())
// SCSI READ
sendSCSIRead(t, env.clientConn, 0, 1, 2, cmdSN)
cmdSN++
resp, _ = ReadPDU(env.clientConn)
statSNs = append(statSNs, resp.StatSN())
// Logout
logout := &PDU{}
logout.SetOpcode(OpLogoutReq)
logout.SetOpSpecific1(FlagF)
logout.SetInitiatorTaskTag(3)
logout.SetCmdSN(cmdSN)
WritePDU(env.clientConn, logout)
resp, _ = ReadPDU(env.clientConn)
statSNs = append(statSNs, resp.StatSN())
// All StatSNs should be strictly increasing.
for i := 1; i < len(statSNs); i++ {
if statSNs[i] != statSNs[i-1]+1 {
t.Errorf("StatSN not monotonic at %d: %v", i, statSNs)
}
}
}
func testRXTXShutdownClean(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Send logout.
logout := &PDU{}
logout.SetOpcode(OpLogoutReq)
logout.SetOpSpecific1(FlagF)
logout.SetInitiatorTaskTag(1)
logout.SetCmdSN(2)
WritePDU(env.clientConn, logout)
// Read logout response.
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpLogoutResp {
t.Fatalf("expected LogoutResp, got %s", OpcodeName(resp.Opcode()))
}
// HandleConnection should exit cleanly.
select {
case err := <-env.done:
if err != nil {
t.Fatalf("error after logout: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("session did not exit after logout")
}
}
func testRXTXConnDropReader(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Close client conn — reader should detect EOF.
env.clientConn.Close()
select {
case err := <-env.done:
if err != nil {
t.Fatalf("error on conn drop: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("session did not exit after conn drop")
}
}
func testRXTXConnDropWriter(t *testing.T) {
client, server := net.Pipe()
dev := newMockDevice(1024 * 4096)
config := DefaultTargetConfig()
config.TargetName = testTargetName
resolver := newTestResolverWithDevice(dev)
logger := log.New(io.Discard, "", 0)
sess := NewSession(server, config, resolver, resolver, logger)
done := make(chan error, 1)
go func() {
done <- sess.HandleConnection()
}()
doLogin(t, client)
// Close the server side (writer side for txLoop).
server.Close()
// Session should exit.
select {
case <-done:
// Good — session exited.
case <-time.After(2 * time.Second):
t.Fatal("session did not exit after server-side close")
}
client.Close()
}
// testRXTXR2TStatSNFresh verifies that R2T PDUs carry a fresh StatSN
// (assigned by txLoop) rather than a stale value baked in at build time.
func testRXTXR2TStatSNFresh(t *testing.T) {
env := setupSession(t)
doLogin(t, env.clientConn)
// Send a NOP-Out (immediate) — its NOP-In response will consume one StatSN.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(0x1000)
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
t.Fatal(err)
}
// Read NOP-In response.
nopResp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if nopResp.Opcode() != OpNOPIn {
t.Fatalf("expected NOP-In, got %s", OpcodeName(nopResp.Opcode()))
}
nopStatSN := nopResp.StatSN()
// Now send a WRITE command without immediate data to trigger R2T.
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(1)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0)
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Read R2T — its StatSN should be nopStatSN+1 (fresh, not stale).
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// R2T uses statSNCopy: same as current StatSN (which is nopStatSN+1) without increment.
expectedStatSN := nopStatSN + 1
if r2t.StatSN() != expectedStatSN {
t.Fatalf("R2T StatSN: got %d, want %d (NOP-In was %d)", r2t.StatSN(), expectedStatSN, nopStatSN)
}
// Complete the write by sending Data-Out.
dataOut := &PDU{}
dataOut.SetOpcode(OpSCSIDataOut)
dataOut.SetOpSpecific1(FlagF)
dataOut.SetInitiatorTaskTag(1)
dataOut.SetTargetTransferTag(r2t.TargetTransferTag())
dataOut.SetDataSN(0)
dataOut.SetBufferOffset(0)
dataOut.DataSegment = bytes.Repeat([]byte{0xDD}, 4096)
if err := WritePDU(env.clientConn, dataOut); err != nil {
t.Fatal(err)
}
// Read SCSI response — its StatSN should be nopStatSN+1 (R2T didn't increment).
resp, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpSCSIResp {
t.Fatalf("expected SCSIResp, got %s", OpcodeName(resp.Opcode()))
}
if resp.StatSN() != expectedStatSN {
t.Fatalf("SCSI Resp StatSN: got %d, want %d", resp.StatSN(), expectedStatSN)
}
}
// testRXTXTxErrorExitsClean verifies that when txLoop encounters a write error,
// HandleConnection exits cleanly without hanging.
func testRXTXTxErrorExitsClean(t *testing.T) {
client, server := net.Pipe()
dev := newMockDevice(1024 * 4096)
config := DefaultTargetConfig()
config.TargetName = testTargetName
resolver := newTestResolverWithDevice(dev)
logger := log.New(io.Discard, "", 0)
sess := NewSession(server, config, resolver, resolver, logger)
done := make(chan error, 1)
go func() {
done <- sess.HandleConnection()
}()
doLogin(t, client)
// Close the server side — this makes txLoop's WritePDU fail.
server.Close()
// Send something from client side so rxLoop dispatches and enqueues a response.
// The enqueue should not block forever thanks to txDone select.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(1)
nop.SetImmediate(true)
// Write may fail since server is closed; that's fine.
WritePDU(client, nop)
// HandleConnection should exit cleanly without hanging.
select {
case <-done:
// Good.
case <-time.After(3 * time.Second):
t.Fatal("HandleConnection did not exit after tx write error")
}
// Verify txDone is closed (no goroutine leak).
select {
case <-sess.txDone:
// Good — txLoop exited.
default:
t.Fatal("txDone not closed — txLoop may be leaked")
}
client.Close()
}
// testRXTXLoginPhaseReject verifies that handlers reject PDUs sent before
// login completes and that login can still succeed afterward.
func testRXTXLoginPhaseReject(t *testing.T) {
client, server := net.Pipe()
dev := newMockDevice(1024 * 4096)
config := DefaultTargetConfig()
config.TargetName = testTargetName
resolver := newTestResolverWithDevice(dev)
logger := log.New(io.Discard, "", 0)
sess := NewSession(server, config, resolver, resolver, logger)
done := make(chan error, 1)
go func() {
done <- sess.HandleConnection()
}()
defer func() {
sess.Close()
client.Close()
}()
// Send a TextReq before login — should get a Reject (inline, not buffered).
textParams := NewParams()
textParams.Set("SendTargets", "All")
textReq := makeTextReq(textParams)
textReq.SetCmdSN(1)
if err := WritePDU(client, textReq); err != nil {
t.Fatal(err)
}
resp, err := ReadPDU(client)
if err != nil {
t.Fatal(err)
}
if resp.Opcode() != OpReject {
t.Fatalf("expected Reject, got %s", OpcodeName(resp.Opcode()))
}
if resp.BHS[2] != 0x0b {
t.Fatalf("reject reason: got 0x%02x, want 0x0b", resp.BHS[2])
}
// Login should still work after the reject.
doLogin(t, client)
// Verify session is functional by sending a NOP.
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(0x2222)
nop.SetImmediate(true)
if err := WritePDU(client, nop); err != nil {
t.Fatal(err)
}
nopResp, err := ReadPDU(client)
if err != nil {
t.Fatal(err)
}
if nopResp.Opcode() != OpNOPIn {
t.Fatalf("expected NOP-In, got %s", OpcodeName(nopResp.Opcode()))
}
}
// testRXTXPendingQueueOverflow verifies that flooding non-Data-Out PDUs
// during Data-Out collection causes the session to close (not OOM).
func testRXTXPendingQueueOverflow(t *testing.T) {
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T
cfg.DataOutTimeout = 5 * time.Second
})
doLogin(t, env.clientConn)
// Start WRITE requiring R2T.
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0xAAAA)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0)
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Read R2T.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// Flood 100 NOP-Out PDUs during Data-Out collection.
for i := 0; i < 100; i++ {
nop := &PDU{}
nop.SetOpcode(OpNOPOut)
nop.SetOpSpecific1(FlagF)
nop.SetInitiatorTaskTag(uint32(0xB000 + i))
nop.SetImmediate(true)
if err := WritePDU(env.clientConn, nop); err != nil {
break // session closed
}
}
// Session should exit with an error (pending queue overflow).
select {
case <-env.done:
// Good — session exited.
case <-time.After(3 * time.Second):
t.Fatal("session did not exit after pending overflow")
}
}
// testRXTXDataOutTimeout verifies that collectDataOut times out
// when the initiator never sends Data-Out after R2T.
func testRXTXDataOutTimeout(t *testing.T) {
env := setupSessionWithConfig(t, func(cfg *TargetConfig) {
cfg.MaxRecvDataSegmentLength = 4096
cfg.FirstBurstLength = 0 // force R2T
cfg.DataOutTimeout = 500 * time.Millisecond
})
doLogin(t, env.clientConn)
// Send WRITE command requiring R2T (no immediate data).
cmd := &PDU{}
cmd.SetOpcode(OpSCSICmd)
cmd.SetOpSpecific1(FlagF | FlagW)
cmd.SetInitiatorTaskTag(0xBEEF)
cmd.SetExpectedDataTransferLength(4096)
cmd.SetCmdSN(2)
var cdb [16]byte
cdb[0] = ScsiWrite10
binary.BigEndian.PutUint32(cdb[2:6], 0)
binary.BigEndian.PutUint16(cdb[7:9], 1)
cmd.SetCDB(cdb)
if err := WritePDU(env.clientConn, cmd); err != nil {
t.Fatal(err)
}
// Read R2T.
r2t, err := ReadPDU(env.clientConn)
if err != nil {
t.Fatal(err)
}
if r2t.Opcode() != OpR2T {
t.Fatalf("expected R2T, got %s", OpcodeName(r2t.Opcode()))
}
// Do NOT send Data-Out. Session should time out and exit.
select {
case err := <-env.done:
t.Logf("session exited: %v", err)
case <-time.After(3 * time.Second):
t.Fatal("session did not time out — DataOutTimeout not working")
}
}
weed/storage/blockvol/iscsi/target.go
+7
View File
@@ -144,6 +144,13 @@ func (ts *TargetServer) handleConn(conn net.Conn) {
defer ts.sessions.Done()
defer conn.Close()
// Reject connections accepted during shutdown.
select {
case <-ts.closed:
return
default:
}
ts.logger.Printf("new connection from %s", conn.RemoteAddr())
sess := NewSession(conn, ts.config, ts, ts, ts.logger)
weed/storage/blockvol/iscsi/target_test.go
+45
View File
@@ -22,6 +22,8 @@ func TestTarget(t *testing.T) {
{"add_remove_volume", testAddRemoveVolume},
{"multiple_connections", testMultipleConnections},
{"connect_no_volumes", testConnectNoVolumes},
// Phase 3 bug fix: P3-BUG-6 shutdown race.
{"close_rejects_late_conn", testCloseRejectsLateConn},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -257,3 +259,46 @@ func testConnectNoVolumes(t *testing.T) {
t.Fatal("discovery login should succeed without volumes")
}
}
// testCloseRejectsLateConn verifies that handleConn returns immediately
// when ts.closed is already signaled. We call handleConn directly to
// deterministically exercise the closed-check guard.
func testCloseRejectsLateConn(t *testing.T) {
config := DefaultTargetConfig()
config.TargetName = testTargetName
logger := log.New(io.Discard, "", 0)
ts := NewTargetServer("127.0.0.1:0", config, logger)
ts.AddVolume(testTargetName, newMockDevice(256*4096))
// Close the target BEFORE passing a connection to handleConn.
// This simulates a conn accepted just before ln.Close() took effect.
close(ts.closed)
// Create a pipe to simulate a connection.
client, server := net.Pipe()
defer client.Close()
// Call handleConn directly. It must check ts.closed and return
// immediately without entering the session loop.
ts.sessions.Add(1)
done := make(chan struct{})
go func() {
ts.handleConn(server)
close(done)
}()
select {
case <-done:
// Good — handleConn returned promptly.
case <-time.After(2 * time.Second):
t.Fatal("handleConn did not return after ts.closed was signaled")
}
// The server-side conn should be closed by handleConn's defer.
// Verify by reading from client — should get EOF.
buf := make([]byte, 1)
_, err := client.Read(buf)
if err == nil {
t.Fatal("expected EOF on client after handleConn returned")
}
}
weed/storage/blockvol/qa_phase3_engine_test.go
+503
View File
@@ -0,0 +1,503 @@
package blockvol
import (
"errors"
"math"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
)
// TestQAPhase3Engine tests Phase 3 engine adversarial scenarios:
// adaptive group commit, sharded dirty map, WAL pressure.
func TestQAPhase3Engine(t *testing.T) {
tests := []struct {
name string
run func(t *testing.T)
}{
// QA-1.1: Group Commit Adversarial
{name: "gc_syncfunc_panic", run: testQAGCSyncFuncPanic},
{name: "gc_thundering_herd_1000", run: testQAGCThunderingHerd1000},
{name: "gc_lowwatermark_equals_maxbatch", run: testQAGCLowWatermarkEqualsMaxBatch},
// QA-1.2: Sharded Dirty Map Adversarial
{name: "dm_snapshot_during_heavy_write", run: testQADMSnapshotDuringHeavyWrite},
{name: "dm_put_delete_put_same_lba", run: testQADMPutDeletePutSameLBA},
{name: "dm_max_uint64_lba", run: testQADMMaxUint64LBA},
{name: "dm_non_power_of_2_shards", run: testQADMNonPowerOf2Shards},
{name: "dm_zero_shards", run: testQADMZeroShards},
{name: "dm_len_during_concurrent_writes", run: testQADMLenDuringConcurrentWrites},
// QA-1.3: WAL Pressure Adversarial
{name: "wal_pressure_flusher_slow", run: testQAWALPressureFlusherSlow},
{name: "wal_pressure_threshold_0", run: testQAWALPressureThreshold0},
{name: "wal_pressure_threshold_1", run: testQAWALPressureThreshold1},
{name: "wal_close_during_pressure_block", run: testQAWALCloseDuringPressureBlock},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.run(t)
})
}
}
// --- QA-1.1: Group Commit Adversarial ---
func testQAGCSyncFuncPanic(t *testing.T) {
// syncFunc panics: Run() goroutine crashes, pending waiters must not hang forever.
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
panic("simulated disk panic")
},
MaxDelay: 10 * time.Millisecond,
})
// Wrap Run() with panic recovery.
runDone := make(chan struct{})
go func() {
defer close(runDone)
defer func() { recover() }()
gc.Run()
}()
// Submit should either return an error or hang. Use a timeout.
result := make(chan error, 1)
go func() {
result <- gc.Submit()
}()
select {
case <-result:
// Got a result (error or nil) -- panic was recovered, or waiter unblocked.
case <-runDone:
// Run() exited due to panic. Submit will hang on channel read.
// This is expected behavior: panic in syncFunc is fatal.
// Test passes: no goroutine leak from Run(), panic was contained.
case <-time.After(3 * time.Second):
t.Fatal("syncFunc panic: Run() and Submit both hung for 3s")
}
}
func testQAGCThunderingHerd1000(t *testing.T) {
// 1000 goroutines Submit() simultaneously: all must return, batching should work.
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 50 * time.Millisecond,
MaxBatch: 64,
LowWatermark: 4,
})
go gc.Run()
defer gc.Stop()
const n = 1000
var wg sync.WaitGroup
errs := make([]error, n)
wg.Add(n)
for i := 0; i < n; i++ {
go func(idx int) {
defer wg.Done()
errs[idx] = gc.Submit()
}(i)
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("thundering herd: 1000 submits hung for 10s")
}
for i, err := range errs {
if err != nil {
t.Errorf("Submit[%d]: %v", i, err)
}
}
c := syncCalls.Load()
if c >= 1000 {
t.Errorf("syncCalls = %d, expected batching (< 1000)", c)
}
t.Logf("thundering herd: 1000 submits, %d fsyncs", c)
}
func testQAGCLowWatermarkEqualsMaxBatch(t *testing.T) {
// lowWatermark = maxBatch = 8: skipDelay always true when < 8 pending.
var syncCalls atomic.Uint64
gc := NewGroupCommitter(GroupCommitterConfig{
SyncFunc: func() error {
syncCalls.Add(1)
return nil
},
MaxDelay: 5 * time.Second, // should never wait this long
MaxBatch: 8,
LowWatermark: 8,
})
go gc.Run()
defer gc.Stop()
// Single serial submits: since pending < lowWatermark (8), skipDelay=true.
for i := 0; i < 5; i++ {
if err := gc.Submit(); err != nil {
t.Fatalf("Submit %d: %v", i, err)
}
}
// Each serial submit should have triggered its own fsync (no batching).
if c := syncCalls.Load(); c != 5 {
t.Errorf("syncCalls = %d, want 5 (skipDelay always true)", c)
}
}
// --- QA-1.2: Sharded Dirty Map Adversarial ---
func testQADMSnapshotDuringHeavyWrite(t *testing.T) {
dm := NewDirtyMap(256)
const numWriters = 16
const opsPerWriter = 1000
var wg sync.WaitGroup
// Writers: continuously Put entries.
wg.Add(numWriters)
for g := 0; g < numWriters; g++ {
go func(id int) {
defer wg.Done()
base := uint64(id * opsPerWriter)
for i := uint64(0); i < opsPerWriter; i++ {
dm.Put(base+i, (base+i)*10, base+i+1, 4096)
}
}(g)
}
// Concurrent snapshots.
wg.Add(1)
go func() {
defer wg.Done()
for round := 0; round < 20; round++ {
snap := dm.Snapshot()
// Each snapshot entry should have valid fields.
for _, e := range snap {
if e.Length != 4096 {
t.Errorf("snapshot entry LBA %d has length %d, want 4096", e.Lba, e.Length)
return
}
}
}
}()
wg.Wait()
// Test passes if no race/panic.
}
func testQADMPutDeletePutSameLBA(t *testing.T) {
dm := NewDirtyMap(256)
const iterations = 1000
for i := 0; i < iterations; i++ {
dm.Put(100, uint64(i)*10, uint64(i)+1, 4096)
dm.Delete(100)
dm.Put(100, uint64(i)*10+5, uint64(i)+2, 4096)
}
// Final state: should have the last Put's entry.
off, lsn, length, ok := dm.Get(100)
if !ok {
t.Fatal("Get(100) not found after Put-Delete-Put cycle")
}
expectedOff := uint64(999)*10 + 5
expectedLsn := uint64(999) + 2
if off != expectedOff {
t.Errorf("walOffset = %d, want %d", off, expectedOff)
}
if lsn != expectedLsn {
t.Errorf("lsn = %d, want %d", lsn, expectedLsn)
}
if length != 4096 {
t.Errorf("length = %d, want 4096", length)
}
}
func testQADMMaxUint64LBA(t *testing.T) {
dm := NewDirtyMap(256)
maxLBA := uint64(math.MaxUint64)
dm.Put(maxLBA, 12345, 1, 4096)
off, lsn, length, ok := dm.Get(maxLBA)
if !ok {
t.Fatal("Get(MaxUint64) not found")
}
if off != 12345 || lsn != 1 || length != 4096 {
t.Errorf("got (%d, %d, %d), want (12345, 1, 4096)", off, lsn, length)
}
dm.Delete(maxLBA)
_, _, _, ok = dm.Get(maxLBA)
if ok {
t.Error("Get(MaxUint64) should not be found after delete")
}
}
func testQADMNonPowerOf2Shards(t *testing.T) {
// Non-power-of-2: mask will be wrong (7-1=6=0b110), but should not panic.
// This tests robustness, not correctness of shard distribution.
dm := NewDirtyMap(7)
// Should not panic on basic operations.
for i := uint64(0); i < 100; i++ {
dm.Put(i, i*10, i+1, 4096)
}
// All entries should be retrievable (mask-based routing is deterministic).
for i := uint64(0); i < 100; i++ {
_, _, _, ok := dm.Get(i)
if !ok {
t.Errorf("Get(%d) not found with 7 shards", i)
}
}
if dm.Len() != 100 {
t.Errorf("Len() = %d, want 100", dm.Len())
}
}
func testQADMZeroShards(t *testing.T) {
// NewDirtyMap(0) should default to 1 shard (no panic, no divide-by-zero).
dm := NewDirtyMap(0)
dm.Put(42, 100, 1, 4096)
off, _, _, ok := dm.Get(42)
if !ok {
t.Fatal("Get(42) not found with 0 shards (defaulted to 1)")
}
if off != 100 {
t.Errorf("walOffset = %d, want 100", off)
}
}
func testQADMLenDuringConcurrentWrites(t *testing.T) {
dm := NewDirtyMap(256)
const numWriters = 32
const opsPerWriter = 100
var wg sync.WaitGroup
// Writers.
wg.Add(numWriters)
for g := 0; g < numWriters; g++ {
go func(id int) {
defer wg.Done()
base := uint64(id * opsPerWriter)
for i := uint64(0); i < opsPerWriter; i++ {
dm.Put(base+i, (base+i)*10, base+i+1, 4096)
}
}(g)
}
// Concurrent Len() reader.
wg.Add(1)
go func() {
defer wg.Done()
for round := 0; round < 100; round++ {
n := dm.Len()
if n < 0 {
t.Errorf("Len() = %d, must be non-negative", n)
return
}
}
}()
wg.Wait()
// Final Len should be numWriters * opsPerWriter (each LBA unique).
if dm.Len() != numWriters*opsPerWriter {
t.Errorf("final Len() = %d, want %d", dm.Len(), numWriters*opsPerWriter)
}
}
// --- QA-1.3: WAL Pressure Adversarial ---
func testQAWALPressureFlusherSlow(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "slow_flush.blockvol")
cfg := DefaultConfig()
cfg.WALPressureThreshold = 0.3
cfg.WALFullTimeout = 3 * time.Second
cfg.FlushInterval = 5 * time.Millisecond
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 64 * 1024,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
// Write many blocks. With a small WAL and active flusher, some may succeed
// and some may encounter WAL pressure. None should hang forever.
var succeeded, walFull int
for i := 0; i < 30; i++ {
err := v.WriteLBA(uint64(i%256), makeBlock(byte('A'+i%26)))
if err == nil {
succeeded++
} else if errors.Is(err, ErrWALFull) {
walFull++
} else {
t.Fatalf("WriteLBA(%d): unexpected error: %v", i, err)
}
}
t.Logf("flusher_slow: %d succeeded, %d WAL full", succeeded, walFull)
if succeeded == 0 {
t.Error("no writes succeeded -- flusher not draining?")
}
}
func testQAWALPressureThreshold0(t *testing.T) {
// threshold=0: urgent flush on every write. Should still work.
dir := t.TempDir()
path := filepath.Join(dir, "thresh0.blockvol")
cfg := DefaultConfig()
cfg.WALPressureThreshold = 0.0 // always urgent
cfg.FlushInterval = 5 * time.Millisecond
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 256 * 1024,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
// Every write triggers urgent flush. Should still succeed.
for i := 0; i < 20; i++ {
if err := v.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))); err != nil {
t.Fatalf("WriteLBA(%d): %v", i, err)
}
}
// Verify data readable.
for i := 0; i < 20; i++ {
data, err := v.ReadLBA(uint64(i), 4096)
if err != nil {
t.Fatalf("ReadLBA(%d): %v", i, err)
}
if data[0] != byte('A'+i%26) {
t.Errorf("LBA %d: first byte = %c, want %c", i, data[0], byte('A'+i%26))
}
}
}
func testQAWALPressureThreshold1(t *testing.T) {
// threshold=1.0: never urgent. WAL fills, eventually ErrWALFull.
dir := t.TempDir()
path := filepath.Join(dir, "thresh1.blockvol")
cfg := DefaultConfig()
cfg.WALPressureThreshold = 1.0 // never urgent
cfg.WALFullTimeout = 100 * time.Millisecond
cfg.FlushInterval = 1 * time.Hour // disable periodic flush
entrySize := uint64(walEntryHeaderSize + 4096)
walSize := entrySize * 4 // tiny: 4 entries
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: walSize,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
// Stop flusher to prevent any drainage.
v.flusher.Stop()
// Fill WAL.
var gotWALFull bool
for i := 0; i < 10; i++ {
err := v.WriteLBA(uint64(i%256), makeBlock(byte('X')))
if errors.Is(err, ErrWALFull) {
gotWALFull = true
break
} else if err != nil {
t.Fatalf("WriteLBA(%d): unexpected error: %v", i, err)
}
}
if !gotWALFull {
t.Error("expected ErrWALFull with threshold=1.0 and no flusher")
}
}
func testQAWALCloseDuringPressureBlock(t *testing.T) {
// Writer blocked on WAL full, Close() called -- writer should unblock.
dir := t.TempDir()
path := filepath.Join(dir, "close_pressure.blockvol")
cfg := DefaultConfig()
cfg.WALFullTimeout = 10 * time.Second // long timeout
cfg.FlushInterval = 1 * time.Hour // disable flusher
entrySize := uint64(walEntryHeaderSize + 4096)
walSize := entrySize * 2 // tiny: 2 entries
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: walSize,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
// Stop flusher.
v.flusher.Stop()
// Fill WAL.
for i := 0; i < 2; i++ {
if err := v.WriteLBA(uint64(i), makeBlock(byte('A'+i))); err != nil {
t.Fatalf("WriteLBA(%d): %v", i, err)
}
}
// Start a writer that will block on WAL full.
writeResult := make(chan error, 1)
go func() {
writeResult <- v.WriteLBA(2, makeBlock('Z'))
}()
// Give it time to block.
time.Sleep(50 * time.Millisecond)
// Close the volume -- should unblock the writer.
v.Close()
select {
case err := <-writeResult:
// Writer unblocked (error expected: ErrWALFull or closed fd).
if err == nil {
t.Error("expected error from WriteLBA after Close, got nil")
}
t.Logf("close_during_pressure: writer unblocked with: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("writer still blocked 5s after Close()")
}
}
weed/storage/blockvol/recovery_test.go
+32 -45
View File
@@ -29,9 +29,29 @@ func TestRecovery(t *testing.T) {
}
// simulateCrash closes the volume without syncing and returns the path.
// simulateCrash stops background goroutines and closes the fd without a clean
// shutdown. The caller is responsible for persisting the superblock AFTER
// stopping the flusher (to avoid concurrent superblock writes).
// For the common case, use simulateCrashWithSuper instead.
func simulateCrash(v *BlockVol) string {
path := v.Path()
v.groupCommit.Stop()
v.flusher.Stop()
v.fd.Close()
return path
}
// simulateCrashWithSuper is the safe default: stops background goroutines,
// writes the superblock with current WAL positions, then closes the fd.
func simulateCrashWithSuper(v *BlockVol) string {
path := v.Path()
v.groupCommit.Stop()
v.flusher.Stop()
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
v.fd.Close()
return path
}
@@ -70,14 +90,7 @@ func testRecoverOneEntry(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Update superblock with WAL state.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path := simulateCrash(v)
path := simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -117,13 +130,7 @@ func testRecoverManyEntries(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -158,6 +165,10 @@ func testRecoverTornWrite(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Stop background goroutines before manual superblock write.
v.groupCommit.Stop()
v.flusher.Stop()
// Save superblock with correct WAL state.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
@@ -172,7 +183,8 @@ func testRecoverTornWrite(t *testing.T) {
v.fd.WriteAt([]byte{0xFF, 0xFF}, corruptOff)
v.fd.Sync()
path := simulateCrash(v)
v.fd.Close()
path := v.Path()
v2, err := OpenBlockVol(path)
if err != nil {
@@ -240,14 +252,7 @@ func testRecoverAfterCheckpoint(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
// Update superblock.
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path := simulateCrash(v)
path := simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -279,13 +284,7 @@ func testRecoverIdempotent(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path := simulateCrash(v)
path := simulateCrashWithSuper(v)
// First recovery.
v2, err := OpenBlockVol(path)
@@ -347,13 +346,7 @@ func testRecoverWALFull(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path = simulateCrash(v)
path = simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
@@ -391,13 +384,7 @@ func testRecoverBarrierOnly(t *testing.T) {
t.Fatalf("SyncCache: %v", err)
}
v.super.WALHead = v.wal.LogicalHead()
v.super.WALTail = v.wal.LogicalTail()
v.fd.Seek(0, 0)
v.super.WriteTo(v.fd)
v.fd.Sync()
path := simulateCrash(v)
path := simulateCrashWithSuper(v)
v2, err := OpenBlockVol(path)
if err != nil {
weed/storage/blockvol/scripts/sw-block-attach.sh
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env bash
# sw-block-attach.sh — Discover and attach a SeaweedFS block volume via iSCSI
#
# Prerequisites:
# - Linux host with iscsiadm (open-iscsi) installed
# - Root access (for iscsiadm login)
# - SeaweedFS volume server running with --block.dir set
#
# Usage:
# sudo ./sw-block-attach.sh <target-addr> [volume-name]
#
# Examples:
# sudo ./sw-block-attach.sh 10.0.0.1:3260 # discover all, attach first
# sudo ./sw-block-attach.sh 10.0.0.1:3260 myvol # attach specific volume
# sudo ./sw-block-attach.sh 10.0.0.1:3260 --list # list available targets
# sudo ./sw-block-attach.sh --detach <target-iqn> <portal> # detach a volume
#
# The script will:
# 1. Run iscsiadm discovery against the target portal
# 2. Find the matching volume IQN (or list all)
# 3. Login (attach) the iSCSI target
# 4. Print the attached block device path
set -euo pipefail
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
log() { echo -e "${GREEN}[sw-block]${NC} $*"; }
warn() { echo -e "${YELLOW}[sw-block]${NC} $*"; }
fail() { echo -e "${RED}[sw-block]${NC} $*" >&2; exit 1; }
usage() {
cat <<'EOF'
Usage:
sw-block-attach.sh <target-addr:port> [volume-name] Attach a block volume
sw-block-attach.sh <target-addr:port> --list List available targets
sw-block-attach.sh --detach <target-iqn> <portal> Detach a block volume
sw-block-attach.sh --status Show active sessions
Examples:
sudo ./sw-block-attach.sh 10.0.0.1:3260
sudo ./sw-block-attach.sh 10.0.0.1:3260 myvol
sudo ./sw-block-attach.sh --detach iqn.2024-01.com.seaweedfs:vol.myvol 10.0.0.1:3260
EOF
exit 1
}
# -------------------------------------------------------
# Preflight
# -------------------------------------------------------
if [[ $EUID -ne 0 ]]; then
fail "This script must be run as root (for iscsiadm)"
fi
if ! command -v iscsiadm &>/dev/null; then
fail "iscsiadm not found. Install open-iscsi: apt install open-iscsi"
fi
if [[ $# -lt 1 ]]; then
usage
fi
# -------------------------------------------------------
# --status: show active sessions
# -------------------------------------------------------
if [[ "$1" == "--status" ]]; then
log "Active iSCSI sessions:"
iscsiadm -m session 2>/dev/null || echo " (none)"
exit 0
fi
# -------------------------------------------------------
# --detach: logout from a target
# -------------------------------------------------------
if [[ "$1" == "--detach" ]]; then
if [[ $# -lt 3 ]]; then
fail "Usage: sw-block-attach.sh --detach <target-iqn> <portal>"
fi
TARGET_IQN="$2"
PORTAL="$3"
log "Logging out from $TARGET_IQN at $PORTAL..."
iscsiadm -m node -T "$TARGET_IQN" -p "$PORTAL" --logout || {
fail "Logout failed"
}
iscsiadm -m node -T "$TARGET_IQN" -p "$PORTAL" -o delete 2>/dev/null || true
log "Detached successfully"
exit 0
fi
# -------------------------------------------------------
# Attach mode
# -------------------------------------------------------
PORTAL="$1"
VOL_NAME="${2:-}"
# Add default port if not specified.
if [[ "$PORTAL" != *:* ]]; then
PORTAL="${PORTAL}:3260"
fi
# -------------------------------------------------------
# Step 1: Discovery
# -------------------------------------------------------
log "Discovering targets at $PORTAL..."
DISCOVERY=$(iscsiadm -m discovery -t sendtargets -p "$PORTAL" 2>&1) || {
fail "Discovery failed: $DISCOVERY"
}
if [[ -z "$DISCOVERY" ]]; then
fail "No targets found at $PORTAL"
fi
# -------------------------------------------------------
# --list: just show targets
# -------------------------------------------------------
if [[ "$VOL_NAME" == "--list" ]]; then
log "Available targets:"
echo "$DISCOVERY"
exit 0
fi
# -------------------------------------------------------
# Step 2: Find target IQN
# -------------------------------------------------------
if [[ -n "$VOL_NAME" ]]; then
# Match by volume name suffix.
TARGET_IQN=$(echo "$DISCOVERY" | awk '{print $2}' | grep -F "$VOL_NAME" | head -1)
if [[ -z "$TARGET_IQN" ]]; then
fail "No target found matching '$VOL_NAME'. Available targets:"
echo "$DISCOVERY" >&2
fi
else
# Use the first target found.
TARGET_IQN=$(echo "$DISCOVERY" | awk '{print $2}' | head -1)
if [[ -z "$TARGET_IQN" ]]; then
fail "No targets found"
fi
fi
log "Target: $TARGET_IQN"
# -------------------------------------------------------
# Step 3: Check if already logged in
# -------------------------------------------------------
if iscsiadm -m session 2>/dev/null | grep -q "$TARGET_IQN"; then
warn "Already logged in to $TARGET_IQN"
# Show the device
DEV=$(iscsiadm -m session -P 3 2>/dev/null | grep -A 20 "$TARGET_IQN" | grep "Attached scsi disk" | awk '{print $4}' | head -1)
if [[ -n "$DEV" ]]; then
log "Device: /dev/$DEV"
fi
exit 0
fi
# -------------------------------------------------------
# Step 4: Login
# -------------------------------------------------------
log "Logging in to $TARGET_IQN..."
iscsiadm -m node -T "$TARGET_IQN" -p "$PORTAL" --login || {
fail "Login failed"
}
# Wait for device to appear.
sleep 2
# -------------------------------------------------------
# Step 5: Find attached device
# -------------------------------------------------------
DEV=$(iscsiadm -m session -P 3 2>/dev/null | grep -A 20 "$TARGET_IQN" | grep "Attached scsi disk" | awk '{print $4}' | head -1)
if [[ -z "$DEV" ]]; then
warn "Login succeeded but could not determine device path"
warn "Check: lsblk, or iscsiadm -m session -P 3"
else
log "Attached: /dev/$DEV"
echo ""
echo -e " Block device: ${GREEN}/dev/$DEV${NC}"
echo -e " Target IQN: $TARGET_IQN"
echo -e " Portal: $PORTAL"
echo ""
echo " To detach: sudo $0 --detach $TARGET_IQN $PORTAL"
fi
weed/storage/blockvol/wal_writer.go
+12
View File
@@ -186,6 +186,18 @@ func (w *WALWriter) LogicalTail() uint64 {
return t
}
// UsedFraction returns the fraction of WAL space currently in use (0.0 to 1.0).
func (w *WALWriter) UsedFraction() float64 {
w.mu.Lock()
u := w.used()
s := w.walSize
w.mu.Unlock()
if s == 0 {
return 0
}
return float64(u) / float64(s)
}
// Sync fsyncs the underlying file descriptor.
func (w *WALWriter) Sync() error {
return w.fd.Sync()
weed/storage/blockvol/wal_writer_test.go
+72
View File
@@ -17,6 +17,10 @@ func TestWALWriter(t *testing.T) {
{name: "wal_writer_full", run: testWALWriterFull},
{name: "wal_writer_advance_tail_frees_space", run: testWALWriterAdvanceTailFreesSpace},
{name: "wal_writer_fill_no_flusher", run: testWALWriterFillNoFlusher},
// Phase 3 Task 1.5: WAL UsedFraction.
{name: "wal_fraction_empty", run: testWALFractionEmpty},
{name: "wal_fraction_after_writes", run: testWALFractionAfterWrites},
{name: "wal_fraction_after_advance_tail", run: testWALFractionAfterAdvanceTail},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -210,6 +214,74 @@ func testWALWriterAdvanceTailFreesSpace(t *testing.T) {
}
}
// --- Phase 3 Task 1.5: WAL UsedFraction tests ---
func testWALFractionEmpty(t *testing.T) {
walOffset := uint64(SuperblockSize)
walSize := uint64(64 * 1024)
fd, cleanup := createTestWAL(t, walOffset, walSize)
defer cleanup()
w := NewWALWriter(fd, walOffset, walSize, 0, 0)
frac := w.UsedFraction()
if frac != 0 {
t.Errorf("UsedFraction on empty WAL = %f, want 0", frac)
}
}
func testWALFractionAfterWrites(t *testing.T) {
walOffset := uint64(SuperblockSize)
walSize := uint64(64 * 1024)
fd, cleanup := createTestWAL(t, walOffset, walSize)
defer cleanup()
w := NewWALWriter(fd, walOffset, walSize, 0, 0)
entry := &WALEntry{LSN: 1, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: make([]byte, 4096)}
if _, err := w.Append(entry); err != nil {
t.Fatalf("Append: %v", err)
}
frac := w.UsedFraction()
if frac <= 0 || frac > 1 {
t.Errorf("UsedFraction after write = %f, want (0, 1]", frac)
}
entrySize := float64(walEntryHeaderSize + 4096)
expected := entrySize / float64(walSize)
if diff := frac - expected; diff > 0.001 || diff < -0.001 {
t.Errorf("UsedFraction = %f, want ~%f", frac, expected)
}
}
func testWALFractionAfterAdvanceTail(t *testing.T) {
walOffset := uint64(SuperblockSize)
entrySize := uint64(walEntryHeaderSize + 4096)
walSize := entrySize * 4
fd, cleanup := createTestWAL(t, walOffset, walSize)
defer cleanup()
w := NewWALWriter(fd, walOffset, walSize, 0, 0)
// Write 2 entries.
for i := uint64(1); i <= 2; i++ {
entry := &WALEntry{LSN: i, Type: EntryTypeWrite, LBA: i, Length: 4096, Data: make([]byte, 4096)}
if _, err := w.Append(entry); err != nil {
t.Fatalf("Append: %v", err)
}
}
fracBefore := w.UsedFraction()
// Advance tail past first entry.
w.AdvanceTail(entrySize)
fracAfter := w.UsedFraction()
if fracAfter >= fracBefore {
t.Errorf("UsedFraction should decrease after AdvanceTail: before=%f, after=%f", fracBefore, fracAfter)
}
}
func testWALWriterFillNoFlusher(t *testing.T) {
// QA-001 regression: fill WAL without flusher (tail stays at 0).
// After wrap, head=0 and tail=0 must NOT be treated as "empty".
weed/storage/store_blockvol.go
+93
View File
@@ -0,0 +1,93 @@
package storage
import (
"fmt"
"sync"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
// BlockVolumeStore manages block volumes (iSCSI-backed).
// It is a standalone component held by VolumeServer, not embedded into Store,
// to keep the existing Store codebase unchanged.
type BlockVolumeStore struct {
mu sync.RWMutex
volumes map[string]*blockvol.BlockVol // keyed by volume file path
}
// NewBlockVolumeStore creates a new block volume manager.
func NewBlockVolumeStore() *BlockVolumeStore {
return &BlockVolumeStore{
volumes: make(map[string]*blockvol.BlockVol),
}
}
// AddBlockVolume opens and registers a block volume.
func (bs *BlockVolumeStore) AddBlockVolume(path string, cfgs ...blockvol.BlockVolConfig) (*blockvol.BlockVol, error) {
bs.mu.Lock()
defer bs.mu.Unlock()
if _, ok := bs.volumes[path]; ok {
return nil, fmt.Errorf("block volume already registered: %s", path)
}
vol, err := blockvol.OpenBlockVol(path, cfgs...)
if err != nil {
return nil, fmt.Errorf("open block volume %s: %w", path, err)
}
bs.volumes[path] = vol
glog.V(0).Infof("block volume registered: %s", path)
return vol, nil
}
// RemoveBlockVolume closes and unregisters a block volume.
func (bs *BlockVolumeStore) RemoveBlockVolume(path string) error {
bs.mu.Lock()
defer bs.mu.Unlock()
vol, ok := bs.volumes[path]
if !ok {
return fmt.Errorf("block volume not found: %s", path)
}
if err := vol.Close(); err != nil {
glog.Warningf("error closing block volume %s: %v", path, err)
}
delete(bs.volumes, path)
glog.V(0).Infof("block volume removed: %s", path)
return nil
}
// GetBlockVolume returns a registered block volume by path.
func (bs *BlockVolumeStore) GetBlockVolume(path string) (*blockvol.BlockVol, bool) {
bs.mu.RLock()
defer bs.mu.RUnlock()
vol, ok := bs.volumes[path]
return vol, ok
}
// ListBlockVolumes returns the paths of all registered block volumes.
func (bs *BlockVolumeStore) ListBlockVolumes() []string {
bs.mu.RLock()
defer bs.mu.RUnlock()
paths := make([]string, 0, len(bs.volumes))
for p := range bs.volumes {
paths = append(paths, p)
}
return paths
}
// Close closes all block volumes.
func (bs *BlockVolumeStore) Close() {
bs.mu.Lock()
defer bs.mu.Unlock()
for path, vol := range bs.volumes {
if err := vol.Close(); err != nil {
glog.Warningf("error closing block volume %s: %v", path, err)
}
delete(bs.volumes, path)
}
glog.V(0).Infof("all block volumes closed")
}
weed/storage/store_blockvol_test.go
+109
View File
@@ -0,0 +1,109 @@
package storage
import (
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
func createTestBlockVol(t *testing.T, dir, name string) string {
t.Helper()
path := filepath.Join(dir, name)
vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
VolumeSize: 1024 * 4096,
BlockSize: 4096,
ExtentSize: 65536,
WALSize: 1 << 20,
})
if err != nil {
t.Fatal(err)
}
vol.Close()
return path
}
func TestStoreAddBlockVolume(t *testing.T) {
dir := t.TempDir()
path := createTestBlockVol(t, dir, "vol1.blk")
bs := NewBlockVolumeStore()
defer bs.Close()
vol, err := bs.AddBlockVolume(path)
if err != nil {
t.Fatal(err)
}
if vol == nil {
t.Fatal("expected non-nil volume")
}
// Duplicate add should fail.
_, err = bs.AddBlockVolume(path)
if err == nil {
t.Fatal("expected error on duplicate add")
}
// Should be listed.
paths := bs.ListBlockVolumes()
if len(paths) != 1 || paths[0] != path {
t.Fatalf("ListBlockVolumes: got %v", paths)
}
// Should be gettable.
got, ok := bs.GetBlockVolume(path)
if !ok || got != vol {
t.Fatal("GetBlockVolume failed")
}
}
func TestStoreRemoveBlockVolume(t *testing.T) {
dir := t.TempDir()
path := createTestBlockVol(t, dir, "vol1.blk")
bs := NewBlockVolumeStore()
defer bs.Close()
if _, err := bs.AddBlockVolume(path); err != nil {
t.Fatal(err)
}
if err := bs.RemoveBlockVolume(path); err != nil {
t.Fatal(err)
}
// Should be gone.
if _, ok := bs.GetBlockVolume(path); ok {
t.Fatal("volume should have been removed")
}
// Remove again should fail.
if err := bs.RemoveBlockVolume(path); err == nil {
t.Fatal("expected error on double remove")
}
}
func TestStoreCloseAllBlockVolumes(t *testing.T) {
dir := t.TempDir()
path1 := createTestBlockVol(t, dir, "vol1.blk")
path2 := createTestBlockVol(t, dir, "vol2.blk")
bs := NewBlockVolumeStore()
if _, err := bs.AddBlockVolume(path1); err != nil {
t.Fatal(err)
}
if _, err := bs.AddBlockVolume(path2); err != nil {
t.Fatal(err)
}
if len(bs.ListBlockVolumes()) != 2 {
t.Fatalf("expected 2 volumes, got %d", len(bs.ListBlockVolumes()))
}
bs.Close()
if len(bs.ListBlockVolumes()) != 0 {
t.Fatalf("expected 0 volumes after close, got %d", len(bs.ListBlockVolumes()))
}
}