seaweedfs/weed/storage/volume.go

package storage

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"

	"github.com/seaweedfs/seaweedfs/weed/glog"
)

type Volume struct {
	Id                 needle.VolumeId
	dir                string
	dirIdx             string
	Collection         string
	DataBackend        backend.BackendStorageFile
	nm                 NeedleMapper
	tmpNm              TempNeedleMapper
	needleMapKind      NeedleMapKind
	noWriteOrDelete    bool // if readonly, either noWriteOrDelete or noWriteCanDelete
	noWriteCanDelete   bool // if readonly, either noWriteOrDelete or noWriteCanDelete
	noWriteLock        sync.RWMutex
	hasRemoteFile      atomic.Bool // if the volume is tiered: data lives in a remote backend
	MemoryMapMaxSizeMb uint32

	super_block.SuperBlock

	dataFileAccessLock   sync.RWMutex
	superBlockAccessLock sync.Mutex

	// The batch worker exists only once the volume takes a durable write. Most
	// never do -- read-only, remote-tiered, or written without fsync -- and a
	// parked worker costs its goroutine stack plus a 128-slot channel, which a
	// server holding millions of volumes cannot pay for all of them.
	asyncWorkerLock   sync.Mutex
	asyncRequestsChan chan *needle.AsyncRequest
	asyncWorkerClosed bool

	lastModifiedTsSeconds uint64 // unix time in seconds
	lastAppendAtNs        uint64 // unix time in nanoseconds

	lastCompactIndexOffset uint64
	lastCompactRevision    uint16
	ldbTimeout             int64

	isCompactionInProgress atomic.Bool
	lastDiskCheckNs        atomic.Int64 // unix time in nanoseconds for phantom volume detection

	volumeInfoRWLock sync.RWMutex
	volumeInfo       *volume_server_pb.VolumeInfo
	location         *DiskLocation
	diskId           uint32 // ID of this volume's disk in Store.Locations array

	// lastIoError is the most recent EIO from a read/write/delete; cleared
	// on the next successful or non-EIO op. lastIoErrorCount tracks
	// consecutive EIOs so CollectHeartbeat can require a sustained failure
	// before unmounting the replica — protects against a transient
	// hardware/network blip hitting multiple replicas at once and
	// stranding the only good copy.
	//
	// ioErrorQuarantined is sticky: once CollectHeartbeat sees the streak
	// cross IoErrorTolerance it sets this and never clears it on its own.
	// A subsequent successful read clears the streak counter but must NOT
	// un-quarantine the volume — only MarkVolumeWritable does that, after
	// an operator has decided the disk is healthy. Without the sticky
	// bit, one good read between heartbeats would silently put a known-
	// bad replica back into rotation.
	//
	// All four fields are guarded together so the heartbeat reader sees
	// a consistent snapshot.
	lastIoError        error
	lastIoErrorCount   int32
	ioErrorQuarantined bool
	lastIoErrorLock    sync.RWMutex
}

// noteIoError records an EIO and increments the consecutive-error
// counter. Caller has already verified errors.Is(err, syscall.EIO).
func (v *Volume) noteIoError(err error) {
	v.lastIoErrorLock.Lock()
	defer v.lastIoErrorLock.Unlock()
	v.lastIoError = err
	v.lastIoErrorCount++
}

// clearIoError resets the EIO streak counter only. The sticky quarantine
// bit set by CollectHeartbeat is intentionally left alone — recovery is
// an operator decision via MarkVolumeWritable. Called on any successful
// op or on a non-EIO error (which still breaks the EIO streak; only
// sustained EIOs are diagnostic of a failing volume).
func (v *Volume) clearIoError() {
	v.lastIoErrorLock.Lock()
	defer v.lastIoErrorLock.Unlock()
	v.lastIoError = nil
	v.lastIoErrorCount = 0
}

// resetIoErrorState clears both the EIO streak and the sticky quarantine
// flag. Used by MarkVolumeWritable to rejoin a previously-quarantined
// replica; if the disk is still bad, the next failed op re-arms the
// streak.
func (v *Volume) resetIoErrorState() {
	v.lastIoErrorLock.Lock()
	defer v.lastIoErrorLock.Unlock()
	v.lastIoError = nil
	v.lastIoErrorCount = 0
	v.ioErrorQuarantined = false
}

// markIoQuarantined sets the sticky quarantine flag. Idempotent; safe
// to call from CollectHeartbeat each pass while the volume remains
// quarantined.
func (v *Volume) markIoQuarantined() {
	v.lastIoErrorLock.Lock()
	defer v.lastIoErrorLock.Unlock()
	v.ioErrorQuarantined = true
}

// getIoErrorState returns the latest EIO, the consecutive-EIO count,
// and the sticky quarantine flag as one consistent snapshot.
func (v *Volume) getIoErrorState() (error, int32, bool) {
	v.lastIoErrorLock.RLock()
	defer v.lastIoErrorLock.RUnlock()
	return v.lastIoError, v.lastIoErrorCount, v.ioErrorQuarantined
}

func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, ver needle.Version, memoryMapMaxSizeMb uint32, ldbTimeout int64) (v *Volume, e error) {
	// if replicaPlacement is nil, the superblock will be loaded from disk
	v = &Volume{dir: dirname, dirIdx: dirIdx, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb}
	v.SuperBlock = super_block.SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
	v.needleMapKind = needleMapKind
	v.ldbTimeout = ldbTimeout
	e = v.load(true, true, needleMapKind, preallocate, ver)
	return
}

func (v *Volume) String() string {
	v.noWriteLock.RLock()
	defer v.noWriteLock.RUnlock()
	return fmt.Sprintf("Id:%v dir:%s dirIdx:%s Collection:%s dataFile:%v nm:%v noWrite:%v canDelete:%v", v.Id, v.dir, v.dirIdx, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
}

func VolumeFileName(dir string, collection string, id int) (fileName string) {
	idString := strconv.Itoa(id)
	if collection == "" {
		fileName = path.Join(dir, idString)
	} else {
		fileName = path.Join(dir, collection+"_"+idString)
	}
	return
}

func (v *Volume) DataFileName() (fileName string) {
	return VolumeFileName(v.dir, v.Collection, int(v.Id))
}

func (v *Volume) IndexFileName() (fileName string) {
	return VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
}

func (v *Volume) FileName(ext string) (fileName string) {
	switch ext {
	case ".idx", ".cpx", ".ldb", ".cpldb", ".rdb":
		return VolumeFileName(v.dirIdx, v.Collection, int(v.Id)) + ext
	}
	// .dat, .cpd, .vif
	return VolumeFileName(v.dir, v.Collection, int(v.Id)) + ext
}

// RelocateIndexTo moves the volume's index to newIdxDir and reopens the volume
// against it in place, without unmounting. It takes the data-file write lock —
// so a concurrent read blocks briefly instead of failing — closes the needle
// map and data backend, moves the .idx (and the derived .sdx best-effort), then
// retargets dirIdx and reloads, mirroring CommitCompact's close-swap-load. A
// decode co-locates the rebuilt index with the data so the on-demand mount can
// find the volume; this returns it to the -dir.idx tier once the EC shards are
// gone. A no-op when the index already lives in newIdxDir. A derived .ldb is
// not moved: the reload rebuilds it in newIdxDir from the .idx.
func (v *Volume) RelocateIndexTo(newIdxDir string) error {
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()

	if v.dirIdx == newIdxDir {
		return nil
	}
	oldBase := VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
	if _, err := os.Stat(oldBase + ".idx"); err != nil {
		return nil // nothing co-located to move
	}
	newBase := VolumeFileName(newIdxDir, v.Collection, int(v.Id))

	if v.nm != nil {
		_ = v.nm.Sync()
		v.nm.Close()
		v.nm = nil
	}
	if v.DataBackend != nil {
		_ = v.DataBackend.Sync()
		_ = v.DataBackend.Close()
		v.DataBackend = nil
	}

	if err := RenameOrCopyFile(oldBase+".idx", newBase+".idx"); err != nil {
		// Reopen against the old dir so the volume is not left down; surface a
		// failed reopen since it leaves the volume unusable until the next load.
		if reopenErr := v.load(true, false, v.needleMapKind, 0, v.Version()); reopenErr != nil {
			glog.Errorf("relocate volume %d: reopen after failed .idx move: %v", v.Id, reopenErr)
		}
		return fmt.Errorf("relocate index for volume %d: move .idx: %w", v.Id, err)
	}
	// The .sdx is a derived sorted index; move it when present, but a failure is
	// not fatal — drop the stale copy so the reload rebuilds it in the new dir.
	if _, err := os.Stat(oldBase + ".sdx"); err == nil {
		if err := RenameOrCopyFile(oldBase+".sdx", newBase+".sdx"); err != nil {
			glog.Warningf("relocate volume %d: move .sdx: %v (will rebuild)", v.Id, err)
			_ = os.Remove(oldBase + ".sdx")
		}
	}
	v.dirIdx = newIdxDir
	return v.load(true, false, v.needleMapKind, 0, v.Version())
}

func (v *Volume) Version() needle.Version {
	v.superBlockAccessLock.Lock()
	defer v.superBlockAccessLock.Unlock()
	if v.volumeInfo.Version != 0 {
		v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)
	}
	return v.SuperBlock.Version
}

func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()

	if v.DataBackend == nil {
		return
	}

	datFileSize, modTime, e := v.DataBackend.GetStat()
	if e == nil {
		return uint64(datFileSize), v.nm.IndexFileSize(), modTime
	}
	glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
	return // -1 causes integer overflow and the volume to become unwritable.
}

func (v *Volume) ContentSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.ContentSize()
}

func (v *Volume) doIsEmpty() (bool, error) {
	// check v.DataBackend.GetStat()
	if v.DataBackend == nil {
		return false, fmt.Errorf("v.DataBackend is nil")
	} else {
		datFileSize, _, e := v.DataBackend.GetStat()
		if e != nil {
			glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
			return false, fmt.Errorf("v.DataBackend.GetStat(): %v", e)
		}
		if datFileSize > super_block.SuperBlockSize {
			return false, nil
		}
	}
	// check v.nm.ContentSize()
	if v.nm != nil {
		if v.nm.ContentSize() > 0 {
			return false, nil
		}
	}
	return true, nil
}

func (v *Volume) DeletedSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.DeletedSize()
}

func (v *Volume) FileCount() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return uint64(v.nm.FileCount())
}

func (v *Volume) DeletedCount() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return uint64(v.nm.DeletedCount())
}

func (v *Volume) MaxFileKey() types.NeedleId {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.MaxFileKey()
}

func (v *Volume) IndexFileSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.IndexFileSize()
}

func (v *Volume) DiskType() types.DiskType {
	return v.location.DiskType
}

func (v *Volume) SyncToDisk() {
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()
	if v.nm != nil {
		if err := v.nm.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
		}
	}
	if v.DataBackend != nil {
		if err := v.DataBackend.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
		}
	}
}

// Close cleanly shuts down this volume
func (v *Volume) Close() {
	// Wait for any in-progress compaction to finish and claim the flag so no
	// new compaction can start. This must happen BEFORE acquiring
	// dataFileAccessLock to avoid deadlocking with CommitCompact which holds
	// the flag while waiting for the lock.
	for !v.isCompactionInProgress.CompareAndSwap(false, true) {
		time.Sleep(521 * time.Millisecond)
		glog.Warningf("Volume Close wait for compaction %d", v.Id)
	}
	defer v.isCompactionInProgress.Store(false)

	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()

	v.doClose()
}

// SwapDataBackend atomically replaces the data backend and updates the
// remote-tier flag under dataFileAccessLock, closing the old backend. Both tier
// directions go through here so hasRemoteFile always matches the live backend:
// tier-down passes hasRemoteFile=false (now serving a local .dat), tier-up
// passes true. Keeping the swap and the flag under one lock means the heartbeat
// never observes a half-swapped backend or a flag that disagrees with it — a
// stale-false flag would make doDeleteRequest skip the new .dat's tombstones and
// disable the phantom-.dat guard.
func (v *Volume) SwapDataBackend(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()
	v.swapDataBackendLocked(newBackend, hasRemoteFile)
}

// swapDataBackendLocked is the body of SwapDataBackend for callers that already
// hold dataFileAccessLock (e.g. load() reached while CommitCompact holds the
// lock). Reusing it from those under-lock paths avoids re-entering the
// non-reentrant lock, which would deadlock.
func (v *Volume) swapDataBackendLocked(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
	if v.DataBackend != nil {
		v.DataBackend.Close()
	}
	v.DataBackend = newBackend
	v.hasRemoteFile.Store(hasRemoteFile)
}

func (v *Volume) doClose() {
	if v.nm != nil {
		if err := v.nm.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
		}
		v.nm.Close()
		v.nm = nil
	}
	if v.DataBackend != nil {
		if err := v.DataBackend.Close(); err != nil {
			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
		}
		v.DataBackend = nil
		stats.VolumeServerVolumeGauge.WithLabelValues(v.Collection, "volume").Dec()
	}
}

func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaPlacement.GetCopyCount() > 1
}

// volume is expired if modified time + volume ttl < now
// except when volume is empty
// or when the volume does not have a ttl
// or when volumeSizeLimit is 0 when server just starts
func (v *Volume) expired(contentSize uint64, volumeSizeLimit uint64) bool {
	if volumeSizeLimit == 0 {
		// skip if we don't know size limit
		return false
	}
	if contentSize <= super_block.SuperBlockSize {
		return false
	}
	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
		return false
	}
	glog.V(2).Infof("volume %d now:%v lastModified:%v", v.Id, time.Now().Unix(), v.lastModifiedTsSeconds)
	livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTsSeconds)) / 60
	glog.V(2).Infof("volume %d ttl:%v lived:%v", v.Id, v.Ttl, livedMinutes)
	if int64(v.Ttl.Minutes()) < livedMinutes {
		return true
	}
	return false
}

// wait either maxDelayMinutes or 10% of ttl minutes
func (v *Volume) expiredLongEnough(maxDelayMinutes uint32) bool {
	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
		return false
	}
	removalDelay := v.Ttl.Minutes() / 10
	if removalDelay > maxDelayMinutes {
		removalDelay = maxDelayMinutes
	}

	if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTsSeconds < uint64(time.Now().Unix()) {
		return true
	}
	return false
}

func (v *Volume) collectStatus() (maxFileKey types.NeedleId, datFileSize int64, modTime time.Time, fileCount, deletedCount, deletedSize uint64, ok bool) {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()

	if v.nm == nil || v.DataBackend == nil {
		return
	}

	ok = true

	maxFileKey = v.nm.MaxFileKey()
	datFileSize, modTime, _ = v.DataBackend.GetStat()
	fileCount = uint64(v.nm.FileCount())
	deletedCount = uint64(v.nm.DeletedCount())
	deletedSize = v.nm.DeletedSize()

	return
}

// ToVolumeInformationMessage fills into with what the master is told about this
// volume, allocating a message when into is nil. A heartbeat that keeps only
// the volumes it reports fills the same message for all the rest.
func (v *Volume) ToVolumeInformationMessage(into *master_pb.VolumeInformationMessage) (types.NeedleId, *master_pb.VolumeInformationMessage) {

	maxFileKey, volumeSize, modTime, fileCount, deletedCount, deletedSize, ok := v.collectStatus()

	if !ok {
		return 0, nil
	}

	// Detect phantom volumes: the .dat was unlinked from disk but is still held
	// open as a deleted FD, so the volume keeps serving and heartbeating while no
	// disk-path operation can ever succeed. Skip remote-tiered volumes, whose .dat
	// legitimately lives in cloud storage. Only a present .dat is cached for 30s; a
	// missing one is re-checked every heartbeat so the volume stays suppressed until
	// the file returns. See github.com/seaweedfs/seaweedfs/issues/10004
	if fileCount > 0 && !v.HasRemoteFile() {
		const diskCheckIntervalNs = 30 * int64(time.Second)
		now := time.Now().UnixNano()
		if now-v.lastDiskCheckNs.Load() > diskCheckIntervalNs {
			if _, err := os.Stat(v.FileName(".dat")); os.IsNotExist(err) {
				glog.Warningf("Volume %d: data file %s missing (held open as deleted FD) - not reporting to master", v.Id, v.FileName(".dat"))
				return 0, nil
			}
			v.lastDiskCheckNs.Store(now)
		}
	}

	volumeInfo := into
	if volumeInfo == nil {
		volumeInfo = &master_pb.VolumeInformationMessage{}
	}
	volumeInfo.Id = uint32(v.Id)
	volumeInfo.Size = uint64(volumeSize)
	volumeInfo.Collection = v.Collection
	volumeInfo.FileCount = fileCount
	volumeInfo.DeleteCount = deletedCount
	volumeInfo.DeletedByteCount = deletedSize
	volumeInfo.ReadOnly = v.IsReadOnly()
	volumeInfo.ReplicaPlacement = uint32(v.ReplicaPlacement.Byte())
	volumeInfo.Version = uint32(v.Version())
	volumeInfo.Ttl = v.Ttl.ToUint32()
	volumeInfo.CompactRevision = uint32(v.SuperBlock.CompactionRevision)
	volumeInfo.ModifiedAtSecond = modTime.Unix()
	volumeInfo.DiskType = string(v.location.DiskType)
	volumeInfo.DiskId = v.diskId
	volumeInfo.RemoteStorageName, volumeInfo.RemoteStorageKey = v.RemoteStorageNameKey()

	return maxFileKey, volumeInfo
}

func (v *Volume) RemoteStorageNameKey() (storageName, storageKey string) {
	if v.volumeInfo == nil {
		return
	}
	if len(v.volumeInfo.GetFiles()) == 0 {
		return
	}
	return v.volumeInfo.GetFiles()[0].BackendName(), v.volumeInfo.GetFiles()[0].GetKey()
}

func (v *Volume) IsReadOnly() bool {
	readOnly, _, _, _ := v.ReadOnlyReasons()
	return readOnly
}

// ReadOnlyReasons reports whether the volume refuses writes and why, reading the
// flags once so the reasons cannot disagree with the verdict.
func (v *Volume) ReadOnlyReasons() (readOnly, noWriteOrDelete, noWriteCanDelete, diskSpaceLow bool) {
	v.noWriteLock.RLock()
	noWriteOrDelete, noWriteCanDelete = v.noWriteOrDelete, v.noWriteCanDelete
	v.noWriteLock.RUnlock()
	// The location is attached when the volume joins a disk location, which is
	// after NewVolume hands it back.
	diskSpaceLow = v.location != nil && v.location.isDiskSpaceLow.Load()
	return noWriteOrDelete || noWriteCanDelete || diskSpaceLow, noWriteOrDelete, noWriteCanDelete, diskSpaceLow
}

func (v *Volume) PersistReadOnly(readOnly bool, canDelete bool) {
	v.volumeInfoRWLock.Lock()
	defer v.volumeInfoRWLock.Unlock()
	v.volumeInfo.ReadOnly = readOnly
	v.volumeInfo.ReadOnlyCanDelete = readOnly && canDelete
	v.SaveVolumeInfo()
}