volume: detect phantom volumes held open as deleted FDs (#10011)

* volume: detect phantom volumes held open as deleted FDs

Add disk-file validation in heartbeat collection to prevent reporting
phantom volumes that exist in memory but are deleted from disk. This
unblocks re-replication when files are unlinked while the volume server
holds them open via file descriptors.

Cache disk checks per-volume with 30-second TTL to avoid syscall overhead.
Implement in both Go and Rust volume servers.

* volume: make last_disk_check_ns field public for heartbeat access

* volume: only check for phantom volumes when size > 0

Skip phantom volume detection for zero-size volumes (e.g., test volumes).
Phantom volumes only occur when disk files are deleted while the process
holds them open via FDs - which requires the volume to have had actual data.
Test volumes with zero size should not trigger disk file existence checks.

* volume: only check for phantom volumes when size > 0

Skip phantom volume detection for zero-size volumes (e.g., test volumes).
Phantom volumes only occur when disk files are deleted while the process
holds them open via FDs - which requires the volume to have had actual data.
Test volumes with zero size should not trigger disk file existence checks.

* volume: only check for phantom volumes if file_count > 0

Use file_count as the indicator for whether a volume held actual data,
rather than volume size. Phantom volumes only occur when a volume that
had files is deleted while the process holds open file descriptors.
Test volumes with no file count won't trigger the phantom detection check.

* volume: stat the .dat with its extension when detecting phantom volumes

DataFileName()/IndexFileName() return the extensionless base path, so os.Stat
saw every volume's files as missing and dropped it from the heartbeat, leaving
the master with no locations and breaking deletes/lookups. Stat FileName(".dat")
instead, skip remote-tiered volumes whose .dat lives in cloud storage, and
re-check a missing file every heartbeat rather than caching the negative.

seaweed-volume/src/server/heartbeat.rs

@@ -4,9 +4,10 @@
 //! matching Go's `server/volume_grpc_client_to_master.go`.
 
 use std::collections::HashMap;
+use std::path::Path;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use tokio::sync::broadcast;
 use tracing::{error, info, warn};
@@ -827,6 +828,27 @@ fn build_heartbeat_with_ec_status(
                 delete_vids.push(vol.id);
                 should_delete_volume = true;
             } else if !vol.is_expired(volume_size, volume_size_limit) {
+                // Detect phantom volumes: the .dat was unlinked from disk but is still
+                // held open as a deleted FD, so the volume keeps serving and heartbeating
+                // while no disk-path operation can ever succeed. Skip remote-tiered volumes,
+                // whose .dat legitimately lives in cloud storage. Only a present .dat is
+                // cached for 30s; a missing one is re-checked every heartbeat so the volume
+                // stays suppressed until the file returns. See issues/10004
+                if vol.file_count() > 0 && !vol.has_remote_file {
+                    const DISK_CHECK_INTERVAL_NS: i64 = 30 * 1_000_000_000;
+                    let now_ns = SystemTime::now()
+                        .duration_since(UNIX_EPOCH)
+                        .unwrap_or(Duration::ZERO)
+                        .as_nanos() as i64;
+                    if now_ns - vol.last_disk_check_ns.load(Ordering::Relaxed) > DISK_CHECK_INTERVAL_NS {
+                        if !Path::new(&vol.file_name(".dat")).exists() {
+                            warn!("Volume {}: data file {} missing (held open as deleted FD) - not reporting to master", vol.id.0, vol.file_name(".dat"));
+                            continue;
+                        }
+                        vol.last_disk_check_ns.store(now_ns, Ordering::Relaxed);
+                    }
+                }
+
                 let (remote_storage_name, remote_storage_key) = vol.remote_storage_name_key();
                 volumes.push(master_pb::VolumeInformationMessage {
                     id: vol.id.0,

seaweed-volume/src/storage/volume.rs

@@ -503,6 +503,7 @@ pub struct Volume {
 
     last_modified_ts_seconds: u64,
     last_append_at_ns: u64,
+    pub last_disk_check_ns: Arc<std::sync::atomic::AtomicI64>, // for phantom volume detection cache
 
     last_compact_index_offset: u64,
     last_compact_revision: u16,
@@ -575,6 +576,7 @@ impl Volume {
             location_disk_space_low: Arc::new(AtomicBool::new(false)),
             last_modified_ts_seconds: 0,
             last_append_at_ns: 0,
+            last_disk_check_ns: Arc::new(std::sync::atomic::AtomicI64::new(0)),
             last_compact_index_offset: 0,
             last_compact_revision: 0,
             is_compacting: false,
@@ -608,6 +610,7 @@ impl Volume {
             location_disk_space_low: Arc::new(AtomicBool::new(false)),
             last_modified_ts_seconds: 0,
             last_append_at_ns: 0,
+            last_disk_check_ns: Arc::new(std::sync::atomic::AtomicI64::new(0)),
             last_compact_index_offset: 0,
             last_compact_revision: 0,
             is_compacting: false,

weed/storage/volume.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"os"
 	"path"
 	"strconv"
 	"sync"
@@ -47,6 +48,7 @@ type Volume struct {
 	ldbTimeout             int64
 
 	isCompactionInProgress atomic.Bool
+	lastDiskCheckNs        atomic.Int64 // unix time in nanoseconds for phantom volume detection
 
 	volumeInfoRWLock sync.RWMutex
 	volumeInfo       *volume_server_pb.VolumeInfo
@@ -412,6 +414,24 @@ func (v *Volume) ToVolumeInformationMessage() (types.NeedleId, *master_pb.Volume
 		return 0, nil
 	}
 
+	// Detect phantom volumes: the .dat was unlinked from disk but is still held
+	// open as a deleted FD, so the volume keeps serving and heartbeating while no
+	// disk-path operation can ever succeed. Skip remote-tiered volumes, whose .dat
+	// legitimately lives in cloud storage. Only a present .dat is cached for 30s; a
+	// missing one is re-checked every heartbeat so the volume stays suppressed until
+	// the file returns. See github.com/seaweedfs/seaweedfs/issues/10004
+	if fileCount > 0 && !v.HasRemoteFile() {
+		const diskCheckIntervalNs = 30 * int64(time.Second)
+		now := time.Now().UnixNano()
+		if now-v.lastDiskCheckNs.Load() > diskCheckIntervalNs {
+			if _, err := os.Stat(v.FileName(".dat")); os.IsNotExist(err) {
+				glog.Warningf("Volume %d: data file %s missing (held open as deleted FD) - not reporting to master", v.Id, v.FileName(".dat"))
+				return 0, nil
+			}
+			v.lastDiskCheckNs.Store(now)
+		}
+	}
+
 	volumeInfo := &master_pb.VolumeInformationMessage{
 		Id:               uint32(v.Id),
 		Size:             uint64(volumeSize),