diff --git a/seaweed-volume/src/server/heartbeat.rs b/seaweed-volume/src/server/heartbeat.rs index 2d668c67d..00070960f 100644 --- a/seaweed-volume/src/server/heartbeat.rs +++ b/seaweed-volume/src/server/heartbeat.rs @@ -4,9 +4,10 @@ //! matching Go's `server/volume_grpc_client_to_master.go`. use std::collections::HashMap; +use std::path::Path; use std::sync::atomic::Ordering; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::broadcast; use tracing::{error, info, warn}; @@ -827,6 +828,27 @@ fn build_heartbeat_with_ec_status( delete_vids.push(vol.id); should_delete_volume = true; } else if !vol.is_expired(volume_size, volume_size_limit) { + // Detect phantom volumes: the .dat was unlinked from disk but is still + // held open as a deleted FD, so the volume keeps serving and heartbeating + // while no disk-path operation can ever succeed. Skip remote-tiered volumes, + // whose .dat legitimately lives in cloud storage. Only a present .dat is + // cached for 30s; a missing one is re-checked every heartbeat so the volume + // stays suppressed until the file returns. See issues/10004 + if vol.file_count() > 0 && !vol.has_remote_file { + const DISK_CHECK_INTERVAL_NS: i64 = 30 * 1_000_000_000; + let now_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_nanos() as i64; + if now_ns - vol.last_disk_check_ns.load(Ordering::Relaxed) > DISK_CHECK_INTERVAL_NS { + if !Path::new(&vol.file_name(".dat")).exists() { + warn!("Volume {}: data file {} missing (held open as deleted FD) - not reporting to master", vol.id.0, vol.file_name(".dat")); + continue; + } + vol.last_disk_check_ns.store(now_ns, Ordering::Relaxed); + } + } + let (remote_storage_name, remote_storage_key) = vol.remote_storage_name_key(); volumes.push(master_pb::VolumeInformationMessage { id: vol.id.0, diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 09269e37d..d7f39d397 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -503,6 +503,7 @@ pub struct Volume { last_modified_ts_seconds: u64, last_append_at_ns: u64, + pub last_disk_check_ns: Arc, // for phantom volume detection cache last_compact_index_offset: u64, last_compact_revision: u16, @@ -575,6 +576,7 @@ impl Volume { location_disk_space_low: Arc::new(AtomicBool::new(false)), last_modified_ts_seconds: 0, last_append_at_ns: 0, + last_disk_check_ns: Arc::new(std::sync::atomic::AtomicI64::new(0)), last_compact_index_offset: 0, last_compact_revision: 0, is_compacting: false, @@ -608,6 +610,7 @@ impl Volume { location_disk_space_low: Arc::new(AtomicBool::new(false)), last_modified_ts_seconds: 0, last_append_at_ns: 0, + last_disk_check_ns: Arc::new(std::sync::atomic::AtomicI64::new(0)), last_compact_index_offset: 0, last_compact_revision: 0, is_compacting: false, diff --git a/weed/storage/volume.go b/weed/storage/volume.go index b326f9ab3..f7a5423f2 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "os" "path" "strconv" "sync" @@ -47,6 +48,7 @@ type Volume struct { ldbTimeout int64 isCompactionInProgress atomic.Bool + lastDiskCheckNs atomic.Int64 // unix time in nanoseconds for phantom volume detection volumeInfoRWLock sync.RWMutex volumeInfo *volume_server_pb.VolumeInfo @@ -412,6 +414,24 @@ func (v *Volume) ToVolumeInformationMessage() (types.NeedleId, *master_pb.Volume return 0, nil } + // Detect phantom volumes: the .dat was unlinked from disk but is still held + // open as a deleted FD, so the volume keeps serving and heartbeating while no + // disk-path operation can ever succeed. Skip remote-tiered volumes, whose .dat + // legitimately lives in cloud storage. Only a present .dat is cached for 30s; a + // missing one is re-checked every heartbeat so the volume stays suppressed until + // the file returns. See github.com/seaweedfs/seaweedfs/issues/10004 + if fileCount > 0 && !v.HasRemoteFile() { + const diskCheckIntervalNs = 30 * int64(time.Second) + now := time.Now().UnixNano() + if now-v.lastDiskCheckNs.Load() > diskCheckIntervalNs { + if _, err := os.Stat(v.FileName(".dat")); os.IsNotExist(err) { + glog.Warningf("Volume %d: data file %s missing (held open as deleted FD) - not reporting to master", v.Id, v.FileName(".dat")) + return 0, nil + } + v.lastDiskCheckNs.Store(now) + } + } + volumeInfo := &master_pb.VolumeInformationMessage{ Id: uint32(v.Id), Size: uint64(volumeSize),