diff --git a/seaweed-volume/src/storage/mod.rs b/seaweed-volume/src/storage/mod.rs index ff104b569..57e195ba5 100644 --- a/seaweed-volume/src/storage/mod.rs +++ b/seaweed-volume/src/storage/mod.rs @@ -9,3 +9,4 @@ pub mod store_ec_reconcile; pub mod super_block; pub mod types; pub mod volume; +pub mod volume_idx_repair; diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index f19a7efec..6c3b0cbe2 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use std::sync::{Condvar, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; -use tracing::warn; +use tracing::{info, warn}; #[cfg(test)] use crate::storage::idx; @@ -771,6 +771,23 @@ impl Volume { } if also_load_index { + // Recover rows that deletes on a tiered read-only volume overwrote + // at the front of .idx. Best effort: a volume that cannot be + // repaired is still servable for everything the surviving rows + // index. + match self.repair_idx_head_tombstones() { + Ok(0) => {} + Ok(restored) => info!( + volume_id = self.id.0, + restored, "recovered overwritten .idx rows from .dat" + ), + Err(e) => warn!( + volume_id = self.id.0, + error = %e, + "recover overwritten .idx rows" + ), + } + self.load_index()?; // Match Go: CheckVolumeDataIntegrity after loading index (volume_loading.go L154-159) @@ -990,7 +1007,11 @@ impl Volume { Ok(()) } - fn read_exact_at_backend(&self, buf: &mut [u8], offset: u64) -> Result<(), VolumeError> { + pub(crate) fn read_exact_at_backend( + &self, + buf: &mut [u8], + offset: u64, + ) -> Result<(), VolumeError> { if let Some(dat_file) = self.dat_file.as_ref() { #[cfg(unix)] { diff --git a/seaweed-volume/src/storage/volume_idx_repair.rs b/seaweed-volume/src/storage/volume_idx_repair.rs new file mode 100644 index 000000000..612341b96 --- /dev/null +++ b/seaweed-volume/src/storage/volume_idx_repair.rs @@ -0,0 +1,421 @@ +//! Recovery for .idx rows that deletes on a tiered read-only volume +//! overwrote. Mirrors `weed/storage/volume_idx_repair.go`. + +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufReader, Write}; +use std::path::Path; + +use tracing::info; + +use crate::storage::idx; +use crate::storage::needle::needle::needle_body_length; +use crate::storage::needle::Needle; +use crate::storage::types::*; +use crate::storage::volume::{fsync_dir, Volume, VolumeError}; + +/// Needles found in the head of .dat, keyed by id, plus the ids in .dat order. +type DatHeadNeedles = (HashMap, Vec); + +impl Volume { + /// Restore the .idx rows that deletes on a tiered read-only volume used to + /// overwrite. + /// + /// The sorted-file needle map opened .idx read-write but never seeded its + /// write position, so a delete wrote its (key, offset 0, tombstone) row at + /// .idx offset 0 and advanced one row at a time instead of appending. Every + /// delete therefore replaced one more row at the front of .idx, and the rows + /// it replaced -- the Put rows indexing the first needles in .dat -- were + /// lost. Reads for those needles return not-found even though .dat still + /// holds them intact. + /// + /// The damage leaves a fingerprint: .idx begins with a run of offset-0 + /// tombstones. A healthy .idx never does. Its first row is the Put for the + /// first needle in .dat, and an offset-0 tombstone -- a delete against a + /// tiered volume, which appends no .dat record and so has no extent to point + /// at -- can only ever land at the tail. + /// + /// .idx and .dat grow in lockstep, so the clobbered rows indexed exactly the + /// first N records of .dat. Re-deriving them is a header-only walk over the + /// head of .dat, which stays cheap even when .dat is served from a remote + /// tier. + /// + /// The recovered rows go back in front, where the rows they replace used to + /// sit, and every existing row keeps its relative order behind them. That + /// restores .idx to .dat append order, which several readers lean on: the + /// fingerprint is gone so a later load stops after one row, and the last row + /// is the .dat-tail needle again. + pub(crate) fn repair_idx_head_tombstones(&self) -> Result { + if !self.has_data_backend() { + return Ok(0); + } + let version = self.version(); + if !version.is_supported() { + return Ok(0); + } + let first_needle_offset = self.super_block.block_size() as i64; + let idx_path = self.file_name(".idx"); + + let clobbered = idx_head_tombstone_count(&idx_path)?; + if clobbered == 0 { + return Ok(0); + } + + info!( + volume_id = self.id.0, + idx = %idx_path, + clobbered, + "idx starts with offset-0 tombstones, recovering the rows they overwrote from .dat" + ); + + let (mut lost, order) = self.scan_dat_head(version, first_needle_offset, clobbered)?; + drop_indexed_keys(&idx_path, &mut lost)?; + if lost.is_empty() { + return Ok(0); + } + + let mut rows = Vec::with_capacity(lost.len() * NEEDLE_MAP_ENTRY_SIZE); + let mut restored = 0; + for key in order { + let Some((offset, size)) = lost.get(&key).copied() else { + continue; + }; + idx::write_index_entry(&mut rows, key, offset, size)?; + restored += 1; + } + prepend_idx_rows(&idx_path, &rows)?; + Ok(restored) + } + + /// Read the headers of the first `limit` records of .dat and return the + /// needles they hold, in .dat order. A record with an invalid size is a + /// delete marker, so the key it names drops out of the result rather than + /// being resurrected. + fn scan_dat_head( + &self, + version: Version, + first_needle_offset: i64, + limit: usize, + ) -> Result { + let mut found: HashMap = HashMap::new(); + let mut order: Vec = Vec::new(); + let mut offset = first_needle_offset; + + for _ in 0..limit { + let mut header = [0u8; NEEDLE_HEADER_SIZE]; + match self.read_exact_at_backend(&mut header, offset as u64) { + Ok(()) => {} + Err(VolumeError::Io(ref e)) if e.kind() == io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e), + } + let (_cookie, id, size) = Needle::parse_header(&header); + if size.0 == 0 && id.is_empty() { + break; + } + if size.is_valid() { + if found + .insert(id, (Offset::from_actual_offset(offset), size)) + .is_none() + { + order.push(id); + } + } else { + found.remove(&id); + } + offset += NEEDLE_HEADER_SIZE as i64 + needle_body_length(size, version); + } + + Ok((found, order)) + } +} + +/// Report how many rows at the front of .idx are offset-0 tombstones. A healthy +/// .idx answers 0 on its first row. +fn idx_head_tombstone_count(idx_path: &str) -> Result { + if !Path::new(idx_path).exists() { + return Ok(0); + } + let mut reader = BufReader::new(File::open(idx_path)?); + + let mut clobbered = 0usize; + let walk = idx::walk_index_file(&mut reader, 0, |_key, offset, size| { + if !offset.is_zero() || !size.is_tombstone() { + return Err(stop_walk()); + } + clobbered += 1; + Ok(()) + }); + finish_walk(walk)?; + Ok(clobbered) +} + +/// Remove every candidate the .idx already names, whether by a Put row or a +/// tombstone. What is left is only what the clobbered rows held. +fn drop_indexed_keys( + idx_path: &str, + candidates: &mut HashMap, +) -> Result<(), VolumeError> { + if candidates.is_empty() { + return Ok(()); + } + let mut reader = BufReader::new(File::open(idx_path)?); + let walk = idx::walk_index_file(&mut reader, 0, |key, _offset, _size| { + candidates.remove(&key); + if candidates.is_empty() { + return Err(stop_walk()); + } + Ok(()) + }); + finish_walk(walk) +} + +/// Rewrite .idx as `rows` followed by its current contents, through a temp file +/// and a rename so a crash mid-write never leaves a partial index at the live +/// name. +fn prepend_idx_rows(idx_path: &str, rows: &[u8]) -> Result<(), VolumeError> { + let tmp_path = format!("{}.tmp", idx_path); + let mut src = File::open(idx_path)?; + let commit = (|| -> io::Result<()> { + let src_perm = src.metadata()?.permissions(); + let mut dst = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp_path)?; + // The rename replaces .idx with this file, so it has to carry the mode + // the index already had rather than whatever the umask allows. + dst.set_permissions(src_perm)?; + dst.write_all(rows)?; + io::copy(&mut src, &mut dst)?; + dst.sync_all()?; + drop(dst); + fs::rename(&tmp_path, idx_path)?; + fsync_dir(idx_path) + })(); + if commit.is_err() { + let _ = fs::remove_file(&tmp_path); + } + Ok(commit?) +} + +/// Sentinel that ends an `idx::walk_index_file` early once the answer is known. +fn stop_walk() -> io::Error { + io::Error::new(io::ErrorKind::Interrupted, "stop idx walk") +} + +fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> { + match walk { + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => Ok(()), + other => Ok(other?), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::needle::crc::CRC; + use crate::storage::needle_map::NeedleMapKind; + use std::os::unix::fs::{FileExt, PermissionsExt}; + use tempfile::TempDir; + + fn open_volume(dir: &str) -> Volume { + Volume::new( + dir, + dir, + "", + VolumeId(1), + NeedleMapKind::InMemory, + None, + None, + 0, + Version::current(), + ) + .unwrap() + } + + fn write_test_volume(dir: &str, needle_count: u64) -> Vec> { + let mut v = open_volume(dir); + let mut written = Vec::new(); + for i in 1..=needle_count { + let data = format!("needle-{}-payload", i).into_bytes(); + let mut n = Needle { + id: NeedleId(i), + cookie: Cookie(0x1234_0000 + i as u32), + data_size: data.len() as u32, + checksum: CRC::new(&data), + data: data.clone(), + ..Needle::default() + }; + v.write_needle(&mut n, true).unwrap(); + written.push(data); + } + written + } + + /// Reproduce the damage a delete on a tiered read-only volume used to do: it + /// writes (key, offset 0, tombstone) rows over the front of .idx instead of + /// appending them. + fn clobber_idx_head(idx_path: &str, keys: &[u64]) { + let file = OpenOptions::new().write(true).open(idx_path).unwrap(); + for (i, key) in keys.iter().enumerate() { + let mut row = Vec::new(); + idx::write_index_entry( + &mut row, + NeedleId(*key), + Offset::from_actual_offset(0), + TOMBSTONE_FILE_SIZE, + ) + .unwrap(); + file.write_at(&row, (i * NEEDLE_MAP_ENTRY_SIZE) as u64) + .unwrap(); + } + } + + fn idx_size(idx_path: &str) -> u64 { + std::fs::metadata(idx_path).unwrap().len() + } + + fn idx_rows(idx_path: &str) -> Vec<(NeedleId, i64, Size)> { + let mut reader = BufReader::new(File::open(idx_path).unwrap()); + let mut rows = Vec::new(); + idx::walk_index_file(&mut reader, 0, |key, offset, size| { + rows.push((key, offset.to_actual_offset(), size)); + Ok(()) + }) + .unwrap(); + rows + } + + fn read_needle_data(v: &Volume, id: u64) -> Result, VolumeError> { + let mut n = Needle { + id: NeedleId(id), + ..Needle::default() + }; + v.read_needle(&mut n)?; + Ok(n.data) + } + + #[test] + fn test_repair_idx_head_tombstones_restores_clobbered_rows() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let idx_path = format!("{}/1.idx", dir); + + let written = write_test_volume(dir, 12); + let size_before = idx_size(&idx_path); + + // The rewrite replaces .idx wholesale, so it must not widen the mode. + fs::set_permissions(&idx_path, fs::Permissions::from_mode(0o600)).unwrap(); + + // Deletes against needles 9..12 land on the front of .idx and take the + // rows indexing needles 1..4 with them. + clobber_idx_head(&idx_path, &[9, 10, 11, 12]); + + let v = open_volume(dir); + for i in 1..=8u64 { + assert_eq!( + read_needle_data(&v, i).unwrap(), + written[(i - 1) as usize], + "needle {} not recovered", + i + ); + } + + let want = size_before + 4 * NEEDLE_MAP_ENTRY_SIZE as u64; + assert_eq!(idx_size(&idx_path), want, "idx size after recovery"); + + assert_eq!( + fs::metadata(&idx_path).unwrap().permissions().mode() & 0o777, + 0o600, + "idx mode after recovery" + ); + + // The recovered rows go back in front, so .idx is in .dat append order + // again: the fingerprint is gone and the last row is still the .dat tail. + let rows = idx_rows(&idx_path); + assert_eq!( + rows[0].1, + v.super_block.block_size() as i64, + "first row does not index the first needle in .dat" + ); + for i in 1..4 { + assert!( + rows[i - 1].1 < rows[i].1, + "recovered rows are not in .dat order: {:?} then {:?}", + rows[i - 1], + rows[i] + ); + } + + // The recovery is idempotent: a second load finds nothing left to restore. + drop(v); + let _v = open_volume(dir); + assert_eq!(idx_size(&idx_path), want, "idx grew on second load"); + } + + #[test] + fn test_repair_idx_head_tombstones_leaves_healthy_idx_alone() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let idx_path = format!("{}/1.idx", dir); + + write_test_volume(dir, 6); + let mut v = open_volume(dir); + v.delete_needle(&mut Needle { + id: NeedleId(3), + ..Needle::default() + }) + .unwrap(); + drop(v); + + let size_before = idx_size(&idx_path); + let v = open_volume(dir); + + assert_eq!( + idx_size(&idx_path), + size_before, + "healthy idx was rewritten" + ); + assert!( + matches!( + read_needle_data(&v, 3), + Err(VolumeError::Deleted) | Err(VolumeError::NotFound) + ), + "needle 3 should stay deleted" + ); + } + + /// A needle deleted before the damage must not be resurrected: its tombstone + /// row survives at the tail of .idx, so the key is still indexed and stays + /// out of the recovery. + #[test] + fn test_repair_idx_head_tombstones_keeps_deleted_needles_deleted() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let idx_path = format!("{}/1.idx", dir); + + write_test_volume(dir, 8); + let mut v = open_volume(dir); + v.delete_needle(&mut Needle { + id: NeedleId(2), + ..Needle::default() + }) + .unwrap(); + drop(v); + + clobber_idx_head(&idx_path, &[7, 8]); + + let v = open_volume(dir); + assert!( + matches!( + read_needle_data(&v, 2), + Err(VolumeError::Deleted) | Err(VolumeError::NotFound) + ), + "needle 2 was resurrected" + ); + assert!( + read_needle_data(&v, 1).is_ok(), + "needle 1 should have been recovered" + ); + } +} diff --git a/weed/storage/volume_idx_repair.go b/weed/storage/volume_idx_repair.go new file mode 100644 index 000000000..fef2eb715 --- /dev/null +++ b/weed/storage/volume_idx_repair.go @@ -0,0 +1,236 @@ +package storage + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/idx" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// errStopIdxWalk ends an idx.WalkIndexFile early once the answer is known. +var errStopIdxWalk = errors.New("stop idx walk") + +// repairIdxHeadTombstones restores the .idx rows that deletes on a tiered +// read-only volume used to overwrite. +// +// The sorted-file needle map opened .idx read-write but never seeded its write +// position, so a delete wrote its (key, offset 0, tombstone) row at .idx offset +// 0 and advanced one row at a time instead of appending. Every delete therefore +// replaced one more row at the front of .idx, and the rows it replaced -- the +// Put rows indexing the first needles in .dat -- were lost. Reads for those +// needles return not-found even though .dat still holds them intact. +// +// The damage leaves a fingerprint: .idx begins with a run of offset-0 +// tombstones. A healthy .idx never does. Its first row is the Put for the first +// needle in .dat, and an offset-0 tombstone -- a delete against a tiered volume, +// which appends no .dat record and so has no extent to point at -- can only ever +// land at the tail. +// +// .idx and .dat grow in lockstep, so the clobbered rows indexed exactly the +// first N records of .dat. Re-deriving them is a header-only walk over the head +// of .dat, which stays cheap even when .dat is served from a remote tier. +// +// The recovered rows go back in front, where the rows they replace used to sit, +// and every existing row keeps its relative order behind them. That restores +// .idx to .dat append order, which several readers lean on: the fingerprint is +// gone so a later load stops after one row, the last row is the .dat-tail needle +// again so CheckVolumeDataIntegrity keeps its O(1) path, and +// BinarySearchByAppendAtNs sees ascending append timestamps. +func (v *Volume) repairIdxHeadTombstones() (restored int, err error) { + if v.DataBackend == nil { + return 0, nil + } + version := v.Version() + if !needle.IsSupportedVersion(version) { + return 0, nil + } + firstNeedleOffset := int64(v.SuperBlock.BlockSize()) + idxFileName := v.FileName(".idx") + + clobbered, err := idxHeadTombstoneCount(idxFileName) + if err != nil || clobbered == 0 { + return 0, err + } + + glog.V(0).Infof("volume %d: %s starts with %d offset-0 tombstones, recovering the .idx rows they overwrote from %s", + v.Id, idxFileName, clobbered, v.FileName(".dat")) + + lost, order, err := scanDatHead(v.DataBackend, version, firstNeedleOffset, clobbered) + if err != nil { + return 0, err + } + if err = dropIndexedKeys(idxFileName, lost); err != nil { + return 0, err + } + if len(lost) == 0 { + return 0, nil + } + + var rows []byte + for _, key := range order { + nv, ok := lost[key] + if !ok { + continue + } + rows = append(rows, nv.ToBytes()...) + restored++ + } + return restored, prependIdxRows(idxFileName, rows) +} + +// idxHeadTombstoneCount reports how many rows at the front of .idx are offset-0 +// tombstones. A healthy .idx answers 0 on its first row. +func idxHeadTombstoneCount(idxFileName string) (clobbered int, err error) { + idxFile, err := os.Open(idxFileName) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + defer idxFile.Close() + + err = idx.WalkIndexFile(idxFile, 0, func(_ types.NeedleId, offset types.Offset, size types.Size) error { + if !offset.IsZero() || !size.IsTombstone() { + return errStopIdxWalk + } + clobbered++ + return nil + }) + if errors.Is(err, errStopIdxWalk) { + err = nil + } + return clobbered, err +} + +// scanDatHead reads the headers of the first limit records of .dat and returns +// the needles they hold, in .dat order. A record with an invalid size is a +// delete marker, so the key it names drops out of the result rather than being +// resurrected. +func scanDatHead(datBackend backend.BackendStorageFile, version needle.Version, firstNeedleOffset int64, limit int) (map[types.NeedleId]needle_map.NeedleValue, []types.NeedleId, error) { + scanner := &datHeadScanner{ + limit: limit, + found: make(map[types.NeedleId]needle_map.NeedleValue), + } + if err := ScanVolumeFileFrom(version, datBackend, firstNeedleOffset, scanner); err != nil { + return nil, nil, fmt.Errorf("scan head of %s: %w", datBackend.Name(), err) + } + return scanner.found, scanner.order, nil +} + +type datHeadScanner struct { + limit int + visited int + found map[types.NeedleId]needle_map.NeedleValue + order []types.NeedleId +} + +func (s *datHeadScanner) VisitSuperBlock(super_block.SuperBlock) error { return nil } + +func (s *datHeadScanner) ReadNeedleBody() bool { return false } + +func (s *datHeadScanner) VisitNeedle(n *needle.Needle, offset int64, _, _ []byte) error { + if n.Size.IsValid() { + if _, seen := s.found[n.Id]; !seen { + s.order = append(s.order, n.Id) + } + s.found[n.Id] = needle_map.NeedleValue{Key: n.Id, Offset: types.ToOffset(offset), Size: n.Size} + } else { + delete(s.found, n.Id) + } + s.visited++ + if s.visited >= s.limit { + return io.EOF + } + return nil +} + +// dropIndexedKeys removes every candidate the .idx already names, whether by a +// Put row or a tombstone. What is left is only what the clobbered rows held. +func dropIndexedKeys(idxFileName string, candidates map[types.NeedleId]needle_map.NeedleValue) error { + if len(candidates) == 0 { + return nil + } + idxFile, err := os.Open(idxFileName) + if err != nil { + return err + } + defer idxFile.Close() + + err = idx.WalkIndexFile(idxFile, 0, func(key types.NeedleId, _ types.Offset, _ types.Size) error { + delete(candidates, key) + if len(candidates) == 0 { + return errStopIdxWalk + } + return nil + }) + if errors.Is(err, errStopIdxWalk) { + err = nil + } + return err +} + +// prependIdxRows rewrites .idx as rows followed by its current contents, +// through a temp file and a rename so a crash mid-write never leaves a partial +// index at the live name. +func prependIdxRows(idxFileName string, rows []byte) error { + src, err := os.Open(idxFileName) + if err != nil { + return err + } + defer src.Close() + + srcStat, err := src.Stat() + if err != nil { + return err + } + + tmpFileName := idxFileName + ".tmp" + dst, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcStat.Mode().Perm()) + if err != nil { + return err + } + committed := false + defer func() { + dst.Close() + if !committed { + os.Remove(tmpFileName) + } + }() + + // The rename replaces .idx with this file, so it has to carry the mode the + // index already had -- O_CREATE alone leaves it at the umask's mercy. + if err = dst.Chmod(srcStat.Mode().Perm()); err != nil { + return err + } + if _, err = dst.Write(rows); err != nil { + return err + } + if _, err = io.Copy(dst, src); err != nil { + return err + } + if err = dst.Sync(); err != nil { + return err + } + if err = dst.Close(); err != nil { + return err + } + if err = os.Rename(tmpFileName, idxFileName); err != nil { + return err + } + if err = util.FsyncDir(filepath.Dir(idxFileName)); err != nil { + return err + } + committed = true + return nil +} diff --git a/weed/storage/volume_idx_repair_test.go b/weed/storage/volume_idx_repair_test.go new file mode 100644 index 000000000..e78f03e9b --- /dev/null +++ b/weed/storage/volume_idx_repair_test.go @@ -0,0 +1,221 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" +) + +// clobberIdxHead reproduces the damage a delete on a tiered read-only volume +// used to do: it writes (key, offset 0, tombstone) rows over the front of .idx +// instead of appending them. +func clobberIdxHead(t *testing.T, idxPath string, keys []uint64) { + t.Helper() + f, err := os.OpenFile(idxPath, os.O_WRONLY, 0644) + if err != nil { + t.Fatalf("open idx: %v", err) + } + defer f.Close() + for i, key := range keys { + row := needle_map.ToBytes(types.Uint64ToNeedleId(key), types.Offset{}, types.TombstoneFileSize) + if _, err := f.WriteAt(row, int64(i*types.NeedleMapEntrySize)); err != nil { + t.Fatalf("clobber row %d: %v", i, err) + } + } +} + +func writeTestVolume(t *testing.T, dir string, needleCount int) map[uint64]*needle.Needle { + t.Helper() + v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("volume creation: %v", err) + } + written := make(map[uint64]*needle.Needle) + for i := 1; i <= needleCount; i++ { + n := newRandomNeedle(uint64(i)) + if _, _, _, err := v.writeNeedle2(n, true, false); err != nil { + v.Close() + t.Fatalf("write needle %d: %v", i, err) + } + written[uint64(i)] = n + } + v.Close() + return written +} + +func mustReload(t *testing.T, dir string) *Volume { + t.Helper() + v, err := loadVolumeWithoutWorker(dir, dir, "", 1, NeedleMapInMemory, 0) + if err != nil { + t.Fatalf("reload volume: %v", err) + } + return v +} + +func mustRead(t *testing.T, v *Volume, id uint64) []byte { + t.Helper() + n := newEmptyNeedle(id) + if _, err := v.readNeedle(n, nil, nil); err != nil { + t.Fatalf("read needle %d: %v", id, err) + } + return n.Data +} + +// TestRepairIdxHeadTombstones_RestoresClobberedRows checks that a volume whose +// .idx head was overwritten with offset-0 tombstones serves its first needles +// again after a reload, without a full weed fix rebuild. +func TestRepairIdxHeadTombstones_RestoresClobberedRows(t *testing.T) { + dir := t.TempDir() + const needleCount = 12 + const clobbered = 4 + + written := writeTestVolume(t, dir, needleCount) + idxPath := filepath.Join(dir, "1.idx") + sizeBefore := fileSize(t, idxPath) + + // The rewrite replaces .idx wholesale, so it must not widen the mode. + if err := os.Chmod(idxPath, 0600); err != nil { + t.Fatalf("chmod idx: %v", err) + } + + // Deletes against needles 9..12 land on the front of .idx and take the + // rows indexing needles 1..4 with them. + clobberIdxHead(t, idxPath, []uint64{9, 10, 11, 12}) + + v := mustReload(t, dir) + defer v.Close() + + for i := uint64(1); i <= needleCount; i++ { + if i >= 9 { + // genuinely deleted by the tombstones that did the damage + continue + } + if got, want := mustRead(t, v, i), written[i].Data; string(got) != string(want) { + t.Fatalf("needle %d: data mismatch after recovery", i) + } + } + + if got, want := fileSize(t, idxPath), sizeBefore+int64(clobbered*types.NeedleMapEntrySize); got != want { + t.Fatalf("idx size after recovery: got %d, want %d", got, want) + } + + if st, err := os.Stat(idxPath); err != nil { + t.Fatalf("stat idx: %v", err) + } else if got := st.Mode().Perm(); got != 0600 { + t.Fatalf("idx mode after recovery: got %o, want 600", got) + } + + // The recovered rows go back in front, so .idx is in .dat append order + // again: the fingerprint is gone and the last row is still the .dat tail. + entries := readAllIdxEntries(t, idxPath) + if entries[0].offset.ToActualOffset() != int64(v.SuperBlock.BlockSize()) { + t.Fatalf("first row does not index the first needle in .dat: %+v", entries[0]) + } + for i := 1; i < clobbered; i++ { + if entries[i-1].offset.ToActualOffset() >= entries[i].offset.ToActualOffset() { + t.Fatalf("recovered rows are not in .dat order: %+v then %+v", entries[i-1], entries[i]) + } + } + + // The recovery is idempotent: a second load finds nothing left to restore. + v.Close() + v = mustReload(t, dir) + if got, want := fileSize(t, idxPath), sizeBefore+int64(clobbered*types.NeedleMapEntrySize); got != want { + t.Fatalf("idx grew on second load: got %d, want %d", got, want) + } +} + +// TestRepairIdxHeadTombstones_ReadOnlyVolume covers the shape the damage +// actually occurs in: a read-only volume, whose reads go through the sorted +// needle map. The recovery has to land before .sdx is regenerated, or the +// restored rows never reach the map that answers the read. +func TestRepairIdxHeadTombstones_ReadOnlyVolume(t *testing.T) { + dir := t.TempDir() + written := writeTestVolume(t, dir, 8) + clobberIdxHead(t, filepath.Join(dir, "1.idx"), []uint64{7, 8}) + + if err := volume_info.SaveVolumeInfo(filepath.Join(dir, "1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.GetCurrentVersion()), + ReadOnly: true, + }); err != nil { + t.Fatalf("save vif: %v", err) + } + + v := mustReload(t, dir) + defer v.Close() + + if _, isSorted := v.nm.(*SortedFileNeedleMap); !isSorted { + t.Fatalf("expected a sorted needle map, got %T", v.nm) + } + for i := uint64(1); i <= 2; i++ { + if got, want := mustRead(t, v, i), written[i].Data; string(got) != string(want) { + t.Fatalf("needle %d: data mismatch after recovery", i) + } + } +} + +// TestRepairIdxHeadTombstones_LeavesHealthyIdxAlone guards the fingerprint: a +// volume with ordinary deletes must not be rewritten. +func TestRepairIdxHeadTombstones_LeavesHealthyIdxAlone(t *testing.T) { + dir := t.TempDir() + writeTestVolume(t, dir, 6) + idxPath := filepath.Join(dir, "1.idx") + + v := mustReload(t, dir) + if _, err := v.doDeleteRequest(newEmptyNeedle(3)); err != nil { + t.Fatalf("delete needle 3: %v", err) + } + v.Close() + + sizeBefore := fileSize(t, idxPath) + entriesBefore := readAllIdxEntries(t, idxPath) + + v = mustReload(t, dir) + defer v.Close() + + if got := fileSize(t, idxPath); got != sizeBefore { + t.Fatalf("healthy idx was rewritten: got %d, want %d", got, sizeBefore) + } + if got := readAllIdxEntries(t, idxPath); len(got) != len(entriesBefore) { + t.Fatalf("healthy idx row count changed: got %d, want %d", len(got), len(entriesBefore)) + } + if _, err := v.readNeedle(newEmptyNeedle(3), nil, nil); err != ErrorDeleted { + t.Fatalf("needle 3 should stay deleted, got %v", err) + } +} + +// TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted checks that a needle +// deleted before the damage is not resurrected: its tombstone row survives at +// the tail of .idx, so the key is still indexed and stays out of the recovery. +// It stays unreadable either way -- as deleted if its Put row survived, as +// not-found if the tombstone is all the .idx has left of it. +func TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted(t *testing.T) { + dir := t.TempDir() + writeTestVolume(t, dir, 8) + idxPath := filepath.Join(dir, "1.idx") + + v := mustReload(t, dir) + if _, err := v.doDeleteRequest(newEmptyNeedle(2)); err != nil { + t.Fatalf("delete needle 2: %v", err) + } + v.Close() + + clobberIdxHead(t, idxPath, []uint64{7, 8}) + + v = mustReload(t, dir) + defer v.Close() + + if _, err := v.readNeedle(newEmptyNeedle(2), nil, nil); err != ErrorDeleted && err != ErrorNotFound { + t.Fatalf("needle 2 was resurrected, got %v", err) + } + if _, err := v.readNeedle(newEmptyNeedle(1), nil, nil); err != nil { + t.Fatalf("needle 1 should have been recovered: %v", err) + } +} diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index 3ede17c75..8bcbf4aeb 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -252,6 +252,14 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind } glog.Fatalf("check volume idx file %s: %v", v.FileName(".idx"), err) } + // Recover rows that deletes on a tiered read-only volume overwrote at + // the front of .idx. Best effort: a volume that cannot be repaired is + // still servable for everything the surviving rows index. + if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil { + glog.Warningf("volume %d: recover overwritten %s rows: %v", v.Id, v.FileName(".idx"), repairErr) + } else if restored > 0 { + glog.V(0).Infof("volume %d: recovered %d overwritten %s rows from %s", v.Id, restored, v.FileName(".idx"), v.FileName(".dat")) + } var indexFile *os.File if v.noWriteOrDelete { glog.V(0).Infoln("open to read file", v.FileName(".idx"))