mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 04:06:44 +00:00
volume: recover .idx rows overwritten by tiered deletes (#10474)
* volume: recover .idx rows overwritten by tiered deletes A delete on a read-only volume backed by a remote tier used to write its tombstone row at .idx offset 0 rather than appending it, so each delete overwrote one more row at the front and lost the Put rows indexing the first needles in .dat. Those needles 404 even though .dat still holds them, and rebuilding .idx with weed fix means stopping the server and pulling the whole .dat back from the tier. The damage has a fingerprint -- .idx opening with a run of offset-0 tombstones, which a healthy .idx never does -- and .idx and .dat grow in lockstep, so the lost rows indexed exactly the first N .dat records. Detect it at load and re-derive them from a header-only walk over the head of .dat, cheap even against a remote tier, appending only the keys the .idx no longer names. * rust volume: mirror the .idx head tombstone recovery Port the Go detection and repair: an .idx opening with a run of offset-0 tombstones lost the Put rows indexing the first needles in .dat, so re-derive them at load from a header-only walk over the head of .dat and append the keys the .idx no longer names. * volume: put recovered .idx rows back in front instead of appending Appending left the offset-0 tombstone run at the head, so every later load re-walked .idx to the tail to notice the volume was already recovered, and the rows for the head of .dat sat past the .dat-tail row -- costing CheckVolumeDataIntegrity its O(1) path and breaking the ascending append order BinarySearchByAppendAtNs assumes. Rewrite .idx as the recovered rows followed by its current contents, through a temp file and a rename. .idx is back in .dat append order, so a later load stops after reading one row. * volume: keep the .idx mode when the repair replaces it The recovery renames a fresh temp file over .idx, so a fixed 0644 (Go) or whatever the umask allows (Rust) would silently widen an index an operator had locked down. Carry the mode off the file being replaced.
This commit is contained in:
Chris Lu
@@ -9,3 +9,4 @@ pub mod store_ec_reconcile;
|
||||
pub mod super_block;
|
||||
pub mod types;
|
||||
pub mod volume;
|
||||
pub mod volume_idx_repair;
|
||||
|
||||
@@ -17,7 +17,7 @@ use std::sync::Arc;
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::warn;
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::storage::idx;
|
||||
@@ -771,6 +771,23 @@ impl Volume {
|
||||
}
|
||||
|
||||
if also_load_index {
|
||||
// Recover rows that deletes on a tiered read-only volume overwrote
|
||||
// at the front of .idx. Best effort: a volume that cannot be
|
||||
// repaired is still servable for everything the surviving rows
|
||||
// index.
|
||||
match self.repair_idx_head_tombstones() {
|
||||
Ok(0) => {}
|
||||
Ok(restored) => info!(
|
||||
volume_id = self.id.0,
|
||||
restored, "recovered overwritten .idx rows from .dat"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
volume_id = self.id.0,
|
||||
error = %e,
|
||||
"recover overwritten .idx rows"
|
||||
),
|
||||
}
|
||||
|
||||
self.load_index()?;
|
||||
|
||||
// Match Go: CheckVolumeDataIntegrity after loading index (volume_loading.go L154-159)
|
||||
@@ -990,7 +1007,11 @@ impl Volume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_exact_at_backend(&self, buf: &mut [u8], offset: u64) -> Result<(), VolumeError> {
|
||||
pub(crate) fn read_exact_at_backend(
|
||||
&self,
|
||||
buf: &mut [u8],
|
||||
offset: u64,
|
||||
) -> Result<(), VolumeError> {
|
||||
if let Some(dat_file) = self.dat_file.as_ref() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
//! Recovery for .idx rows that deletes on a tiered read-only volume
|
||||
//! overwrote. Mirrors `weed/storage/volume_idx_repair.go`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, BufReader, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use tracing::info;
|
||||
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::needle_body_length;
|
||||
use crate::storage::needle::Needle;
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume::{fsync_dir, Volume, VolumeError};
|
||||
|
||||
/// Needles found in the head of .dat, keyed by id, plus the ids in .dat order.
|
||||
type DatHeadNeedles = (HashMap<NeedleId, (Offset, Size)>, Vec<NeedleId>);
|
||||
|
||||
impl Volume {
|
||||
/// Restore the .idx rows that deletes on a tiered read-only volume used to
|
||||
/// overwrite.
|
||||
///
|
||||
/// The sorted-file needle map opened .idx read-write but never seeded its
|
||||
/// write position, so a delete wrote its (key, offset 0, tombstone) row at
|
||||
/// .idx offset 0 and advanced one row at a time instead of appending. Every
|
||||
/// delete therefore replaced one more row at the front of .idx, and the rows
|
||||
/// it replaced -- the Put rows indexing the first needles in .dat -- were
|
||||
/// lost. Reads for those needles return not-found even though .dat still
|
||||
/// holds them intact.
|
||||
///
|
||||
/// The damage leaves a fingerprint: .idx begins with a run of offset-0
|
||||
/// tombstones. A healthy .idx never does. Its first row is the Put for the
|
||||
/// first needle in .dat, and an offset-0 tombstone -- a delete against a
|
||||
/// tiered volume, which appends no .dat record and so has no extent to point
|
||||
/// at -- can only ever land at the tail.
|
||||
///
|
||||
/// .idx and .dat grow in lockstep, so the clobbered rows indexed exactly the
|
||||
/// first N records of .dat. Re-deriving them is a header-only walk over the
|
||||
/// head of .dat, which stays cheap even when .dat is served from a remote
|
||||
/// tier.
|
||||
///
|
||||
/// The recovered rows go back in front, where the rows they replace used to
|
||||
/// sit, and every existing row keeps its relative order behind them. That
|
||||
/// restores .idx to .dat append order, which several readers lean on: the
|
||||
/// fingerprint is gone so a later load stops after one row, and the last row
|
||||
/// is the .dat-tail needle again.
|
||||
pub(crate) fn repair_idx_head_tombstones(&self) -> Result<usize, VolumeError> {
|
||||
if !self.has_data_backend() {
|
||||
return Ok(0);
|
||||
}
|
||||
let version = self.version();
|
||||
if !version.is_supported() {
|
||||
return Ok(0);
|
||||
}
|
||||
let first_needle_offset = self.super_block.block_size() as i64;
|
||||
let idx_path = self.file_name(".idx");
|
||||
|
||||
let clobbered = idx_head_tombstone_count(&idx_path)?;
|
||||
if clobbered == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
info!(
|
||||
volume_id = self.id.0,
|
||||
idx = %idx_path,
|
||||
clobbered,
|
||||
"idx starts with offset-0 tombstones, recovering the rows they overwrote from .dat"
|
||||
);
|
||||
|
||||
let (mut lost, order) = self.scan_dat_head(version, first_needle_offset, clobbered)?;
|
||||
drop_indexed_keys(&idx_path, &mut lost)?;
|
||||
if lost.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut rows = Vec::with_capacity(lost.len() * NEEDLE_MAP_ENTRY_SIZE);
|
||||
let mut restored = 0;
|
||||
for key in order {
|
||||
let Some((offset, size)) = lost.get(&key).copied() else {
|
||||
continue;
|
||||
};
|
||||
idx::write_index_entry(&mut rows, key, offset, size)?;
|
||||
restored += 1;
|
||||
}
|
||||
prepend_idx_rows(&idx_path, &rows)?;
|
||||
Ok(restored)
|
||||
}
|
||||
|
||||
/// Read the headers of the first `limit` records of .dat and return the
|
||||
/// needles they hold, in .dat order. A record with an invalid size is a
|
||||
/// delete marker, so the key it names drops out of the result rather than
|
||||
/// being resurrected.
|
||||
fn scan_dat_head(
|
||||
&self,
|
||||
version: Version,
|
||||
first_needle_offset: i64,
|
||||
limit: usize,
|
||||
) -> Result<DatHeadNeedles, VolumeError> {
|
||||
let mut found: HashMap<NeedleId, (Offset, Size)> = HashMap::new();
|
||||
let mut order: Vec<NeedleId> = Vec::new();
|
||||
let mut offset = first_needle_offset;
|
||||
|
||||
for _ in 0..limit {
|
||||
let mut header = [0u8; NEEDLE_HEADER_SIZE];
|
||||
match self.read_exact_at_backend(&mut header, offset as u64) {
|
||||
Ok(()) => {}
|
||||
Err(VolumeError::Io(ref e)) if e.kind() == io::ErrorKind::UnexpectedEof => break,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
let (_cookie, id, size) = Needle::parse_header(&header);
|
||||
if size.0 == 0 && id.is_empty() {
|
||||
break;
|
||||
}
|
||||
if size.is_valid() {
|
||||
if found
|
||||
.insert(id, (Offset::from_actual_offset(offset), size))
|
||||
.is_none()
|
||||
{
|
||||
order.push(id);
|
||||
}
|
||||
} else {
|
||||
found.remove(&id);
|
||||
}
|
||||
offset += NEEDLE_HEADER_SIZE as i64 + needle_body_length(size, version);
|
||||
}
|
||||
|
||||
Ok((found, order))
|
||||
}
|
||||
}
|
||||
|
||||
/// Report how many rows at the front of .idx are offset-0 tombstones. A healthy
|
||||
/// .idx answers 0 on its first row.
|
||||
fn idx_head_tombstone_count(idx_path: &str) -> Result<usize, VolumeError> {
|
||||
if !Path::new(idx_path).exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut reader = BufReader::new(File::open(idx_path)?);
|
||||
|
||||
let mut clobbered = 0usize;
|
||||
let walk = idx::walk_index_file(&mut reader, 0, |_key, offset, size| {
|
||||
if !offset.is_zero() || !size.is_tombstone() {
|
||||
return Err(stop_walk());
|
||||
}
|
||||
clobbered += 1;
|
||||
Ok(())
|
||||
});
|
||||
finish_walk(walk)?;
|
||||
Ok(clobbered)
|
||||
}
|
||||
|
||||
/// Remove every candidate the .idx already names, whether by a Put row or a
|
||||
/// tombstone. What is left is only what the clobbered rows held.
|
||||
fn drop_indexed_keys(
|
||||
idx_path: &str,
|
||||
candidates: &mut HashMap<NeedleId, (Offset, Size)>,
|
||||
) -> Result<(), VolumeError> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut reader = BufReader::new(File::open(idx_path)?);
|
||||
let walk = idx::walk_index_file(&mut reader, 0, |key, _offset, _size| {
|
||||
candidates.remove(&key);
|
||||
if candidates.is_empty() {
|
||||
return Err(stop_walk());
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
finish_walk(walk)
|
||||
}
|
||||
|
||||
/// Rewrite .idx as `rows` followed by its current contents, through a temp file
|
||||
/// and a rename so a crash mid-write never leaves a partial index at the live
|
||||
/// name.
|
||||
fn prepend_idx_rows(idx_path: &str, rows: &[u8]) -> Result<(), VolumeError> {
|
||||
let tmp_path = format!("{}.tmp", idx_path);
|
||||
let mut src = File::open(idx_path)?;
|
||||
let commit = (|| -> io::Result<()> {
|
||||
let src_perm = src.metadata()?.permissions();
|
||||
let mut dst = OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(&tmp_path)?;
|
||||
// The rename replaces .idx with this file, so it has to carry the mode
|
||||
// the index already had rather than whatever the umask allows.
|
||||
dst.set_permissions(src_perm)?;
|
||||
dst.write_all(rows)?;
|
||||
io::copy(&mut src, &mut dst)?;
|
||||
dst.sync_all()?;
|
||||
drop(dst);
|
||||
fs::rename(&tmp_path, idx_path)?;
|
||||
fsync_dir(idx_path)
|
||||
})();
|
||||
if commit.is_err() {
|
||||
let _ = fs::remove_file(&tmp_path);
|
||||
}
|
||||
Ok(commit?)
|
||||
}
|
||||
|
||||
/// Sentinel that ends an `idx::walk_index_file` early once the answer is known.
|
||||
fn stop_walk() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::Interrupted, "stop idx walk")
|
||||
}
|
||||
|
||||
fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> {
|
||||
match walk {
|
||||
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => Ok(()),
|
||||
other => Ok(other?),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::needle::crc::CRC;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use std::os::unix::fs::{FileExt, PermissionsExt};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn open_volume(dir: &str) -> Volume {
|
||||
Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
"",
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
Version::current(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn write_test_volume(dir: &str, needle_count: u64) -> Vec<Vec<u8>> {
|
||||
let mut v = open_volume(dir);
|
||||
let mut written = Vec::new();
|
||||
for i in 1..=needle_count {
|
||||
let data = format!("needle-{}-payload", i).into_bytes();
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(0x1234_0000 + i as u32),
|
||||
data_size: data.len() as u32,
|
||||
checksum: CRC::new(&data),
|
||||
data: data.clone(),
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
written.push(data);
|
||||
}
|
||||
written
|
||||
}
|
||||
|
||||
/// Reproduce the damage a delete on a tiered read-only volume used to do: it
|
||||
/// writes (key, offset 0, tombstone) rows over the front of .idx instead of
|
||||
/// appending them.
|
||||
fn clobber_idx_head(idx_path: &str, keys: &[u64]) {
|
||||
let file = OpenOptions::new().write(true).open(idx_path).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let mut row = Vec::new();
|
||||
idx::write_index_entry(
|
||||
&mut row,
|
||||
NeedleId(*key),
|
||||
Offset::from_actual_offset(0),
|
||||
TOMBSTONE_FILE_SIZE,
|
||||
)
|
||||
.unwrap();
|
||||
file.write_at(&row, (i * NEEDLE_MAP_ENTRY_SIZE) as u64)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn idx_size(idx_path: &str) -> u64 {
|
||||
std::fs::metadata(idx_path).unwrap().len()
|
||||
}
|
||||
|
||||
fn idx_rows(idx_path: &str) -> Vec<(NeedleId, i64, Size)> {
|
||||
let mut reader = BufReader::new(File::open(idx_path).unwrap());
|
||||
let mut rows = Vec::new();
|
||||
idx::walk_index_file(&mut reader, 0, |key, offset, size| {
|
||||
rows.push((key, offset.to_actual_offset(), size));
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
rows
|
||||
}
|
||||
|
||||
fn read_needle_data(v: &Volume, id: u64) -> Result<Vec<u8>, VolumeError> {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(id),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut n)?;
|
||||
Ok(n.data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repair_idx_head_tombstones_restores_clobbered_rows() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
|
||||
let written = write_test_volume(dir, 12);
|
||||
let size_before = idx_size(&idx_path);
|
||||
|
||||
// The rewrite replaces .idx wholesale, so it must not widen the mode.
|
||||
fs::set_permissions(&idx_path, fs::Permissions::from_mode(0o600)).unwrap();
|
||||
|
||||
// Deletes against needles 9..12 land on the front of .idx and take the
|
||||
// rows indexing needles 1..4 with them.
|
||||
clobber_idx_head(&idx_path, &[9, 10, 11, 12]);
|
||||
|
||||
let v = open_volume(dir);
|
||||
for i in 1..=8u64 {
|
||||
assert_eq!(
|
||||
read_needle_data(&v, i).unwrap(),
|
||||
written[(i - 1) as usize],
|
||||
"needle {} not recovered",
|
||||
i
|
||||
);
|
||||
}
|
||||
|
||||
let want = size_before + 4 * NEEDLE_MAP_ENTRY_SIZE as u64;
|
||||
assert_eq!(idx_size(&idx_path), want, "idx size after recovery");
|
||||
|
||||
assert_eq!(
|
||||
fs::metadata(&idx_path).unwrap().permissions().mode() & 0o777,
|
||||
0o600,
|
||||
"idx mode after recovery"
|
||||
);
|
||||
|
||||
// The recovered rows go back in front, so .idx is in .dat append order
|
||||
// again: the fingerprint is gone and the last row is still the .dat tail.
|
||||
let rows = idx_rows(&idx_path);
|
||||
assert_eq!(
|
||||
rows[0].1,
|
||||
v.super_block.block_size() as i64,
|
||||
"first row does not index the first needle in .dat"
|
||||
);
|
||||
for i in 1..4 {
|
||||
assert!(
|
||||
rows[i - 1].1 < rows[i].1,
|
||||
"recovered rows are not in .dat order: {:?} then {:?}",
|
||||
rows[i - 1],
|
||||
rows[i]
|
||||
);
|
||||
}
|
||||
|
||||
// The recovery is idempotent: a second load finds nothing left to restore.
|
||||
drop(v);
|
||||
let _v = open_volume(dir);
|
||||
assert_eq!(idx_size(&idx_path), want, "idx grew on second load");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repair_idx_head_tombstones_leaves_healthy_idx_alone() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
|
||||
write_test_volume(dir, 6);
|
||||
let mut v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
let size_before = idx_size(&idx_path);
|
||||
let v = open_volume(dir);
|
||||
|
||||
assert_eq!(
|
||||
idx_size(&idx_path),
|
||||
size_before,
|
||||
"healthy idx was rewritten"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
read_needle_data(&v, 3),
|
||||
Err(VolumeError::Deleted) | Err(VolumeError::NotFound)
|
||||
),
|
||||
"needle 3 should stay deleted"
|
||||
);
|
||||
}
|
||||
|
||||
/// A needle deleted before the damage must not be resurrected: its tombstone
|
||||
/// row survives at the tail of .idx, so the key is still indexed and stays
|
||||
/// out of the recovery.
|
||||
#[test]
|
||||
fn test_repair_idx_head_tombstones_keeps_deleted_needles_deleted() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
|
||||
write_test_volume(dir, 8);
|
||||
let mut v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
clobber_idx_head(&idx_path, &[7, 8]);
|
||||
|
||||
let v = open_volume(dir);
|
||||
assert!(
|
||||
matches!(
|
||||
read_needle_data(&v, 2),
|
||||
Err(VolumeError::Deleted) | Err(VolumeError::NotFound)
|
||||
),
|
||||
"needle 2 was resurrected"
|
||||
);
|
||||
assert!(
|
||||
read_needle_data(&v, 1).is_ok(),
|
||||
"needle 1 should have been recovered"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// errStopIdxWalk ends an idx.WalkIndexFile early once the answer is known.
|
||||
var errStopIdxWalk = errors.New("stop idx walk")
|
||||
|
||||
// repairIdxHeadTombstones restores the .idx rows that deletes on a tiered
|
||||
// read-only volume used to overwrite.
|
||||
//
|
||||
// The sorted-file needle map opened .idx read-write but never seeded its write
|
||||
// position, so a delete wrote its (key, offset 0, tombstone) row at .idx offset
|
||||
// 0 and advanced one row at a time instead of appending. Every delete therefore
|
||||
// replaced one more row at the front of .idx, and the rows it replaced -- the
|
||||
// Put rows indexing the first needles in .dat -- were lost. Reads for those
|
||||
// needles return not-found even though .dat still holds them intact.
|
||||
//
|
||||
// The damage leaves a fingerprint: .idx begins with a run of offset-0
|
||||
// tombstones. A healthy .idx never does. Its first row is the Put for the first
|
||||
// needle in .dat, and an offset-0 tombstone -- a delete against a tiered volume,
|
||||
// which appends no .dat record and so has no extent to point at -- can only ever
|
||||
// land at the tail.
|
||||
//
|
||||
// .idx and .dat grow in lockstep, so the clobbered rows indexed exactly the
|
||||
// first N records of .dat. Re-deriving them is a header-only walk over the head
|
||||
// of .dat, which stays cheap even when .dat is served from a remote tier.
|
||||
//
|
||||
// The recovered rows go back in front, where the rows they replace used to sit,
|
||||
// and every existing row keeps its relative order behind them. That restores
|
||||
// .idx to .dat append order, which several readers lean on: the fingerprint is
|
||||
// gone so a later load stops after one row, the last row is the .dat-tail needle
|
||||
// again so CheckVolumeDataIntegrity keeps its O(1) path, and
|
||||
// BinarySearchByAppendAtNs sees ascending append timestamps.
|
||||
func (v *Volume) repairIdxHeadTombstones() (restored int, err error) {
|
||||
if v.DataBackend == nil {
|
||||
return 0, nil
|
||||
}
|
||||
version := v.Version()
|
||||
if !needle.IsSupportedVersion(version) {
|
||||
return 0, nil
|
||||
}
|
||||
firstNeedleOffset := int64(v.SuperBlock.BlockSize())
|
||||
idxFileName := v.FileName(".idx")
|
||||
|
||||
clobbered, err := idxHeadTombstoneCount(idxFileName)
|
||||
if err != nil || clobbered == 0 {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
glog.V(0).Infof("volume %d: %s starts with %d offset-0 tombstones, recovering the .idx rows they overwrote from %s",
|
||||
v.Id, idxFileName, clobbered, v.FileName(".dat"))
|
||||
|
||||
lost, order, err := scanDatHead(v.DataBackend, version, firstNeedleOffset, clobbered)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err = dropIndexedKeys(idxFileName, lost); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(lost) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var rows []byte
|
||||
for _, key := range order {
|
||||
nv, ok := lost[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, nv.ToBytes()...)
|
||||
restored++
|
||||
}
|
||||
return restored, prependIdxRows(idxFileName, rows)
|
||||
}
|
||||
|
||||
// idxHeadTombstoneCount reports how many rows at the front of .idx are offset-0
|
||||
// tombstones. A healthy .idx answers 0 on its first row.
|
||||
func idxHeadTombstoneCount(idxFileName string) (clobbered int, err error) {
|
||||
idxFile, err := os.Open(idxFileName)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
defer idxFile.Close()
|
||||
|
||||
err = idx.WalkIndexFile(idxFile, 0, func(_ types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
if !offset.IsZero() || !size.IsTombstone() {
|
||||
return errStopIdxWalk
|
||||
}
|
||||
clobbered++
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, errStopIdxWalk) {
|
||||
err = nil
|
||||
}
|
||||
return clobbered, err
|
||||
}
|
||||
|
||||
// scanDatHead reads the headers of the first limit records of .dat and returns
|
||||
// the needles they hold, in .dat order. A record with an invalid size is a
|
||||
// delete marker, so the key it names drops out of the result rather than being
|
||||
// resurrected.
|
||||
func scanDatHead(datBackend backend.BackendStorageFile, version needle.Version, firstNeedleOffset int64, limit int) (map[types.NeedleId]needle_map.NeedleValue, []types.NeedleId, error) {
|
||||
scanner := &datHeadScanner{
|
||||
limit: limit,
|
||||
found: make(map[types.NeedleId]needle_map.NeedleValue),
|
||||
}
|
||||
if err := ScanVolumeFileFrom(version, datBackend, firstNeedleOffset, scanner); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan head of %s: %w", datBackend.Name(), err)
|
||||
}
|
||||
return scanner.found, scanner.order, nil
|
||||
}
|
||||
|
||||
type datHeadScanner struct {
|
||||
limit int
|
||||
visited int
|
||||
found map[types.NeedleId]needle_map.NeedleValue
|
||||
order []types.NeedleId
|
||||
}
|
||||
|
||||
func (s *datHeadScanner) VisitSuperBlock(super_block.SuperBlock) error { return nil }
|
||||
|
||||
func (s *datHeadScanner) ReadNeedleBody() bool { return false }
|
||||
|
||||
func (s *datHeadScanner) VisitNeedle(n *needle.Needle, offset int64, _, _ []byte) error {
|
||||
if n.Size.IsValid() {
|
||||
if _, seen := s.found[n.Id]; !seen {
|
||||
s.order = append(s.order, n.Id)
|
||||
}
|
||||
s.found[n.Id] = needle_map.NeedleValue{Key: n.Id, Offset: types.ToOffset(offset), Size: n.Size}
|
||||
} else {
|
||||
delete(s.found, n.Id)
|
||||
}
|
||||
s.visited++
|
||||
if s.visited >= s.limit {
|
||||
return io.EOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropIndexedKeys removes every candidate the .idx already names, whether by a
|
||||
// Put row or a tombstone. What is left is only what the clobbered rows held.
|
||||
func dropIndexedKeys(idxFileName string, candidates map[types.NeedleId]needle_map.NeedleValue) error {
|
||||
if len(candidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
idxFile, err := os.Open(idxFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer idxFile.Close()
|
||||
|
||||
err = idx.WalkIndexFile(idxFile, 0, func(key types.NeedleId, _ types.Offset, _ types.Size) error {
|
||||
delete(candidates, key)
|
||||
if len(candidates) == 0 {
|
||||
return errStopIdxWalk
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, errStopIdxWalk) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// prependIdxRows rewrites .idx as rows followed by its current contents,
|
||||
// through a temp file and a rename so a crash mid-write never leaves a partial
|
||||
// index at the live name.
|
||||
func prependIdxRows(idxFileName string, rows []byte) error {
|
||||
src, err := os.Open(idxFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
srcStat, err := src.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpFileName := idxFileName + ".tmp"
|
||||
dst, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcStat.Mode().Perm())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
dst.Close()
|
||||
if !committed {
|
||||
os.Remove(tmpFileName)
|
||||
}
|
||||
}()
|
||||
|
||||
// The rename replaces .idx with this file, so it has to carry the mode the
|
||||
// index already had -- O_CREATE alone leaves it at the umask's mercy.
|
||||
if err = dst.Chmod(srcStat.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = dst.Write(rows); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = io.Copy(dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = dst.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = dst.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(tmpFileName, idxFileName); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = util.FsyncDir(filepath.Dir(idxFileName)); err != nil {
|
||||
return err
|
||||
}
|
||||
committed = true
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
)
|
||||
|
||||
// clobberIdxHead reproduces the damage a delete on a tiered read-only volume
|
||||
// used to do: it writes (key, offset 0, tombstone) rows over the front of .idx
|
||||
// instead of appending them.
|
||||
func clobberIdxHead(t *testing.T, idxPath string, keys []uint64) {
|
||||
t.Helper()
|
||||
f, err := os.OpenFile(idxPath, os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("open idx: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
for i, key := range keys {
|
||||
row := needle_map.ToBytes(types.Uint64ToNeedleId(key), types.Offset{}, types.TombstoneFileSize)
|
||||
if _, err := f.WriteAt(row, int64(i*types.NeedleMapEntrySize)); err != nil {
|
||||
t.Fatalf("clobber row %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestVolume(t *testing.T, dir string, needleCount int) map[uint64]*needle.Needle {
|
||||
t.Helper()
|
||||
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("volume creation: %v", err)
|
||||
}
|
||||
written := make(map[uint64]*needle.Needle)
|
||||
for i := 1; i <= needleCount; i++ {
|
||||
n := newRandomNeedle(uint64(i))
|
||||
if _, _, _, err := v.writeNeedle2(n, true, false); err != nil {
|
||||
v.Close()
|
||||
t.Fatalf("write needle %d: %v", i, err)
|
||||
}
|
||||
written[uint64(i)] = n
|
||||
}
|
||||
v.Close()
|
||||
return written
|
||||
}
|
||||
|
||||
func mustReload(t *testing.T, dir string) *Volume {
|
||||
t.Helper()
|
||||
v, err := loadVolumeWithoutWorker(dir, dir, "", 1, NeedleMapInMemory, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("reload volume: %v", err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func mustRead(t *testing.T, v *Volume, id uint64) []byte {
|
||||
t.Helper()
|
||||
n := newEmptyNeedle(id)
|
||||
if _, err := v.readNeedle(n, nil, nil); err != nil {
|
||||
t.Fatalf("read needle %d: %v", id, err)
|
||||
}
|
||||
return n.Data
|
||||
}
|
||||
|
||||
// TestRepairIdxHeadTombstones_RestoresClobberedRows checks that a volume whose
|
||||
// .idx head was overwritten with offset-0 tombstones serves its first needles
|
||||
// again after a reload, without a full weed fix rebuild.
|
||||
func TestRepairIdxHeadTombstones_RestoresClobberedRows(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
const needleCount = 12
|
||||
const clobbered = 4
|
||||
|
||||
written := writeTestVolume(t, dir, needleCount)
|
||||
idxPath := filepath.Join(dir, "1.idx")
|
||||
sizeBefore := fileSize(t, idxPath)
|
||||
|
||||
// The rewrite replaces .idx wholesale, so it must not widen the mode.
|
||||
if err := os.Chmod(idxPath, 0600); err != nil {
|
||||
t.Fatalf("chmod idx: %v", err)
|
||||
}
|
||||
|
||||
// Deletes against needles 9..12 land on the front of .idx and take the
|
||||
// rows indexing needles 1..4 with them.
|
||||
clobberIdxHead(t, idxPath, []uint64{9, 10, 11, 12})
|
||||
|
||||
v := mustReload(t, dir)
|
||||
defer v.Close()
|
||||
|
||||
for i := uint64(1); i <= needleCount; i++ {
|
||||
if i >= 9 {
|
||||
// genuinely deleted by the tombstones that did the damage
|
||||
continue
|
||||
}
|
||||
if got, want := mustRead(t, v, i), written[i].Data; string(got) != string(want) {
|
||||
t.Fatalf("needle %d: data mismatch after recovery", i)
|
||||
}
|
||||
}
|
||||
|
||||
if got, want := fileSize(t, idxPath), sizeBefore+int64(clobbered*types.NeedleMapEntrySize); got != want {
|
||||
t.Fatalf("idx size after recovery: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if st, err := os.Stat(idxPath); err != nil {
|
||||
t.Fatalf("stat idx: %v", err)
|
||||
} else if got := st.Mode().Perm(); got != 0600 {
|
||||
t.Fatalf("idx mode after recovery: got %o, want 600", got)
|
||||
}
|
||||
|
||||
// The recovered rows go back in front, so .idx is in .dat append order
|
||||
// again: the fingerprint is gone and the last row is still the .dat tail.
|
||||
entries := readAllIdxEntries(t, idxPath)
|
||||
if entries[0].offset.ToActualOffset() != int64(v.SuperBlock.BlockSize()) {
|
||||
t.Fatalf("first row does not index the first needle in .dat: %+v", entries[0])
|
||||
}
|
||||
for i := 1; i < clobbered; i++ {
|
||||
if entries[i-1].offset.ToActualOffset() >= entries[i].offset.ToActualOffset() {
|
||||
t.Fatalf("recovered rows are not in .dat order: %+v then %+v", entries[i-1], entries[i])
|
||||
}
|
||||
}
|
||||
|
||||
// The recovery is idempotent: a second load finds nothing left to restore.
|
||||
v.Close()
|
||||
v = mustReload(t, dir)
|
||||
if got, want := fileSize(t, idxPath), sizeBefore+int64(clobbered*types.NeedleMapEntrySize); got != want {
|
||||
t.Fatalf("idx grew on second load: got %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepairIdxHeadTombstones_ReadOnlyVolume covers the shape the damage
|
||||
// actually occurs in: a read-only volume, whose reads go through the sorted
|
||||
// needle map. The recovery has to land before .sdx is regenerated, or the
|
||||
// restored rows never reach the map that answers the read.
|
||||
func TestRepairIdxHeadTombstones_ReadOnlyVolume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
written := writeTestVolume(t, dir, 8)
|
||||
clobberIdxHead(t, filepath.Join(dir, "1.idx"), []uint64{7, 8})
|
||||
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(dir, "1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
ReadOnly: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("save vif: %v", err)
|
||||
}
|
||||
|
||||
v := mustReload(t, dir)
|
||||
defer v.Close()
|
||||
|
||||
if _, isSorted := v.nm.(*SortedFileNeedleMap); !isSorted {
|
||||
t.Fatalf("expected a sorted needle map, got %T", v.nm)
|
||||
}
|
||||
for i := uint64(1); i <= 2; i++ {
|
||||
if got, want := mustRead(t, v, i), written[i].Data; string(got) != string(want) {
|
||||
t.Fatalf("needle %d: data mismatch after recovery", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepairIdxHeadTombstones_LeavesHealthyIdxAlone guards the fingerprint: a
|
||||
// volume with ordinary deletes must not be rewritten.
|
||||
func TestRepairIdxHeadTombstones_LeavesHealthyIdxAlone(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeTestVolume(t, dir, 6)
|
||||
idxPath := filepath.Join(dir, "1.idx")
|
||||
|
||||
v := mustReload(t, dir)
|
||||
if _, err := v.doDeleteRequest(newEmptyNeedle(3)); err != nil {
|
||||
t.Fatalf("delete needle 3: %v", err)
|
||||
}
|
||||
v.Close()
|
||||
|
||||
sizeBefore := fileSize(t, idxPath)
|
||||
entriesBefore := readAllIdxEntries(t, idxPath)
|
||||
|
||||
v = mustReload(t, dir)
|
||||
defer v.Close()
|
||||
|
||||
if got := fileSize(t, idxPath); got != sizeBefore {
|
||||
t.Fatalf("healthy idx was rewritten: got %d, want %d", got, sizeBefore)
|
||||
}
|
||||
if got := readAllIdxEntries(t, idxPath); len(got) != len(entriesBefore) {
|
||||
t.Fatalf("healthy idx row count changed: got %d, want %d", len(got), len(entriesBefore))
|
||||
}
|
||||
if _, err := v.readNeedle(newEmptyNeedle(3), nil, nil); err != ErrorDeleted {
|
||||
t.Fatalf("needle 3 should stay deleted, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted checks that a needle
|
||||
// deleted before the damage is not resurrected: its tombstone row survives at
|
||||
// the tail of .idx, so the key is still indexed and stays out of the recovery.
|
||||
// It stays unreadable either way -- as deleted if its Put row survived, as
|
||||
// not-found if the tombstone is all the .idx has left of it.
|
||||
func TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeTestVolume(t, dir, 8)
|
||||
idxPath := filepath.Join(dir, "1.idx")
|
||||
|
||||
v := mustReload(t, dir)
|
||||
if _, err := v.doDeleteRequest(newEmptyNeedle(2)); err != nil {
|
||||
t.Fatalf("delete needle 2: %v", err)
|
||||
}
|
||||
v.Close()
|
||||
|
||||
clobberIdxHead(t, idxPath, []uint64{7, 8})
|
||||
|
||||
v = mustReload(t, dir)
|
||||
defer v.Close()
|
||||
|
||||
if _, err := v.readNeedle(newEmptyNeedle(2), nil, nil); err != ErrorDeleted && err != ErrorNotFound {
|
||||
t.Fatalf("needle 2 was resurrected, got %v", err)
|
||||
}
|
||||
if _, err := v.readNeedle(newEmptyNeedle(1), nil, nil); err != nil {
|
||||
t.Fatalf("needle 1 should have been recovered: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -252,6 +252,14 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
|
||||
}
|
||||
glog.Fatalf("check volume idx file %s: %v", v.FileName(".idx"), err)
|
||||
}
|
||||
// Recover rows that deletes on a tiered read-only volume overwrote at
|
||||
// the front of .idx. Best effort: a volume that cannot be repaired is
|
||||
// still servable for everything the surviving rows index.
|
||||
if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil {
|
||||
glog.Warningf("volume %d: recover overwritten %s rows: %v", v.Id, v.FileName(".idx"), repairErr)
|
||||
} else if restored > 0 {
|
||||
glog.V(0).Infof("volume %d: recovered %d overwritten %s rows from %s", v.Id, restored, v.FileName(".idx"), v.FileName(".dat"))
|
||||
}
|
||||
var indexFile *os.File
|
||||
if v.noWriteOrDelete {
|
||||
glog.V(0).Infoln("open to read file", v.FileName(".idx"))
|
||||
|
||||
Reference in New Issue
Block a user