From 3e558c15e9ca0c1df9e81de6ca80fd04e0f222d4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 30 Jun 2026 03:52:13 -0700 Subject: [PATCH] proto: add EC bitrot checksum messages + CHECKSUM scrub mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror weed/pb/volume_server.proto byte-for-byte (field numbers + types) so the .ecsum sidecar payload is wire-identical across the Go and Rust binaries: EcBitrotProtection / EcShardChecksums / ChecksumAlgorithm, VolumeScrubMode.CHECKSUM=4, and VolumeEcShardsCopyRequest.copy_ecsum_file. No code uses them yet — the .ecsum format, producer, mount-load, copy, and scrub land in following commits. Claude-Session: https://claude.ai/code/session_015EE9Sc9EvNp8BCVva4RKdo --- seaweed-volume/proto/volume_server.proto | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto index 8db9f15e6..da1b941bd 100644 --- a/seaweed-volume/proto/volume_server.proto +++ b/seaweed-volume/proto/volume_server.proto @@ -442,6 +442,7 @@ message VolumeEcShardsCopyRequest { bool copy_ecj_file = 6; bool copy_vif_file = 7; uint32 disk_id = 8; // Target disk ID for storing EC shards + bool copy_ecsum_file = 9; // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop } message VolumeEcShardsCopyResponse { } @@ -587,6 +588,29 @@ message EcShardConfig { uint32 parity_shards = 2; // Number of parity shards (e.g., 4) int64 encode_ts_ns = 3; // encode time (unix nanos); a read served from a shard of a different encode run is rejected } +// EcBitrotProtection is the entire content of a bitrot checksum sidecar +// (.ecsum for the legacy generation, .ecsum.v for vacuum +// generation N): per-shard, per-block CRC32C so a CHECKSUM scrub can detect +// silent bit rot in any shard (including parity) without decoding. Field +// numbers and types match weed/pb/volume_server.proto byte-for-byte so the +// serialized sidecar payload is wire-identical across the Go and Rust binaries. +message EcBitrotProtection { + ChecksumAlgorithm algorithm = 1; // CRC32C (Castagnoli) + uint32 block_size = 2; // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB + uint32 generation = 3; // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version + EcShardConfig ec_shard_config = 4; // data/parity shard counts at encode time + repeated EcShardChecksums shards = 5; // one entry per shard id in the active layout + bytes encode_uuid = 6; // random per-encode identity, for stale-sidecar detection across in-place re-encodes +} +message EcShardChecksums { + uint32 shard_id = 1; // 0..MaxShardCount-1 (custom EC ratios go up to 32) + int64 covered_size = 2; // shard byte length these checksums cover (must equal the on-disk shard length) + bytes block_crc32c = 3; // packed little-endian uint32[] = ceil(covered_size/block_size) entries +} +enum ChecksumAlgorithm { + CHECKSUM_NONE = 0; + CHECKSUM_CRC32C = 1; +} message OldVersionVolumeInfo { repeated RemoteFile files = 1; uint32 version = 2; @@ -664,6 +688,7 @@ enum VolumeScrubMode { INDEX = 1; FULL = 2; LOCAL = 3; + CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar } message ScrubVolumeRequest {