From 6f816c955dd8ff346bce9401367f485b032b2c07 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 9 Jul 2026 11:19:31 -0700 Subject: [PATCH] volume.fsck: fix orphan purge against the rust volume server (#10289) * rust volume: accept odd-length needle id hex in file ids Go formats the needle id with strconv.FormatUint and parses it back with strconv.ParseUint, neither of which pads to an even number of hex digits. hex::decode rejected such file ids with "Odd number of digits", so volume.fsck could not purge orphans from a rust volume server. Parse the needle id and cookie with from_str_radix, matching Go's ParseNeedleId and ParseCookie. * storage: emit even-length needle id hex in NeedleId.FileId volume.fsck and volume.check_disk build purge file ids here with unpadded FormatUint hex, while every other fid formatter strips whole leading zero bytes. Pad to even length so the output matches the canonical fid format and strict hex parsers accept it. --- seaweed-volume/src/storage/needle/needle.rs | 35 +++++++++++++-------- weed/storage/needle/file_id_test.go | 26 +++++++++++++++ weed/storage/types/needle_id_type.go | 7 ++++- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/seaweed-volume/src/storage/needle/needle.rs b/seaweed-volume/src/storage/needle/needle.rs index bbc55c9d0..7df518e9d 100644 --- a/seaweed-volume/src/storage/needle/needle.rs +++ b/seaweed-volume/src/storage/needle/needle.rs @@ -743,19 +743,14 @@ pub fn parse_needle_id_cookie(s: &str) -> Result<(NeedleId, Cookie), String> { let needle_id_hex = &hex_part[..split]; let cookie_hex = &hex_part[split..]; - let needle_id_bytes = hex::decode(needle_id_hex).map_err(|e| format!("Parse needleId error: {}", e))?; - let cookie_bytes = hex::decode(cookie_hex).map_err(|e| format!("Parse cookie error: {}", e))?; - - // Pad needle id to 8 bytes - let mut nid_buf = [0u8; 8]; - if needle_id_bytes.len() > 8 { - return Err(format!("KeyHash is too long.")); - } - let start = 8 - needle_id_bytes.len(); - nid_buf[start..].copy_from_slice(&needle_id_bytes); - - let mut key = NeedleId::from_bytes(&nid_buf[0..8]); - let cookie = Cookie::from_bytes(&cookie_bytes[0..4]); + // Go parses both with strconv.ParseUint, which accepts odd-length hex + let mut key = NeedleId( + u64::from_str_radix(needle_id_hex, 16) + .map_err(|e| format!("Parse needleId error: {}", e))?, + ); + let cookie = Cookie( + u32::from_str_radix(cookie_hex, 16).map_err(|e| format!("Parse cookie error: {}", e))?, + ); // Apply delta if present (Go: n.Id += Uint64ToNeedleId(d)) if let Some(d) = delta { @@ -941,4 +936,18 @@ mod tests { assert_eq!(key, NeedleId(1)); assert_eq!(cookie, Cookie(0x12345678)); } + + #[test] + fn test_parse_odd_length_needle_id() { + // Go's shell formats purge fids with strconv.FormatUint, which does not + // pad the needle id hex to an even length. + let (key, cookie) = parse_needle_id_cookie("100000000").unwrap(); + assert_eq!(key, NeedleId(1)); + assert_eq!(cookie, Cookie(0)); + + let fid = FileId::parse("3,12300000000").unwrap(); + assert_eq!(fid.volume_id, VolumeId(3)); + assert_eq!(fid.key, NeedleId(0x123)); + assert_eq!(fid.cookie, Cookie(0)); + } } diff --git a/weed/storage/needle/file_id_test.go b/weed/storage/needle/file_id_test.go index 191c4aa96..f3f3f226f 100644 --- a/weed/storage/needle/file_id_test.go +++ b/weed/storage/needle/file_id_test.go @@ -54,3 +54,29 @@ func TestParseFileIdFromString(t *testing.T) { t.Errorf("%s : needleId is too long", fidStr1) } } + +func TestNeedleIdFileId(t *testing.T) { + tests := []struct { + key types.NeedleId + volumeId uint32 + want string + }{ + {types.NeedleId(1), 3, "3,0100000000"}, + {types.NeedleId(0x123), 3, "3,012300000000"}, + {types.NeedleId(0x1234), 7, "7,123400000000"}, + } + for _, tt := range tests { + fid := tt.key.FileId(tt.volumeId) + if fid != tt.want { + t.Errorf("NeedleId(%d).FileId(%d) = %s, want %s", tt.key, tt.volumeId, fid, tt.want) + } + fileId, err := ParseFileIdFromString(fid) + if err != nil { + t.Errorf("%s : should be OK: %v", fid, err) + continue + } + if fileId.VolumeId != VolumeId(tt.volumeId) || fileId.Key != tt.key || fileId.Cookie != 0 { + t.Errorf("src : %s, dest : %v", fid, fileId) + } + } +} diff --git a/weed/storage/types/needle_id_type.go b/weed/storage/types/needle_id_type.go index a927b6e3c..8ac3cc7ff 100644 --- a/weed/storage/types/needle_id_type.go +++ b/weed/storage/types/needle_id_type.go @@ -36,7 +36,12 @@ func (k NeedleId) String() string { } func (k NeedleId) FileId(volumeId uint32) string { - return fmt.Sprintf("%d,%s00000000", volumeId, k.String()) + // pad to even length so strict hex fid parsers accept it + needleIdHex := k.String() + if len(needleIdHex)%2 == 1 { + needleIdHex = "0" + needleIdHex + } + return fmt.Sprintf("%d,%s00000000", volumeId, needleIdHex) } func ParseNeedleId(idString string) (NeedleId, error) {