mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-06 00:04:02 +00:00
feat(filer.sync.verify): reclassify chunk-slice-order ETag diffs as CHUNK_REORDER (#10177)
* feat(filer.sync.verify): reclassify chunk-slice-order ETag diffs as CHUNK_REORDER filer.ETagChunks concatenates per-chunk MD5s in stored slice order without normalising by offset, so byte-identical content written by two different paths (S3 multipart part-completion order on the source vs filer.backup replication arrival order on the destination) yields different file ETags. filer.sync.verify reported these as ETAG_MISMATCH even though the files are equal. Add a second-pass check on every ETAG_MISMATCH: when both sides derive their ETag from chunks (no attr.Md5) and hold a manifest-free, non-overlapping chunk set that, once sorted by offset, matches element-wise on (offset, size, ETag), classify the file as CHUNK_REORDER. Such files are content-equal, so they are not counted as errors and do not affect the exit code; they are listed only at higher verbosity (weed -v=1), while the summary always shows their count. The check stays conservative: a stored attr.Md5 (order-independent content hash), a differing chunk count, an overlapping/duplicate offset (whose visible bytes are resolved by timestamp), or a manifest chunk all remain ETAG_MISMATCH. * filer.sync.verify: decline chunk-reorder fast path on empty per-chunk ETag An empty or undecodable per-chunk ETag is not a content fingerprint, so element-wise (offset, size, ETag) equality can't prove the bytes match. Treating "" == "" as content-equal could reclassify a genuine divergence as CHUNK_REORDER and drop it from the error count. Decline such chunk sets so they stay ETAG_MISMATCH. * filer.sync.verify: emit CHUNK_REORDER in JSON output regardless of -v The -v=1 gate belongs to the human text report only. Applying it before the jsonOutput branch dropped the per-file CHUNK_REORDER records from NDJSON while the summary still counted them, so a machine consumer saw a non-zero count with no records to reconcile it. Gate the text path only; JSON always emits. --------- Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
@@ -58,6 +58,12 @@ var cmdFilerSyncVerify = &Command{
|
||||
-modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and
|
||||
-isActivePassive for unidirectional comparison.
|
||||
|
||||
A chunk-derived ETag (no stored attr.Md5) can differ between clusters only
|
||||
because the chunk slice was assembled in a different order, since ETagChunks
|
||||
does not sort by offset. Such files are byte-identical and reported as
|
||||
CHUNK_REORDER: not counted as errors, always counted in the summary, listed
|
||||
only at -v=1.
|
||||
|
||||
Exits with code 0 on agreement, 2 on differences or operational errors.
|
||||
|
||||
`,
|
||||
@@ -123,6 +129,7 @@ type VerifyResult struct {
|
||||
missingCount atomic.Int64
|
||||
sizeMismatch atomic.Int64
|
||||
etagMismatch atomic.Int64
|
||||
chunkReorder atomic.Int64 // content-equal; chunks differ only in slice order
|
||||
onlyInB atomic.Int64
|
||||
skippedRecent atomic.Int64
|
||||
|
||||
@@ -139,6 +146,7 @@ const (
|
||||
diffOnlyInB // in B but not in A
|
||||
diffSizeMismatch // size differs
|
||||
diffETagMismatch // etag differs
|
||||
diffChunkReorder // etag differs but content is equal (chunk slice order only)
|
||||
)
|
||||
|
||||
// diffRecord is the JSON Lines schema for a single diff entry.
|
||||
@@ -167,6 +175,7 @@ type summaryRecord struct {
|
||||
Missing int64 `json:"missing"`
|
||||
SizeMismatch int64 `json:"sizeMismatch"`
|
||||
ETagMismatch int64 `json:"etagMismatch"`
|
||||
ChunkReorder int64 `json:"chunkReorder"`
|
||||
OnlyInB int64 `json:"onlyInB"`
|
||||
TotalErrors int64 `json:"totalErrors"`
|
||||
}
|
||||
@@ -236,6 +245,7 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
|
||||
Missing: result.missingCount.Load(),
|
||||
SizeMismatch: result.sizeMismatch.Load(),
|
||||
ETagMismatch: result.etagMismatch.Load(),
|
||||
ChunkReorder: result.chunkReorder.Load(),
|
||||
OnlyInB: result.onlyInB.Load(),
|
||||
TotalErrors: totalErrors,
|
||||
}
|
||||
@@ -250,6 +260,9 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
|
||||
fmt.Fprintf(os.Stdout, " Missing in B: %d\n", result.missingCount.Load())
|
||||
fmt.Fprintf(os.Stdout, " Size mismatch: %d\n", result.sizeMismatch.Load())
|
||||
fmt.Fprintf(os.Stdout, " ETag mismatch: %d\n", result.etagMismatch.Load())
|
||||
if n := result.chunkReorder.Load(); n > 0 {
|
||||
fmt.Fprintf(os.Stdout, " Chunk reorder (equal): %d (content-equal, not an error; use -v=1 to list)\n", n)
|
||||
}
|
||||
if !isActivePassive {
|
||||
fmt.Fprintf(os.Stdout, " Only in B: %d\n", result.onlyInB.Load())
|
||||
}
|
||||
@@ -496,11 +509,88 @@ func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyRe
|
||||
etagA := filer.ETag(entryA)
|
||||
etagB := filer.ETag(entryB)
|
||||
if etagA != etagB {
|
||||
reportDiff(diffETagMismatch, dir, entryA, entryB, result)
|
||||
if isChunkReorder(entryA, entryB) {
|
||||
// Same bytes, just a different chunk slice order — not a real diff.
|
||||
reportDiff(diffChunkReorder, dir, entryA, entryB, result)
|
||||
} else {
|
||||
reportDiff(diffETagMismatch, dir, entryA, entryB, result)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// isChunkReorder reports whether two entries with differing file ETags are
|
||||
// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes
|
||||
// per-chunk MD5s in stored order without sorting by offset, so the same bytes
|
||||
// written by different paths (S3 multipart vs filer.backup) can hash
|
||||
// differently. If both chunk lists sorted by offset match element-wise on
|
||||
// (offset, size, ETag), the files cover the same bytes with the same content
|
||||
// and only the slice order differed.
|
||||
//
|
||||
// Applies only when both ETags come from chunks (a stored attr.Md5 is
|
||||
// order-independent, so a mismatch there is a real content difference) and the
|
||||
// chunks form a manifest-free, non-overlapping set whose bytes are fully
|
||||
// determined by the list — overlaps would need timestamp-based visible-interval
|
||||
// resolution, so those stay ETAG_MISMATCH.
|
||||
func isChunkReorder(entryA, entryB *filer_pb.Entry) bool {
|
||||
if !usesChunkETag(entryA) || !usesChunkETag(entryB) {
|
||||
return false
|
||||
}
|
||||
a := sortedSimpleChunks(entryA.GetChunks())
|
||||
b := sortedSimpleChunks(entryB.GetChunks())
|
||||
// nil means overlapping or manifest chunks we do not fast-path; a single
|
||||
// chunk hashes directly and a differing count is a real divergence.
|
||||
if a == nil || b == nil || len(a) != len(b) || len(a) < 2 {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry,
|
||||
// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's
|
||||
// own condition so it agrees with how the compared ETag was computed.
|
||||
func usesChunkETag(entry *filer_pb.Entry) bool {
|
||||
return entry.Attributes == nil || entry.Attributes.Md5 == nil
|
||||
}
|
||||
|
||||
// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a
|
||||
// manifest-free, non-overlapping set whose visible bytes are fully determined
|
||||
// by the list and every chunk carries a usable per-chunk ETag; otherwise nil.
|
||||
// Overlapping ranges would require resolving the visible interval by chunk
|
||||
// timestamp, which this fast path does not attempt.
|
||||
func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
|
||||
sorted := make([]*filer_pb.FileChunk, len(chunks))
|
||||
copy(sorted, chunks)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].Offset != sorted[j].Offset {
|
||||
return sorted[i].Offset < sorted[j].Offset
|
||||
}
|
||||
return sorted[i].ETag < sorted[j].ETag
|
||||
})
|
||||
prevEnd := int64(0)
|
||||
for i, c := range sorted {
|
||||
if c.IsChunkManifest {
|
||||
return nil
|
||||
}
|
||||
// An empty or undecodable ETag is not a content fingerprint, so
|
||||
// element-wise (offset, size, ETag) equality cannot prove the bytes
|
||||
// match — decline rather than assert a false content-equality.
|
||||
if len(util.Base64Md5ToBytes(c.ETag)) == 0 {
|
||||
return nil
|
||||
}
|
||||
if i > 0 && c.Offset < prevEnd {
|
||||
return nil
|
||||
}
|
||||
prevEnd = c.Offset + int64(c.Size)
|
||||
}
|
||||
return sorted
|
||||
}
|
||||
|
||||
// mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil.
|
||||
// Returns relation, absolute delta in seconds, and human-readable string.
|
||||
func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) {
|
||||
@@ -557,11 +647,19 @@ func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.En
|
||||
result.sizeMismatch.Add(1)
|
||||
case diffETagMismatch:
|
||||
result.etagMismatch.Add(1)
|
||||
case diffChunkReorder:
|
||||
result.chunkReorder.Add(1)
|
||||
}
|
||||
|
||||
if result.jsonOutput {
|
||||
writeJSONDiff(result, diffType, dir, entryA, entryB)
|
||||
} else {
|
||||
// A chunk reorder is content-equal, not a real diff: keep it out of the
|
||||
// default text report and list it only at -v=1. JSON always emits it so
|
||||
// the per-record stream matches the summary count.
|
||||
if diffType == diffChunkReorder && !glog.V(1) {
|
||||
return
|
||||
}
|
||||
writeTextDiff(result, diffType, dir, entryA, entryB)
|
||||
}
|
||||
}
|
||||
@@ -590,6 +688,9 @@ func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, en
|
||||
ann := annotation(entryA, entryB)
|
||||
fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n",
|
||||
entryPath, filer.ETag(entryA), filer.ETag(entryB), ann)
|
||||
case diffChunkReorder:
|
||||
fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n",
|
||||
entryPath, filer.ETag(entryA), filer.ETag(entryB))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -640,6 +741,14 @@ func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, en
|
||||
rec.MtimeRelation = relation
|
||||
rec.MtimeDelta = delta
|
||||
rec.Hint = hintFor(relation)
|
||||
case diffChunkReorder:
|
||||
rec.Type = "CHUNK_REORDER"
|
||||
rec.A = toEntryRecord(entryA)
|
||||
rec.B = toEntryRecord(entryB)
|
||||
relation, delta, _ := mtimeRelation(entryA, entryB)
|
||||
rec.MtimeRelation = relation
|
||||
rec.MtimeDelta = delta
|
||||
// no hint: content-equal, no action needed
|
||||
}
|
||||
|
||||
writeJSONLine(result, rec)
|
||||
|
||||
@@ -4,11 +4,14 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
@@ -101,6 +104,34 @@ func verifyDirEntry(name string) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{Name: name, IsDirectory: true}
|
||||
}
|
||||
|
||||
// verifyChunk builds a FileChunk whose ETag is the base64 MD5 of seed, so
|
||||
// filer.ETagChunks (which base64-decodes each chunk ETag) can hash it. Distinct
|
||||
// seeds yield distinct per-chunk MD5s.
|
||||
func verifyChunk(offset int64, size uint64, seed string) *filer_pb.FileChunk {
|
||||
return &filer_pb.FileChunk{
|
||||
Offset: offset,
|
||||
Size: size,
|
||||
ETag: util.Base64Encode(util.Md5([]byte(seed))),
|
||||
}
|
||||
}
|
||||
|
||||
// verifyChunkedEntry builds a chunk-backed entry with no stored attr.Md5, so
|
||||
// its file ETag is derived from ETagChunks (the order-sensitive path). FileSize
|
||||
// is the max chunk end, matching filer.FileSize, so size comparison passes.
|
||||
func verifyChunkedEntry(name string, chunks []*filer_pb.FileChunk) *filer_pb.Entry {
|
||||
var total uint64
|
||||
for _, c := range chunks {
|
||||
if end := uint64(c.Offset) + c.Size; end > total {
|
||||
total = end
|
||||
}
|
||||
}
|
||||
return &filer_pb.Entry{
|
||||
Name: name,
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: total}, // Md5 nil → ETagChunks path
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
|
||||
// --- tests ---
|
||||
|
||||
// TestVerifySyncMissingFile confirms that a file present in A but absent in B
|
||||
@@ -280,6 +311,271 @@ func TestVerifySyncETagMismatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncChunkReorder confirms that two entries holding the SAME
|
||||
// chunks (same offsets, same per-chunk MD5s) in a DIFFERENT slice order are
|
||||
// classified as CHUNK_REORDER (content-equal), not ETAG_MISMATCH, and are
|
||||
// therefore not counted as errors. This is the S3-multipart vs filer.backup
|
||||
// reordering false positive.
|
||||
func TestVerifySyncChunkReorder(t *testing.T) {
|
||||
c0 := verifyChunk(0, 100, "chunk-0")
|
||||
c1 := verifyChunk(100, 100, "chunk-1")
|
||||
c2 := verifyChunk(200, 100, "chunk-2")
|
||||
|
||||
// A stores [c0,c1,c2]; B stores a permutation [c2,c0,c1] of the same chunks.
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1, c2})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c2, c0, c1})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 1 {
|
||||
t.Errorf("chunkReorder = %d, want 1", got)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 0 {
|
||||
t.Errorf("etagMismatch = %d, want 0 (reordered chunks are content-equal)", got)
|
||||
}
|
||||
if got := result.sizeMismatch.Load(); got != 0 {
|
||||
t.Errorf("sizeMismatch = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncGenuineChunkDivergence confirms that when one chunk's content
|
||||
// actually differs (same offsets and count, different per-chunk MD5), the file
|
||||
// stays classified as ETAG_MISMATCH and is NOT downgraded to CHUNK_REORDER.
|
||||
func TestVerifySyncGenuineChunkDivergence(t *testing.T) {
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
|
||||
verifyChunk(0, 100, "chunk-0"),
|
||||
verifyChunk(100, 100, "chunk-1"),
|
||||
verifyChunk(200, 100, "chunk-2"),
|
||||
})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
|
||||
verifyChunk(0, 100, "chunk-0"),
|
||||
verifyChunk(100, 100, "chunk-1"),
|
||||
verifyChunk(200, 100, "chunk-2-DIFFERENT"), // real content divergence
|
||||
})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 1 {
|
||||
t.Errorf("etagMismatch = %d, want 1 (genuine chunk content divergence)", got)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 0 {
|
||||
t.Errorf("chunkReorder = %d, want 0 (content actually differs)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncChunkCountDiffStaysEtagMismatch confirms that a differing chunk
|
||||
// count with an equal file size (e.g. differently split content) is NOT treated
|
||||
// as a reordering — it stays ETAG_MISMATCH.
|
||||
func TestVerifySyncChunkCountDiffStaysEtagMismatch(t *testing.T) {
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
|
||||
verifyChunk(0, 100, "a"),
|
||||
verifyChunk(100, 100, "b"),
|
||||
verifyChunk(200, 100, "c"),
|
||||
})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
|
||||
verifyChunk(0, 150, "x"),
|
||||
verifyChunk(150, 150, "y"),
|
||||
})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.sizeMismatch.Load(); got != 0 {
|
||||
t.Errorf("sizeMismatch = %d, want 0 (both total 300 bytes)", got)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 1 {
|
||||
t.Errorf("etagMismatch = %d, want 1", got)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 0 {
|
||||
t.Errorf("chunkReorder = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncDuplicateOffsetStaysEtagMismatch confirms that overlapping
|
||||
// chunks at the same offset — where the visible bytes are resolved by chunk
|
||||
// timestamp, not by the raw chunk list — are NOT fast-pathed to CHUNK_REORDER.
|
||||
// Both sides hold the same two ETags at offset 0 in a different order, so the
|
||||
// file ETags differ; the visible content is ambiguous from the list alone, so
|
||||
// this must stay ETAG_MISMATCH.
|
||||
func TestVerifySyncDuplicateOffsetStaysEtagMismatch(t *testing.T) {
|
||||
c1 := verifyChunk(0, 100, "v1")
|
||||
c2 := verifyChunk(0, 100, "v2") // same offset → overlap
|
||||
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c1, c2})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c1})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 1 {
|
||||
t.Errorf("etagMismatch = %d, want 1 (overlapping offsets are not a safe reorder)", got)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 0 {
|
||||
t.Errorf("chunkReorder = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncManifestChunkStaysEtagMismatch confirms that entries containing
|
||||
// a manifest chunk are not fast-pathed to CHUNK_REORDER: manifest chunks
|
||||
// represent compacted, possibly overlapping history that this check does not
|
||||
// resolve.
|
||||
func TestVerifySyncManifestChunkStaysEtagMismatch(t *testing.T) {
|
||||
manifest := func(offset int64, size uint64, seed string) *filer_pb.FileChunk {
|
||||
c := verifyChunk(offset, size, seed)
|
||||
c.IsChunkManifest = true
|
||||
return c
|
||||
}
|
||||
a0, a1 := manifest(0, 100, "m0"), verifyChunk(100, 100, "m1")
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a0, a1})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a1, a0})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 1 {
|
||||
t.Errorf("etagMismatch = %d, want 1 (manifest chunks are not fast-pathed)", got)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 0 {
|
||||
t.Errorf("chunkReorder = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncEmptyChunkETagStaysEtagMismatch confirms that a chunk with an
|
||||
// empty per-chunk ETag is never fast-pathed to CHUNK_REORDER: an empty ETag is
|
||||
// not a content fingerprint, so (offset, size, ETag) equality cannot prove the
|
||||
// bytes match. The two sides reorder their non-empty chunks so the file ETags
|
||||
// differ and the reorder check is reached, but the empty-ETag chunk in the
|
||||
// middle must keep it ETAG_MISMATCH rather than assert a false content-equality.
|
||||
func TestVerifySyncEmptyChunkETagStaysEtagMismatch(t *testing.T) {
|
||||
c0 := verifyChunk(0, 100, "e0")
|
||||
empty := &filer_pb.FileChunk{Offset: 100, Size: 100} // no ETag → not a fingerprint
|
||||
c2 := verifyChunk(200, 100, "e2")
|
||||
|
||||
clientA := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c0, empty, c2})},
|
||||
},
|
||||
}
|
||||
clientB := &verifyTestFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c0, empty})},
|
||||
},
|
||||
}
|
||||
|
||||
result := &VerifyResult{}
|
||||
sem := make(chan struct{}, verifySyncConcurrency)
|
||||
if err := compareDirectory(context.Background(), clientA, clientB,
|
||||
"/root", "/root", false, time.Time{}, sem, result); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := result.etagMismatch.Load(); got != 1 {
|
||||
t.Errorf("etagMismatch = %d, want 1 (empty per-chunk ETag is not a content fingerprint)", got)
|
||||
}
|
||||
if got := result.chunkReorder.Load(); got != 0 {
|
||||
t.Errorf("chunkReorder = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// captureStdout runs fn with os.Stdout redirected to a pipe and returns what it
|
||||
// wrote. Tests that assert on emitted diff records use it.
|
||||
func captureStdout(t *testing.T, fn func()) string {
|
||||
t.Helper()
|
||||
orig := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Pipe: %v", err)
|
||||
}
|
||||
os.Stdout = w
|
||||
fn()
|
||||
w.Close()
|
||||
os.Stdout = orig
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("read captured stdout: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// TestVerifySyncChunkReorderJSONEmittedByDefault confirms that in JSON output
|
||||
// mode a CHUNK_REORDER record is emitted at default verbosity: the -v=1 gate
|
||||
// suppresses only the human text report, so the NDJSON per-record stream stays
|
||||
// consistent with the summary count a machine consumer reads.
|
||||
func TestVerifySyncChunkReorderJSONEmittedByDefault(t *testing.T) {
|
||||
c0 := verifyChunk(0, 100, "chunk-0")
|
||||
c1 := verifyChunk(100, 100, "chunk-1")
|
||||
entryA := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1})
|
||||
entryB := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c1, c0})
|
||||
|
||||
result := &VerifyResult{jsonOutput: true}
|
||||
out := captureStdout(t, func() {
|
||||
reportDiff(diffChunkReorder, "/root", entryA, entryB, result)
|
||||
})
|
||||
|
||||
if got := result.chunkReorder.Load(); got != 1 {
|
||||
t.Fatalf("chunkReorder = %d, want 1", got)
|
||||
}
|
||||
if !strings.Contains(out, `"type":"CHUNK_REORDER"`) {
|
||||
t.Errorf("JSON output missing CHUNK_REORDER record at default verbosity; got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifySyncCutoffTime verifies that entries newer than cutoffTime are
|
||||
// skipped in both the A-only (MISSING) and B-only (ONLY_IN_B) branches.
|
||||
func TestVerifySyncCutoffTime(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user