From ebeab4b6ecf6c7161d4f4934fe01f951bbcc148d Mon Sep 17 00:00:00 2001 From: Jaehoon Kim Date: Thu, 2 Jul 2026 02:38:21 +0900 Subject: [PATCH] feat(filer.sync.verify): reclassify chunk-slice-order ETag diffs as CHUNK_REORDER (#10177) * feat(filer.sync.verify): reclassify chunk-slice-order ETag diffs as CHUNK_REORDER filer.ETagChunks concatenates per-chunk MD5s in stored slice order without normalising by offset, so byte-identical content written by two different paths (S3 multipart part-completion order on the source vs filer.backup replication arrival order on the destination) yields different file ETags. filer.sync.verify reported these as ETAG_MISMATCH even though the files are equal. Add a second-pass check on every ETAG_MISMATCH: when both sides derive their ETag from chunks (no attr.Md5) and hold a manifest-free, non-overlapping chunk set that, once sorted by offset, matches element-wise on (offset, size, ETag), classify the file as CHUNK_REORDER. Such files are content-equal, so they are not counted as errors and do not affect the exit code; they are listed only at higher verbosity (weed -v=1), while the summary always shows their count. The check stays conservative: a stored attr.Md5 (order-independent content hash), a differing chunk count, an overlapping/duplicate offset (whose visible bytes are resolved by timestamp), or a manifest chunk all remain ETAG_MISMATCH. * filer.sync.verify: decline chunk-reorder fast path on empty per-chunk ETag An empty or undecodable per-chunk ETag is not a content fingerprint, so element-wise (offset, size, ETag) equality can't prove the bytes match. Treating "" == "" as content-equal could reclassify a genuine divergence as CHUNK_REORDER and drop it from the error count. Decline such chunk sets so they stay ETAG_MISMATCH. * filer.sync.verify: emit CHUNK_REORDER in JSON output regardless of -v The -v=1 gate belongs to the human text report only. Applying it before the jsonOutput branch dropped the per-file CHUNK_REORDER records from NDJSON while the summary still counted them, so a machine consumer saw a non-zero count with no records to reconcile it. Gate the text path only; JSON always emits. --------- Co-authored-by: Chris Lu --- weed/command/filer_sync_verify.go | 111 +++++++++- weed/command/filer_sync_verify_test.go | 296 +++++++++++++++++++++++++ 2 files changed, 406 insertions(+), 1 deletion(-) diff --git a/weed/command/filer_sync_verify.go b/weed/command/filer_sync_verify.go index 0031eb0e3..378103f8e 100644 --- a/weed/command/filer_sync_verify.go +++ b/weed/command/filer_sync_verify.go @@ -58,6 +58,12 @@ var cmdFilerSyncVerify = &Command{ -modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and -isActivePassive for unidirectional comparison. + A chunk-derived ETag (no stored attr.Md5) can differ between clusters only + because the chunk slice was assembled in a different order, since ETagChunks + does not sort by offset. Such files are byte-identical and reported as + CHUNK_REORDER: not counted as errors, always counted in the summary, listed + only at -v=1. + Exits with code 0 on agreement, 2 on differences or operational errors. `, @@ -123,6 +129,7 @@ type VerifyResult struct { missingCount atomic.Int64 sizeMismatch atomic.Int64 etagMismatch atomic.Int64 + chunkReorder atomic.Int64 // content-equal; chunks differ only in slice order onlyInB atomic.Int64 skippedRecent atomic.Int64 @@ -139,6 +146,7 @@ const ( diffOnlyInB // in B but not in A diffSizeMismatch // size differs diffETagMismatch // etag differs + diffChunkReorder // etag differs but content is equal (chunk slice order only) ) // diffRecord is the JSON Lines schema for a single diff entry. @@ -167,6 +175,7 @@ type summaryRecord struct { Missing int64 `json:"missing"` SizeMismatch int64 `json:"sizeMismatch"` ETagMismatch int64 `json:"etagMismatch"` + ChunkReorder int64 `json:"chunkReorder"` OnlyInB int64 `json:"onlyInB"` TotalErrors int64 `json:"totalErrors"` } @@ -236,6 +245,7 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string, Missing: result.missingCount.Load(), SizeMismatch: result.sizeMismatch.Load(), ETagMismatch: result.etagMismatch.Load(), + ChunkReorder: result.chunkReorder.Load(), OnlyInB: result.onlyInB.Load(), TotalErrors: totalErrors, } @@ -250,6 +260,9 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string, fmt.Fprintf(os.Stdout, " Missing in B: %d\n", result.missingCount.Load()) fmt.Fprintf(os.Stdout, " Size mismatch: %d\n", result.sizeMismatch.Load()) fmt.Fprintf(os.Stdout, " ETag mismatch: %d\n", result.etagMismatch.Load()) + if n := result.chunkReorder.Load(); n > 0 { + fmt.Fprintf(os.Stdout, " Chunk reorder (equal): %d (content-equal, not an error; use -v=1 to list)\n", n) + } if !isActivePassive { fmt.Fprintf(os.Stdout, " Only in B: %d\n", result.onlyInB.Load()) } @@ -496,11 +509,88 @@ func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyRe etagA := filer.ETag(entryA) etagB := filer.ETag(entryB) if etagA != etagB { - reportDiff(diffETagMismatch, dir, entryA, entryB, result) + if isChunkReorder(entryA, entryB) { + // Same bytes, just a different chunk slice order — not a real diff. + reportDiff(diffChunkReorder, dir, entryA, entryB, result) + } else { + reportDiff(diffETagMismatch, dir, entryA, entryB, result) + } return } } +// isChunkReorder reports whether two entries with differing file ETags are +// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes +// per-chunk MD5s in stored order without sorting by offset, so the same bytes +// written by different paths (S3 multipart vs filer.backup) can hash +// differently. If both chunk lists sorted by offset match element-wise on +// (offset, size, ETag), the files cover the same bytes with the same content +// and only the slice order differed. +// +// Applies only when both ETags come from chunks (a stored attr.Md5 is +// order-independent, so a mismatch there is a real content difference) and the +// chunks form a manifest-free, non-overlapping set whose bytes are fully +// determined by the list — overlaps would need timestamp-based visible-interval +// resolution, so those stay ETAG_MISMATCH. +func isChunkReorder(entryA, entryB *filer_pb.Entry) bool { + if !usesChunkETag(entryA) || !usesChunkETag(entryB) { + return false + } + a := sortedSimpleChunks(entryA.GetChunks()) + b := sortedSimpleChunks(entryB.GetChunks()) + // nil means overlapping or manifest chunks we do not fast-path; a single + // chunk hashes directly and a differing count is a real divergence. + if a == nil || b == nil || len(a) != len(b) || len(a) < 2 { + return false + } + for i := range a { + if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag { + return false + } + } + return true +} + +// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry, +// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's +// own condition so it agrees with how the compared ETag was computed. +func usesChunkETag(entry *filer_pb.Entry) bool { + return entry.Attributes == nil || entry.Attributes.Md5 == nil +} + +// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a +// manifest-free, non-overlapping set whose visible bytes are fully determined +// by the list and every chunk carries a usable per-chunk ETag; otherwise nil. +// Overlapping ranges would require resolving the visible interval by chunk +// timestamp, which this fast path does not attempt. +func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk { + sorted := make([]*filer_pb.FileChunk, len(chunks)) + copy(sorted, chunks) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Offset != sorted[j].Offset { + return sorted[i].Offset < sorted[j].Offset + } + return sorted[i].ETag < sorted[j].ETag + }) + prevEnd := int64(0) + for i, c := range sorted { + if c.IsChunkManifest { + return nil + } + // An empty or undecodable ETag is not a content fingerprint, so + // element-wise (offset, size, ETag) equality cannot prove the bytes + // match — decline rather than assert a false content-equality. + if len(util.Base64Md5ToBytes(c.ETag)) == 0 { + return nil + } + if i > 0 && c.Offset < prevEnd { + return nil + } + prevEnd = c.Offset + int64(c.Size) + } + return sorted +} + // mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil. // Returns relation, absolute delta in seconds, and human-readable string. func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) { @@ -557,11 +647,19 @@ func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.En result.sizeMismatch.Add(1) case diffETagMismatch: result.etagMismatch.Add(1) + case diffChunkReorder: + result.chunkReorder.Add(1) } if result.jsonOutput { writeJSONDiff(result, diffType, dir, entryA, entryB) } else { + // A chunk reorder is content-equal, not a real diff: keep it out of the + // default text report and list it only at -v=1. JSON always emits it so + // the per-record stream matches the summary count. + if diffType == diffChunkReorder && !glog.V(1) { + return + } writeTextDiff(result, diffType, dir, entryA, entryB) } } @@ -590,6 +688,9 @@ func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, en ann := annotation(entryA, entryB) fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n", entryPath, filer.ETag(entryA), filer.ETag(entryB), ann) + case diffChunkReorder: + fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n", + entryPath, filer.ETag(entryA), filer.ETag(entryB)) } } @@ -640,6 +741,14 @@ func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, en rec.MtimeRelation = relation rec.MtimeDelta = delta rec.Hint = hintFor(relation) + case diffChunkReorder: + rec.Type = "CHUNK_REORDER" + rec.A = toEntryRecord(entryA) + rec.B = toEntryRecord(entryB) + relation, delta, _ := mtimeRelation(entryA, entryB) + rec.MtimeRelation = relation + rec.MtimeDelta = delta + // no hint: content-equal, no action needed } writeJSONLine(result, rec) diff --git a/weed/command/filer_sync_verify_test.go b/weed/command/filer_sync_verify_test.go index 1fc6a6b24..c26014bcc 100644 --- a/weed/command/filer_sync_verify_test.go +++ b/weed/command/filer_sync_verify_test.go @@ -4,11 +4,14 @@ import ( "context" "fmt" "io" + "os" + "strings" "sync/atomic" "testing" "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -101,6 +104,34 @@ func verifyDirEntry(name string) *filer_pb.Entry { return &filer_pb.Entry{Name: name, IsDirectory: true} } +// verifyChunk builds a FileChunk whose ETag is the base64 MD5 of seed, so +// filer.ETagChunks (which base64-decodes each chunk ETag) can hash it. Distinct +// seeds yield distinct per-chunk MD5s. +func verifyChunk(offset int64, size uint64, seed string) *filer_pb.FileChunk { + return &filer_pb.FileChunk{ + Offset: offset, + Size: size, + ETag: util.Base64Encode(util.Md5([]byte(seed))), + } +} + +// verifyChunkedEntry builds a chunk-backed entry with no stored attr.Md5, so +// its file ETag is derived from ETagChunks (the order-sensitive path). FileSize +// is the max chunk end, matching filer.FileSize, so size comparison passes. +func verifyChunkedEntry(name string, chunks []*filer_pb.FileChunk) *filer_pb.Entry { + var total uint64 + for _, c := range chunks { + if end := uint64(c.Offset) + c.Size; end > total { + total = end + } + } + return &filer_pb.Entry{ + Name: name, + Attributes: &filer_pb.FuseAttributes{FileSize: total}, // Md5 nil → ETagChunks path + Chunks: chunks, + } +} + // --- tests --- // TestVerifySyncMissingFile confirms that a file present in A but absent in B @@ -280,6 +311,271 @@ func TestVerifySyncETagMismatch(t *testing.T) { } } +// TestVerifySyncChunkReorder confirms that two entries holding the SAME +// chunks (same offsets, same per-chunk MD5s) in a DIFFERENT slice order are +// classified as CHUNK_REORDER (content-equal), not ETAG_MISMATCH, and are +// therefore not counted as errors. This is the S3-multipart vs filer.backup +// reordering false positive. +func TestVerifySyncChunkReorder(t *testing.T) { + c0 := verifyChunk(0, 100, "chunk-0") + c1 := verifyChunk(100, 100, "chunk-1") + c2 := verifyChunk(200, 100, "chunk-2") + + // A stores [c0,c1,c2]; B stores a permutation [c2,c0,c1] of the same chunks. + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1, c2})}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c2, c0, c1})}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.chunkReorder.Load(); got != 1 { + t.Errorf("chunkReorder = %d, want 1", got) + } + if got := result.etagMismatch.Load(); got != 0 { + t.Errorf("etagMismatch = %d, want 0 (reordered chunks are content-equal)", got) + } + if got := result.sizeMismatch.Load(); got != 0 { + t.Errorf("sizeMismatch = %d, want 0", got) + } +} + +// TestVerifySyncGenuineChunkDivergence confirms that when one chunk's content +// actually differs (same offsets and count, different per-chunk MD5), the file +// stays classified as ETAG_MISMATCH and is NOT downgraded to CHUNK_REORDER. +func TestVerifySyncGenuineChunkDivergence(t *testing.T) { + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{ + verifyChunk(0, 100, "chunk-0"), + verifyChunk(100, 100, "chunk-1"), + verifyChunk(200, 100, "chunk-2"), + })}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{ + verifyChunk(0, 100, "chunk-0"), + verifyChunk(100, 100, "chunk-1"), + verifyChunk(200, 100, "chunk-2-DIFFERENT"), // real content divergence + })}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.etagMismatch.Load(); got != 1 { + t.Errorf("etagMismatch = %d, want 1 (genuine chunk content divergence)", got) + } + if got := result.chunkReorder.Load(); got != 0 { + t.Errorf("chunkReorder = %d, want 0 (content actually differs)", got) + } +} + +// TestVerifySyncChunkCountDiffStaysEtagMismatch confirms that a differing chunk +// count with an equal file size (e.g. differently split content) is NOT treated +// as a reordering — it stays ETAG_MISMATCH. +func TestVerifySyncChunkCountDiffStaysEtagMismatch(t *testing.T) { + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{ + verifyChunk(0, 100, "a"), + verifyChunk(100, 100, "b"), + verifyChunk(200, 100, "c"), + })}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{ + verifyChunk(0, 150, "x"), + verifyChunk(150, 150, "y"), + })}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.sizeMismatch.Load(); got != 0 { + t.Errorf("sizeMismatch = %d, want 0 (both total 300 bytes)", got) + } + if got := result.etagMismatch.Load(); got != 1 { + t.Errorf("etagMismatch = %d, want 1", got) + } + if got := result.chunkReorder.Load(); got != 0 { + t.Errorf("chunkReorder = %d, want 0", got) + } +} + +// TestVerifySyncDuplicateOffsetStaysEtagMismatch confirms that overlapping +// chunks at the same offset — where the visible bytes are resolved by chunk +// timestamp, not by the raw chunk list — are NOT fast-pathed to CHUNK_REORDER. +// Both sides hold the same two ETags at offset 0 in a different order, so the +// file ETags differ; the visible content is ambiguous from the list alone, so +// this must stay ETAG_MISMATCH. +func TestVerifySyncDuplicateOffsetStaysEtagMismatch(t *testing.T) { + c1 := verifyChunk(0, 100, "v1") + c2 := verifyChunk(0, 100, "v2") // same offset → overlap + + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c1, c2})}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c1})}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.etagMismatch.Load(); got != 1 { + t.Errorf("etagMismatch = %d, want 1 (overlapping offsets are not a safe reorder)", got) + } + if got := result.chunkReorder.Load(); got != 0 { + t.Errorf("chunkReorder = %d, want 0", got) + } +} + +// TestVerifySyncManifestChunkStaysEtagMismatch confirms that entries containing +// a manifest chunk are not fast-pathed to CHUNK_REORDER: manifest chunks +// represent compacted, possibly overlapping history that this check does not +// resolve. +func TestVerifySyncManifestChunkStaysEtagMismatch(t *testing.T) { + manifest := func(offset int64, size uint64, seed string) *filer_pb.FileChunk { + c := verifyChunk(offset, size, seed) + c.IsChunkManifest = true + return c + } + a0, a1 := manifest(0, 100, "m0"), verifyChunk(100, 100, "m1") + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a0, a1})}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a1, a0})}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.etagMismatch.Load(); got != 1 { + t.Errorf("etagMismatch = %d, want 1 (manifest chunks are not fast-pathed)", got) + } + if got := result.chunkReorder.Load(); got != 0 { + t.Errorf("chunkReorder = %d, want 0", got) + } +} + +// TestVerifySyncEmptyChunkETagStaysEtagMismatch confirms that a chunk with an +// empty per-chunk ETag is never fast-pathed to CHUNK_REORDER: an empty ETag is +// not a content fingerprint, so (offset, size, ETag) equality cannot prove the +// bytes match. The two sides reorder their non-empty chunks so the file ETags +// differ and the reorder check is reached, but the empty-ETag chunk in the +// middle must keep it ETAG_MISMATCH rather than assert a false content-equality. +func TestVerifySyncEmptyChunkETagStaysEtagMismatch(t *testing.T) { + c0 := verifyChunk(0, 100, "e0") + empty := &filer_pb.FileChunk{Offset: 100, Size: 100} // no ETag → not a fingerprint + c2 := verifyChunk(200, 100, "e2") + + clientA := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c0, empty, c2})}, + }, + } + clientB := &verifyTestFilerClient{ + entriesByDir: map[string][]*filer_pb.Entry{ + "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c0, empty})}, + }, + } + + result := &VerifyResult{} + sem := make(chan struct{}, verifySyncConcurrency) + if err := compareDirectory(context.Background(), clientA, clientB, + "/root", "/root", false, time.Time{}, sem, result); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := result.etagMismatch.Load(); got != 1 { + t.Errorf("etagMismatch = %d, want 1 (empty per-chunk ETag is not a content fingerprint)", got) + } + if got := result.chunkReorder.Load(); got != 0 { + t.Errorf("chunkReorder = %d, want 0", got) + } +} + +// captureStdout runs fn with os.Stdout redirected to a pipe and returns what it +// wrote. Tests that assert on emitted diff records use it. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + orig := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stdout = w + fn() + w.Close() + os.Stdout = orig + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read captured stdout: %v", err) + } + return string(data) +} + +// TestVerifySyncChunkReorderJSONEmittedByDefault confirms that in JSON output +// mode a CHUNK_REORDER record is emitted at default verbosity: the -v=1 gate +// suppresses only the human text report, so the NDJSON per-record stream stays +// consistent with the summary count a machine consumer reads. +func TestVerifySyncChunkReorderJSONEmittedByDefault(t *testing.T) { + c0 := verifyChunk(0, 100, "chunk-0") + c1 := verifyChunk(100, 100, "chunk-1") + entryA := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1}) + entryB := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c1, c0}) + + result := &VerifyResult{jsonOutput: true} + out := captureStdout(t, func() { + reportDiff(diffChunkReorder, "/root", entryA, entryB, result) + }) + + if got := result.chunkReorder.Load(); got != 1 { + t.Fatalf("chunkReorder = %d, want 1", got) + } + if !strings.Contains(out, `"type":"CHUNK_REORDER"`) { + t.Errorf("JSON output missing CHUNK_REORDER record at default verbosity; got %q", out) + } +} + // TestVerifySyncCutoffTime verifies that entries newer than cutoffTime are // skipped in both the A-only (MISSING) and B-only (ONLY_IN_B) branches. func TestVerifySyncCutoffTime(t *testing.T) {