seaweedfs/weed/command/filer_sync_verify.go

package command

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path"
	"sort"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/security"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

type SyncVerifyOptions struct {
	filerA          *string
	filerB          *string
	aPath           *string
	bPath           *string
	aSecurity       *string
	bSecurity       *string
	isActivePassive *bool
	modifiedTimeAgo *time.Duration
	jsonOutput      *bool
}

var syncVerifyOptions SyncVerifyOptions

func init() {
	cmdFilerSyncVerify.Run = runFilerSyncVerify // break init cycle
	syncVerifyOptions.filerA = cmdFilerSyncVerify.Flag.String("a", "", "filer A in one SeaweedFS cluster")
	syncVerifyOptions.filerB = cmdFilerSyncVerify.Flag.String("b", "", "filer B in the other SeaweedFS cluster")
	syncVerifyOptions.aPath = cmdFilerSyncVerify.Flag.String("a.path", "/", "directory to verify on filer A")
	syncVerifyOptions.bPath = cmdFilerSyncVerify.Flag.String("b.path", "/", "directory to verify on filer B")
	syncVerifyOptions.aSecurity = cmdFilerSyncVerify.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
	syncVerifyOptions.bSecurity = cmdFilerSyncVerify.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
	syncVerifyOptions.isActivePassive = cmdFilerSyncVerify.Flag.Bool("isActivePassive", false, "one directional comparison from A to B; entries only in B are not reported")
	syncVerifyOptions.modifiedTimeAgo = cmdFilerSyncVerify.Flag.Duration("modifiedTimeAgo", 0, "only verify files modified before this duration ago (e.g. 1h) for sync-lag tolerance")
	syncVerifyOptions.jsonOutput = cmdFilerSyncVerify.Flag.Bool("jsonOutput", false, "emit NDJSON output (one JSON object per line) for external tooling")
}

var cmdFilerSyncVerify = &Command{
	UsageLine: "filer.sync.verify -a=<oneFilerHost>:<oneFilerPort> -b=<otherFilerHost>:<otherFilerPort>",
	Short:     "compare entries between two filers and report differences",
	Long: `compare entries between two filers and report differences, then exit.

	Useful for validating active/passive sync targets agree with the source.
	Reports MISSING (in A but not in B), ONLY_IN_B (in B but not in A; suppressed
	in active-passive mode), SIZE_MISMATCH, and ETAG_MISMATCH. Honors
	-modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and
	-isActivePassive for unidirectional comparison.

	A chunk-derived ETag (no stored attr.Md5) can differ between clusters only
	because the chunk slice was assembled in a different order, since ETagChunks
	does not sort by offset. Such files are byte-identical and reported as
	CHUNK_REORDER: not counted as errors, always counted in the summary, listed
	only at -v=1.

	Exits with code 0 on agreement, 2 on differences or operational errors.

`,
}

func runFilerSyncVerify(cmd *Command, args []string) bool {
	*syncVerifyOptions.aSecurity = util.ResolvePath(*syncVerifyOptions.aSecurity)
	*syncVerifyOptions.bSecurity = util.ResolvePath(*syncVerifyOptions.bSecurity)
	util.LoadSecurityConfiguration()
	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

	grpcDialOptionA := grpcDialOption
	grpcDialOptionB := grpcDialOption
	if *syncVerifyOptions.aSecurity != "" {
		var err error
		if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncVerifyOptions.aSecurity, "grpc.client"); err != nil {
			glog.Fatalf("load security config for filer A: %v", err)
		}
	}
	if *syncVerifyOptions.bSecurity != "" {
		var err error
		if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncVerifyOptions.bSecurity, "grpc.client"); err != nil {
			glog.Fatalf("load security config for filer B: %v", err)
		}
	}

	filerA := pb.ServerAddress(*syncVerifyOptions.filerA)
	filerB := pb.ServerAddress(*syncVerifyOptions.filerB)

	if err := runVerifySync(filerA, filerB, *syncVerifyOptions.aPath, *syncVerifyOptions.bPath,
		*syncVerifyOptions.isActivePassive, *syncVerifyOptions.modifiedTimeAgo,
		*syncVerifyOptions.jsonOutput,
		grpcDialOptionA, grpcDialOptionB); err != nil {
		glog.Errorf("verify sync: %v", err)
		os.Exit(2)
	}
	return true
}

// verifySyncConcurrency caps concurrent directory I/O across the entire
// recursive walk. A single shared semaphore is created in runVerifySync and
// passed down so the limit applies globally — a per-call semaphore would only
// cap concurrency per directory level, allowing fanout to grow as
// verifySyncConcurrency^depth on deep trees.
//
// Trade-off: higher values reduce wall time on wide trees by parallelizing
// listEntries RPCs, at the cost of more concurrent load on both filers and
// more memory from queued goroutines (each waiting goroutine ~2KB stack).
// Each compareDirectory holds a slot only for its own listing+compare phase
// and releases before recursing, so a parent never blocks waiting for
// children to acquire slots.
const verifySyncConcurrency = 5

// isTooRecent reports whether entry's mtime is past cutoff (sync-lag tolerance).
// Returns false when cutoff is zero or attributes are missing.
func isTooRecent(entry *filer_pb.Entry, cutoff time.Time) bool {
	return !cutoff.IsZero() && entry != nil && entry.Attributes != nil && entry.Attributes.Mtime > cutoff.Unix()
}

type VerifyResult struct {
	dirCount      atomic.Int64
	fileCount     atomic.Int64
	missingCount  atomic.Int64
	sizeMismatch  atomic.Int64
	etagMismatch  atomic.Int64
	chunkReorder  atomic.Int64 // content-equal; chunks differ only in slice order
	onlyInB       atomic.Int64
	skippedRecent atomic.Int64

	// outputMu serializes writes to stdout. Multiple goroutines call
	// reportDiff concurrently from compareDirectory worker pool.
	outputMu   sync.Mutex
	jsonOutput bool
}

type verifyDiffType int

const (
	diffMissing      verifyDiffType = iota // in A but not in B
	diffOnlyInB                            // in B but not in A
	diffSizeMismatch                       // size differs
	diffETagMismatch                       // etag differs
	diffChunkReorder                       // etag differs but content is equal (chunk slice order only)
)

// diffRecord is the JSON Lines schema for a single diff entry.
type diffRecord struct {
	Type          string       `json:"type"`
	Path          string       `json:"path"`
	IsDirectory   bool         `json:"isDirectory,omitempty"`
	A             *entryRecord `json:"a,omitempty"`
	B             *entryRecord `json:"b,omitempty"`
	MtimeRelation string       `json:"mtimeRelation,omitempty"` // EQUAL | A_NEWER | B_NEWER
	MtimeDelta    string       `json:"mtimeDelta,omitempty"`    // human-readable, e.g. "5d", "12h"
	Hint          string       `json:"hint,omitempty"`          // late_updates_skip_likely | sync_lag_or_event_miss
}

type entryRecord struct {
	Size  uint64 `json:"size"`
	Mtime int64  `json:"mtime"`
	ETag  string `json:"etag,omitempty"`
}

type summaryRecord struct {
	Type          string `json:"type"`
	Directories   int64  `json:"directories"`
	Files         int64  `json:"files"`
	SkippedRecent int64  `json:"skippedRecent"`
	Missing       int64  `json:"missing"`
	SizeMismatch  int64  `json:"sizeMismatch"`
	ETagMismatch  int64  `json:"etagMismatch"`
	ChunkReorder  int64  `json:"chunkReorder"`
	OnlyInB       int64  `json:"onlyInB"`
	TotalErrors   int64  `json:"totalErrors"`
}

// simpleFilerClient implements filer_pb.FilerClient for gRPC connections
type simpleFilerClient struct {
	grpcAddress    pb.ServerAddress
	grpcDialOption grpc.DialOption
}

func (c *simpleFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
	return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, c.grpcAddress.ToGrpcAddress(), false, c.grpcDialOption)
}

func (c *simpleFilerClient) AdjustedUrl(location *filer_pb.Location) string {
	return location.Url
}

func (c *simpleFilerClient) GetDataCenter() string {
	return ""
}

func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
	isActivePassive bool, modifiedTimeAgo time.Duration,
	jsonOutput bool,
	grpcDialOptionA, grpcDialOptionB grpc.DialOption) error {

	clientA := &simpleFilerClient{grpcAddress: filerA, grpcDialOption: grpcDialOptionA}
	clientB := &simpleFilerClient{grpcAddress: filerB, grpcDialOption: grpcDialOptionB}

	var cutoffTime time.Time
	if modifiedTimeAgo > 0 {
		cutoffTime = time.Now().Add(-modifiedTimeAgo)
	}

	if !jsonOutput {
		if !cutoffTime.IsZero() {
			fmt.Fprintf(os.Stdout, "Verifying files modified before %v (modifiedTimeAgo=%v)\n",
				cutoffTime.Format(time.RFC3339), modifiedTimeAgo)
		}
		fmt.Fprintf(os.Stdout, "Comparing %s%s => %s%s (isActivePassive=%v)\n\n",
			filerA, aPath, filerB, bPath, isActivePassive)
	}

	result := &VerifyResult{jsonOutput: jsonOutput}
	ctx := context.Background()
	sem := make(chan struct{}, verifySyncConcurrency)

	if err := compareDirectory(ctx, clientA, clientB, aPath, bPath, isActivePassive, cutoffTime, sem, result); err != nil {
		return err
	}

	totalErrors := result.missingCount.Load() + result.sizeMismatch.Load() + result.etagMismatch.Load()
	if !isActivePassive {
		totalErrors += result.onlyInB.Load()
	}

	if jsonOutput {
		summary := summaryRecord{
			Type:          "SUMMARY",
			Directories:   result.dirCount.Load(),
			Files:         result.fileCount.Load(),
			SkippedRecent: result.skippedRecent.Load(),
			Missing:       result.missingCount.Load(),
			SizeMismatch:  result.sizeMismatch.Load(),
			ETagMismatch:  result.etagMismatch.Load(),
			ChunkReorder:  result.chunkReorder.Load(),
			OnlyInB:       result.onlyInB.Load(),
			TotalErrors:   totalErrors,
		}
		writeJSONLine(result, summary)
	} else {
		fmt.Fprintf(os.Stdout, "\nSummary:\n")
		fmt.Fprintf(os.Stdout, "  Directories compared: %d\n", result.dirCount.Load())
		fmt.Fprintf(os.Stdout, "  Files verified:       %d\n", result.fileCount.Load())
		if result.skippedRecent.Load() > 0 {
			fmt.Fprintf(os.Stdout, "  Skipped (too recent): %d\n", result.skippedRecent.Load())
		}
		fmt.Fprintf(os.Stdout, "  Missing in B:         %d\n", result.missingCount.Load())
		fmt.Fprintf(os.Stdout, "  Size mismatch:        %d\n", result.sizeMismatch.Load())
		fmt.Fprintf(os.Stdout, "  ETag mismatch:        %d\n", result.etagMismatch.Load())
		if n := result.chunkReorder.Load(); n > 0 {
			fmt.Fprintf(os.Stdout, "  Chunk reorder (equal): %d  (content-equal, not an error; use -v=1 to list)\n", n)
		}
		if !isActivePassive {
			fmt.Fprintf(os.Stdout, "  Only in B:            %d\n", result.onlyInB.Load())
		}
		fmt.Fprintf(os.Stdout, "  Total errors:         %d\n", totalErrors)
	}

	if totalErrors > 0 {
		return fmt.Errorf("found %d differences", totalErrors)
	}
	return nil
}

// dirEntries holds one directory's entries buffered and sorted in Go byte
// order, which the merge in compareDirectory requires (it pairs equal names and
// uses eA.Name < eB.Name as the tie-break). The filer's own list order is not
// trusted: since PR #9824 (4.32) a filer forces byte-lexicographic listing
// regardless of SQL collation, while an older filer whose name column uses a
// locale collation lists case-insensitively. Comparing two such clusters yields
// the same name set in a different order, desyncing the merge into spurious
// MISSING / ONLY_IN_B. Sorting both sides client-side makes the order identical
// regardless of filer version or store backend.
//
// The load runs in a background goroutine so both sides of a compareDirectory
// list concurrently; peek/advance block on `ready` until it completes. Memory
// is O(directory size) per side — acceptable for a one-shot verify CLI.
type dirEntries struct {
	ready   chan struct{} // closed once entries and err are set
	entries []*filer_pb.Entry
	idx     int
	err     error
}

// newDirEntries starts loading the whole directory and sorting it by Name, then
// returns immediately. The goroutine exits when listing completes, errors, or
// ctx is cancelled; it always closes `ready` so consumers never block forever.
func newDirEntries(ctx context.Context, client filer_pb.FilerClient, dir string) *dirEntries {
	s := &dirEntries{ready: make(chan struct{})}
	go func() {
		defer close(s.ready)
		s.err = filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
			func(entry *filer_pb.Entry, isLast bool) error {
				select {
				case <-ctx.Done():
					return ctx.Err()
				default:
				}
				s.entries = append(s.entries, entry)
				return nil
			})
		if s.err == nil {
			sort.Slice(s.entries, func(i, j int) bool {
				return s.entries[i].Name < s.entries[j].Name
			})
		}
	}()
	return s
}

// wait blocks until the directory has finished loading and sorting. Its close
// of `ready` happens-before the return, so entries and err are then safe to read
// without additional synchronisation.
func (s *dirEntries) wait() { <-s.ready }

// peek returns the next entry without consuming it, or nil at end-of-stream.
// It blocks until the directory has finished loading and sorting.
func (s *dirEntries) peek() *filer_pb.Entry {
	s.wait()
	if s.idx >= len(s.entries) {
		return nil
	}
	return s.entries[s.idx]
}

// advance consumes and returns the next entry.
func (s *dirEntries) advance() *filer_pb.Entry {
	e := s.peek()
	if e != nil {
		s.idx++
	}
	return e
}

func compareDirectory(ctx context.Context,
	clientA, clientB filer_pb.FilerClient,
	dirA, dirB string,
	isActivePassive bool,
	cutoffTime time.Time,
	sem chan struct{},
	result *VerifyResult) error {

	// Hold a slot only for this directory's I/O phase (listings + merge).
	// Released before recursing so parents never block waiting for children
	// to acquire slots — see verifySyncConcurrency for the rationale.
	sem <- struct{}{}
	released := false
	releaseSlot := func() {
		if !released {
			released = true
			<-sem
		}
	}
	defer releaseSlot()

	result.dirCount.Add(1)

	// A child context cancels the load goroutines (and their listing RPCs) if
	// compareDirectory returns early, e.g. on error. Both sides load concurrently.
	mergeCtx, cancelMerge := context.WithCancel(ctx)
	defer cancelMerge()

	entriesA := newDirEntries(mergeCtx, clientA, dirA)
	entriesB := newDirEntries(mergeCtx, clientB, dirB)

	// Both sides are fully buffered before merging; surface any listing error
	// here so a failed or cancelled listing can't emit bogus diffs from partial
	// data. wait() makes entries/err visible without extra synchronisation.
	entriesA.wait()
	entriesB.wait()
	if err := entriesA.err; err != nil && err != context.Canceled {
		return fmt.Errorf("list %s on filer A: %v", dirA, err)
	}
	if err := entriesB.err; err != nil && err != context.Canceled {
		return fmt.Errorf("list %s on filer B: %v", dirB, err)
	}

	// collect subdirectories for recursive comparison
	type dirPair struct{ a, b string }
	var subDirs []dirPair

	for entriesA.peek() != nil || entriesB.peek() != nil {
		eA := entriesA.peek()
		eB := entriesB.peek()

		switch {
		case eA != nil && (eB == nil || eA.Name < eB.Name):
			// entry only in A
			entryA := entriesA.advance()
			if entryA.IsDirectory {
				// Always recurse for missing-in-B directories: a recent
				// child write can bump the parent's mtime even though
				// older missing files exist underneath. The cutoff is
				// applied per-file inside countMissingRecursive.
				reportDiff(diffMissing, dirA, entryA, nil, result)
				countMissingRecursive(ctx, clientA, path.Join(dirA, entryA.Name), cutoffTime, result)
			} else if isTooRecent(entryA, cutoffTime) {
				result.skippedRecent.Add(1)
			} else {
				reportDiff(diffMissing, dirA, entryA, nil, result)
			}

		case eB != nil && (eA == nil || eB.Name < eA.Name):
			// entry only in B
			entryB := entriesB.advance()
			if !isActivePassive {
				if isTooRecent(entryB, cutoffTime) {
					result.skippedRecent.Add(1)
				} else {
					reportDiff(diffOnlyInB, dirB, entryB, nil, result)
				}
			}

		default:
			// same name in both
			entryA := entriesA.advance()
			entryB := entriesB.advance()

			if entryA.IsDirectory && entryB.IsDirectory {
				subDirs = append(subDirs, dirPair{
					a: path.Join(dirA, entryA.Name),
					b: path.Join(dirB, entryB.Name),
				})
			} else if !entryA.IsDirectory && !entryB.IsDirectory {
				// Skip if either side was modified recently (sync-lag tolerance).
				if isTooRecent(entryA, cutoffTime) || isTooRecent(entryB, cutoffTime) {
					result.skippedRecent.Add(1)
				} else {
					compareEntries(dirA, entryA, entryB, result)
				}
			} else {
				// type mismatch: one is dir, other is file
				reportDiff(diffMissing, dirA, entryA, nil, result)
				if !isActivePassive {
					reportDiff(diffOnlyInB, dirB, entryB, nil, result)
				}
			}
		}
	}

	// Release our slot before recursing so children can acquire it. Holding
	// it across wg.Wait would deadlock once depth exceeds verifySyncConcurrency.
	releaseSlot()

	if len(subDirs) > 0 {
		// Bounded worker pool: cap goroutines per directory level instead
		// of spawning one per child. A directory with thousands of subdirs
		// would otherwise park ~2KB per waiting goroutine even though
		// only `verifySyncConcurrency` can do I/O at once.
		workers := verifySyncConcurrency
		if len(subDirs) < workers {
			workers = len(subDirs)
		}
		jobs := make(chan dirPair, len(subDirs))
		errCh := make(chan error, 1) // first error wins; others dropped
		var wg sync.WaitGroup

		for i := 0; i < workers; i++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				for pair := range jobs {
					if err := compareDirectory(ctx, clientA, clientB, pair.a, pair.b, isActivePassive, cutoffTime, sem, result); err != nil {
						select {
						case errCh <- err:
						default:
						}
					}
				}
			}()
		}
		for _, pair := range subDirs {
			jobs <- pair
		}
		close(jobs)
		wg.Wait()
		close(errCh)
		if err := <-errCh; err != nil {
			return err
		}
	}

	return nil
}

func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
	result.fileCount.Add(1)

	sizeA := filer.FileSize(entryA)
	sizeB := filer.FileSize(entryB)
	if sizeA != sizeB {
		reportDiff(diffSizeMismatch, dir, entryA, entryB, result)
		return
	}

	etagA := filer.ETag(entryA)
	etagB := filer.ETag(entryB)
	if etagA != etagB {
		if isChunkReorder(entryA, entryB) {
			// Same bytes, just a different chunk slice order — not a real diff.
			reportDiff(diffChunkReorder, dir, entryA, entryB, result)
		} else {
			reportDiff(diffETagMismatch, dir, entryA, entryB, result)
		}
		return
	}
}

// isChunkReorder reports whether two entries with differing file ETags are
// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes
// per-chunk MD5s in stored order without sorting by offset, so the same bytes
// written by different paths (S3 multipart vs filer.backup) can hash
// differently. If both chunk lists sorted by offset match element-wise on
// (offset, size, ETag), the files cover the same bytes with the same content
// and only the slice order differed.
//
// Applies only when both ETags come from chunks (a stored attr.Md5 is
// order-independent, so a mismatch there is a real content difference) and the
// chunks form a manifest-free, non-overlapping set whose bytes are fully
// determined by the list — overlaps would need timestamp-based visible-interval
// resolution, so those stay ETAG_MISMATCH.
func isChunkReorder(entryA, entryB *filer_pb.Entry) bool {
	if !usesChunkETag(entryA) || !usesChunkETag(entryB) {
		return false
	}
	a := sortedSimpleChunks(entryA.GetChunks())
	b := sortedSimpleChunks(entryB.GetChunks())
	// nil means overlapping or manifest chunks we do not fast-path; a single
	// chunk hashes directly and a differing count is a real divergence.
	if a == nil || b == nil || len(a) != len(b) || len(a) < 2 {
		return false
	}
	for i := range a {
		if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag {
			return false
		}
	}
	return true
}

// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry,
// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's
// own condition so it agrees with how the compared ETag was computed.
func usesChunkETag(entry *filer_pb.Entry) bool {
	return entry.Attributes == nil || entry.Attributes.Md5 == nil
}

// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a
// manifest-free, non-overlapping set whose visible bytes are fully determined
// by the list and every chunk carries a usable per-chunk ETag; otherwise nil.
// Overlapping ranges would require resolving the visible interval by chunk
// timestamp, which this fast path does not attempt.
func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
	sorted := make([]*filer_pb.FileChunk, len(chunks))
	copy(sorted, chunks)
	sort.Slice(sorted, func(i, j int) bool {
		if sorted[i].Offset != sorted[j].Offset {
			return sorted[i].Offset < sorted[j].Offset
		}
		return sorted[i].ETag < sorted[j].ETag
	})
	prevEnd := int64(0)
	for i, c := range sorted {
		if c.IsChunkManifest {
			return nil
		}
		// An empty or undecodable ETag is not a content fingerprint, so
		// element-wise (offset, size, ETag) equality cannot prove the bytes
		// match — decline rather than assert a false content-equality.
		if len(util.Base64Md5ToBytes(c.ETag)) == 0 {
			return nil
		}
		if i > 0 && c.Offset < prevEnd {
			return nil
		}
		prevEnd = c.Offset + int64(c.Size)
	}
	return sorted
}

// mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil.
// Returns relation, absolute delta in seconds, and human-readable string.
func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) {
	if entryA == nil || entryB == nil || entryA.Attributes == nil || entryB.Attributes == nil {
		return "", "", 0
	}
	delta := entryB.Attributes.Mtime - entryA.Attributes.Mtime
	abs := delta
	if abs < 0 {
		abs = -abs
	}
	switch {
	case delta == 0:
		return "EQUAL", "0s", 0
	case delta > 0:
		return "B_NEWER", formatSeconds(abs), abs
	default:
		return "A_NEWER", formatSeconds(abs), abs
	}
}

func formatSeconds(s int64) string {
	switch {
	case s < 60:
		return fmt.Sprintf("%ds", s)
	case s < 3600:
		return fmt.Sprintf("%dm", s/60)
	case s < 86400:
		return fmt.Sprintf("%dh", s/3600)
	default:
		return fmt.Sprintf("%dd", s/86400)
	}
}

// hintFor returns an automatic interpretation hint based on mtime relation.
// Only emitted for SIZE_MISMATCH and ETAG_MISMATCH where both entries exist.
func hintFor(relation string) string {
	switch relation {
	case "B_NEWER":
		return "late_updates_skip_likely"
	case "A_NEWER":
		return "sync_lag_or_event_miss"
	}
	return ""
}

func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
	switch diffType {
	case diffMissing:
		result.missingCount.Add(1)
	case diffOnlyInB:
		result.onlyInB.Add(1)
	case diffSizeMismatch:
		result.sizeMismatch.Add(1)
	case diffETagMismatch:
		result.etagMismatch.Add(1)
	case diffChunkReorder:
		result.chunkReorder.Add(1)
	}

	if result.jsonOutput {
		writeJSONDiff(result, diffType, dir, entryA, entryB)
	} else {
		// A chunk reorder is content-equal, not a real diff: keep it out of the
		// default text report and list it only at -v=1. JSON always emits it so
		// the per-record stream matches the summary count.
		if diffType == diffChunkReorder && !glog.V(1) {
			return
		}
		writeTextDiff(result, diffType, dir, entryA, entryB)
	}
}

func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
	entryPath := path.Join(dir, entryA.Name)

	result.outputMu.Lock()
	defer result.outputMu.Unlock()

	switch diffType {
	case diffMissing:
		if entryA.IsDirectory {
			fmt.Fprintf(os.Stdout, "[MISSING]       %s/ (directory)\n", entryPath)
		} else {
			fmt.Fprintf(os.Stdout, "[MISSING]       %s (size=%d, etag=%s)\n",
				entryPath, filer.FileSize(entryA), filer.ETag(entryA))
		}
	case diffOnlyInB:
		fmt.Fprintf(os.Stdout, "[ONLY_IN_B]     %s\n", entryPath)
	case diffSizeMismatch:
		ann := annotation(entryA, entryB)
		fmt.Fprintf(os.Stdout, "[SIZE_MISMATCH] %s (a=%d, b=%d%s)\n",
			entryPath, filer.FileSize(entryA), filer.FileSize(entryB), ann)
	case diffETagMismatch:
		ann := annotation(entryA, entryB)
		fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n",
			entryPath, filer.ETag(entryA), filer.ETag(entryB), ann)
	case diffChunkReorder:
		fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n",
			entryPath, filer.ETag(entryA), filer.ETag(entryB))
	}
}

// annotation builds the trailing ", mtime: ... [hint]" segment for text output.
// Returns empty string if entries are unavailable.
func annotation(entryA, entryB *filer_pb.Entry) string {
	relation, delta, _ := mtimeRelation(entryA, entryB)
	if relation == "" {
		return ""
	}
	switch relation {
	case "EQUAL":
		return ", mtime equal [chunk-level issue]"
	case "B_NEWER":
		return fmt.Sprintf(", B newer +%s [late-updates skip likely]", delta)
	case "A_NEWER":
		return fmt.Sprintf(", A newer +%s [sync lag or event miss]", delta)
	}
	return ""
}

func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
	rec := diffRecord{Path: path.Join(dir, entryA.Name)}

	switch diffType {
	case diffMissing:
		rec.Type = "MISSING"
		rec.IsDirectory = entryA.IsDirectory
		rec.A = toEntryRecord(entryA)
	case diffOnlyInB:
		rec.Type = "ONLY_IN_B"
		// for diffOnlyInB the existing convention passes the entry as entryA
		rec.IsDirectory = entryA.IsDirectory
		rec.B = toEntryRecord(entryA)
	case diffSizeMismatch:
		rec.Type = "SIZE_MISMATCH"
		rec.A = toEntryRecord(entryA)
		rec.B = toEntryRecord(entryB)
		relation, delta, _ := mtimeRelation(entryA, entryB)
		rec.MtimeRelation = relation
		rec.MtimeDelta = delta
		rec.Hint = hintFor(relation)
	case diffETagMismatch:
		rec.Type = "ETAG_MISMATCH"
		rec.A = toEntryRecord(entryA)
		rec.B = toEntryRecord(entryB)
		relation, delta, _ := mtimeRelation(entryA, entryB)
		rec.MtimeRelation = relation
		rec.MtimeDelta = delta
		rec.Hint = hintFor(relation)
	case diffChunkReorder:
		rec.Type = "CHUNK_REORDER"
		rec.A = toEntryRecord(entryA)
		rec.B = toEntryRecord(entryB)
		relation, delta, _ := mtimeRelation(entryA, entryB)
		rec.MtimeRelation = relation
		rec.MtimeDelta = delta
		// no hint: content-equal, no action needed
	}

	writeJSONLine(result, rec)
}

func toEntryRecord(entry *filer_pb.Entry) *entryRecord {
	if entry == nil {
		return nil
	}
	r := &entryRecord{
		Size: filer.FileSize(entry),
		ETag: filer.ETag(entry),
	}
	if entry.Attributes != nil {
		r.Mtime = entry.Attributes.Mtime
	}
	return r
}

// writeJSONLine emits a single JSON object followed by newline. Holds outputMu
// across marshal+write so concurrent goroutines never interleave.
func writeJSONLine(result *VerifyResult, v any) {
	data, err := json.Marshal(v)
	if err != nil {
		glog.Warningf("marshal verify record: %v", err)
		return
	}
	result.outputMu.Lock()
	defer result.outputMu.Unlock()
	os.Stdout.Write(data)
	os.Stdout.Write([]byte{'\n'})
}

func countMissingRecursive(ctx context.Context, client filer_pb.FilerClient, dir string, cutoffTime time.Time, result *VerifyResult) {
	err := filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
		func(entry *filer_pb.Entry, isLast bool) error {
			if entry.IsDirectory {
				countMissingRecursive(ctx, client, path.Join(dir, entry.Name), cutoffTime, result)
				return nil
			}
			if !cutoffTime.IsZero() && entry.Attributes != nil && entry.Attributes.Mtime > cutoffTime.Unix() {
				result.skippedRecent.Add(1)
				return nil
			}
			reportDiff(diffMissing, dir, entry, nil, result)
			return nil
		})
	if err != nil {
		glog.Warningf("list missing directory %s: %v", dir, err)
	}
}