seaweedfs/test/fuse_p2p/peer_chunk_sharing_test.go

package fuse_p2p

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"math/rand/v2"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

// peerReadSuccessMarker is the log line tryPeerRead emits when a peer
// fetch succeeded. The test greps non-writer mount logs for it to
// prove the p2p path fired. At glog verbosity 4 (framework sets -v=4).
const peerReadSuccessMarker = "peer read successful"

// seedConvergenceTimeout bounds how long the test waits for:
//   - the filer registry to list all mounts,
//   - every mount's seed view (via MountList poll) to include all peers,
//   - the first announcer flush cycle to publish chunk holders.
//
// With defaults the mount polls MountList every 30 s and flushes
// ChunkAnnounce every 15 s. Allowing 45 s absorbs one MountList refresh
// plus one announce cycle plus some slop for CI variance.
const seedConvergenceTimeout = 90 * time.Second

// TestPeerChunkSharing_ReadersPullFromPeerCache is the headline p2p
// integration test. It proves at least one non-writer mount can satisfy
// a read from the writer's chunk cache instead of the volume tier.
//
//  1. Bring up 1 master/volume/filer + 3 mounts, all with -peer.enable.
//  2. Mount 0 writes a ~8 MiB file and reads it back so chunks land in
//     its local cache and the announcer publishes them.
//  3. Wait for seed convergence + at least one announcer flush cycle.
//  4. BOTH mount 1 and mount 2 read the file.
//
// Why both readers: with 3 mounts, HRW picks one owner for the chunk.
// If that owner is mount 1, only mount 2's read will hit the peer
// path (mount 1's tryPeerRead bails on owner==self). If that owner is
// mount 2, only mount 1's read will. If that owner is mount 0 (the
// writer), both can. So by reading from both, we deterministically
// guarantee at least one non-writer mount exercises the peer path.
//
// Once the peer fetch populates its local cache, subsequent reads
// short-circuit on IsInCache — so we only get one real shot per mount
// per chunk. That's fine: one success is all the test needs.
func TestPeerChunkSharing_ReadersPullFromPeerCache(t *testing.T) {
	c := startP2PTestCluster(t, 3)

	// ~8 MiB, pseudo-random so compression doesn't collapse it to one block.
	payload := make([]byte, 8*1024*1024)
	rng := rand.New(rand.NewPCG(1, 2))
	for i := range payload {
		payload[i] = byte(rng.Uint32())
	}

	const relPath = "p2p-test.bin"
	writer := c.MountDir(0)
	require.NoError(t, os.WriteFile(filepath.Join(writer, relPath), payload, 0644))

	// Warm mount 0's chunk cache by reading back through its own FUSE.
	// Without this the chunks are on the volume server but not yet
	// in anyone's peer-servable cache.
	readBack, err := os.ReadFile(filepath.Join(writer, relPath))
	require.NoError(t, err)
	require.True(t, bytes.Equal(readBack, payload), "write-then-read on writer mount should match")

	waitForSeedConvergence(t, c, seedConvergenceTimeout)

	// Give the announcer several flush windows to push chunk-holder
	// entries to the HRW owners. First flush may see the writer's seed
	// view incomplete (only self) and defer all fids; subsequent
	// flushes re-check against a refreshed seed view. announce interval
	// is 15 s, so 45 s covers three attempts.
	time.Sleep(45 * time.Second)

	// Read from both non-writer mounts. For any HRW outcome on any
	// chunk, at least one of these reads will NOT have the reader as
	// the HRW owner, so its tryPeerRead will proceed to ChunkLookup +
	// FetchChunk.
	for _, idx := range []int{1, 2} {
		got, err := os.ReadFile(filepath.Join(c.MountDir(idx), relPath))
		require.NoError(t, err, "read from mount %d must succeed\n--- mount%d ---\n%s",
			idx, idx, tailLines(c.MountLog(idx), 80))
		require.Equal(t, md5.Sum(payload), md5.Sum(got),
			"mount %d returned mismatched bytes (len got=%d want=%d)", idx, len(got), len(payload))
	}

	// Content matches alone doesn't prove p2p — the volume fallback
	// would also satisfy the reads. Require at least one non-writer
	// mount's log to contain the peer-read success marker.
	var sawPeerRead bool
	for _, idx := range []int{1, 2} {
		if strings.Contains(c.MountLog(idx), peerReadSuccessMarker) {
			sawPeerRead = true
			break
		}
	}
	if !sawPeerRead {
		t.Fatalf("no non-writer mount logged %q — peer read path never fired.\n"+
			"--- mount0 (writer) tail ---\n%s\n--- mount1 tail ---\n%s\n--- mount2 tail ---\n%s",
			peerReadSuccessMarker,
			tailLines(c.MountLog(0), 60), tailLines(c.MountLog(1), 60), tailLines(c.MountLog(2), 60))
	}
}

// waitForSeedConvergence polls each mount's log looking for any sign
// that MountList returned a peer list containing the other mounts.
func waitForSeedConvergence(t *testing.T, c *p2pTestCluster, timeout time.Duration) {
	t.Helper()
	// The first MountRegister happens synchronously during mount startup,
	// and the first MountList is pulled right after. 30 s is the refresh
	// interval; waiting one full cycle here guarantees every mount has
	// at minimum observed the others in its seed view.
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		ready := true
		for i := range c.mountCmds {
			if !strings.Contains(c.MountLog(i), "peer-grpc listening on") {
				ready = false
				break
			}
		}
		if ready {
			time.Sleep(30 * time.Second)
			return
		}
		time.Sleep(500 * time.Millisecond)
	}
	t.Fatalf("peer-grpc servers did not come up within %s", timeout)
}

// tailLines returns the last n newline-delimited lines of s, or the
// whole thing if shorter. Keeps test failures readable.
func tailLines(s string, n int) string {
	lines := strings.Split(s, "\n")
	if len(lines) <= n {
		return s
	}
	return fmt.Sprintf("... (%d earlier lines omitted) ...\n%s",
		len(lines)-n, strings.Join(lines[len(lines)-n:], "\n"))
}