test: production-shaped push/pull benchmark with per-backend request counts

TestBenchRealImages pushes and pulls three images whose layer sizes are
copied from real manifests in the production appview database (the median,
p75 and p90 images by layer count) and reports, per operation, wall time and
the number of requests to the registry, the fake PDS, the hold and S3, broken
down by endpoint. Skipped unless BENCH_PROFILES is set, so the integration
target does not run it. BENCH_LAT_{PDS,HOLD,S3} inject per-request latency,
which is what makes byte-path changes visible in-process; request counts are
the reliable signal either way.

internal/reqcount counts and delays requests through a handler wrapper and a
client-side RoundTripper. testharness.WithBackendTap wraps the PDS and S3
handlers and puts a counting reverse proxy in front of the hold;
testpds.WithMiddleware is the hook that makes the PDS side possible.

The bench showed a pull costs three hold calls per blob, not two: distribution
installs its notifications listener unconditionally and it re-Stats every blob
after ServeBlob to build the pull event. The backlog's presign memoization
item is rewritten with the measured numbers.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WTdBxLFU5TpwmqVdVsN1wq

internal/reqcount/reqcount.go

@@ -0,0 +1,174 @@
+// Package reqcount counts HTTP requests per backend and can inject a fixed
+// latency in front of each, so an in-process benchmark of the ATCR stack can
+// report round trips per operation. Round-trip counts are the reliable signal
+// in-process: with no latency injected, wall time is dominated by gofakes3's
+// in-memory copies and crane's client-side gzip, not by ATCR code.
+package reqcount
+
+import (
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Tap wraps handlers and counts what passes through them.
+type Tap struct {
+	mu      sync.Mutex
+	counts  map[string]map[string]int
+	latency map[string]time.Duration
+}
+
+// New returns an empty Tap.
+func New() *Tap {
+	return &Tap{
+		counts:  make(map[string]map[string]int),
+		latency: make(map[string]time.Duration),
+	}
+}
+
+// SetLatency makes every request to backend sleep for d before it is served.
+func (t *Tap) SetLatency(backend string, d time.Duration) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.latency[backend] = d
+}
+
+// Wrap returns next wrapped with counting and latency for backend.
+func (t *Tap) Wrap(backend string, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		key := classify(backend, r)
+		t.mu.Lock()
+		m := t.counts[backend]
+		if m == nil {
+			m = make(map[string]int)
+			t.counts[backend] = m
+		}
+		m[key]++
+		d := t.latency[backend]
+		t.mu.Unlock()
+		if d > 0 {
+			time.Sleep(d)
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// Reset clears every count.
+func (t *Tap) Reset() {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.counts = make(map[string]map[string]int)
+}
+
+// Snapshot returns a copy of the counts: backend -> key -> requests.
+func (t *Tap) Snapshot() map[string]map[string]int {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	out := make(map[string]map[string]int, len(t.counts))
+	for b, m := range t.counts {
+		c := make(map[string]int, len(m))
+		for k, n := range m {
+			c[k] = n
+		}
+		out[b] = c
+	}
+	return out
+}
+
+// Total sums one backend's counts.
+func Total(m map[string]int) int {
+	n := 0
+	for _, v := range m {
+		n += v
+	}
+	return n
+}
+
+// Keys returns a backend's keys sorted by count, largest first.
+func Keys(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		if m[keys[i]] != m[keys[j]] {
+			return m[keys[i]] > m[keys[j]]
+		}
+		return keys[i] < keys[j]
+	})
+	return keys
+}
+
+// classify turns a request into a short, stable key for its backend.
+func classify(backend string, r *http.Request) string {
+	q := r.URL.Query()
+	switch backend {
+	case "s3":
+		switch {
+		case q.Has("uploads"):
+			return "POST initiate"
+		case q.Has("uploadId") && q.Has("partNumber"):
+			return "PUT part"
+		case q.Has("uploadId") && r.Method == http.MethodPost:
+			return "POST complete"
+		case q.Has("uploadId") && r.Method == http.MethodDelete:
+			return "DELETE abort"
+		case r.Header.Get("X-Amz-Copy-Source") != "":
+			return "PUT copy"
+		}
+		return r.Method + " object"
+	case "hold":
+		p := strings.TrimPrefix(r.URL.Path, "/xrpc/")
+		if strings.HasSuffix(p, "sync.getBlob") && q.Get("method") != "" {
+			p += "?method=" + q.Get("method")
+		}
+		return r.Method + " " + p
+	default:
+		return r.Method + " " + strings.TrimPrefix(r.URL.Path, "/xrpc/")
+	}
+}
+
+// Transport wraps an http.RoundTripper so client-side requests are counted
+// under backend, keyed by method and a coarse path class. It is meant for the
+// registry itself, which sits in front of the tapped backends.
+func (t *Tap) Transport(backend string, base http.RoundTripper) http.RoundTripper {
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	return roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		t.mu.Lock()
+		m := t.counts[backend]
+		if m == nil {
+			m = make(map[string]int)
+			t.counts[backend] = m
+		}
+		m[r.Method+" "+pathClass(r.URL.Path)]++
+		t.mu.Unlock()
+		return base.RoundTrip(r)
+	})
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func pathClass(p string) string {
+	switch {
+	case !strings.HasPrefix(p, "/v2/") && !strings.HasPrefix(p, "/auth/"):
+		// A redirect the client followed, to S3 or elsewhere.
+		return "redirect"
+	case strings.Contains(p, "/blobs/uploads"):
+		return "blobs/uploads"
+	case strings.Contains(p, "/blobs/"):
+		return "blobs"
+	case strings.Contains(p, "/manifests/"):
+		return "manifests"
+	case strings.HasPrefix(p, "/auth/"):
+		return "auth"
+	case p == "/v2/" || p == "/v2":
+		return "ping"
+	}
+	return "other"
+}

internal/testharness/harness.go

@@ -9,6 +9,8 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -36,11 +38,20 @@ import (
 type Option func(*options)
 
 type options struct {
+	tap         func(backend string, next http.Handler) http.Handler
 	quota       *quota.Config
 	billing     *billing.Config
 	privateHold bool
 }
 
+// WithBackendTap wraps every backend the stack talks to: the fake PDS handler
+// ("pds"), the gofakes3 handler ("s3"), and a reverse proxy placed in front of
+// the hold ("hold") so the appview's XRPC calls pass through it. Benchmarks
+// use it to count round trips per operation and to inject latency.
+func WithBackendTap(tap func(backend string, next http.Handler) http.Handler) Option {
+	return func(o *options) { o.tap = tap }
+}
+
 // WithPrivateHold builds the hold with captain.Public = false. Reads then
 // require the owner or a crew member, so anonymous pulls are refused and a
 // PDS-known stranger is refused too — the mirror of the default public hold,
@@ -114,7 +125,13 @@ func New(t *testing.T, opts ...Option) *Harness {
 	h := &Harness{t: t}
 
 	// 1. Fake PDS.
-	h.PDS = testpds.New(t)
+	var pdsOpts []testpds.Option
+	if o.tap != nil {
+		pdsOpts = append(pdsOpts, testpds.WithMiddleware(func(next http.Handler) http.Handler {
+			return o.tap("pds", next)
+		}))
+	}
+	h.PDS = testpds.New(t, pdsOpts...)
 	atproto.SetDirectory(h.PDS.Directory())
 	t.Cleanup(func() {
 		// Reset to a fresh default so a later non-test process won't see our
@@ -128,7 +145,11 @@ func New(t *testing.T, opts ...Option) *Harness {
 		t.Fatalf("create test bucket: %v", err)
 	}
 	faker := gofakes3.New(backend)
-	s3ts := httptest.NewServer(faker.Server())
+	var s3Handler = faker.Server()
+	if o.tap != nil {
+		s3Handler = o.tap("s3", s3Handler)
+	}
+	s3ts := httptest.NewServer(s3Handler)
 	t.Cleanup(s3ts.Close)
 	h.S3URL = s3ts.URL
 
@@ -139,6 +160,21 @@ func New(t *testing.T, opts ...Option) *Harness {
 	}
 	holdAddr := holdListener.Addr().String()
 	holdPublicURL := "http://" + holdAddr
+	if o.tap != nil {
+		// Everything that addresses the hold by its public URL (the appview,
+		// the DID document, the did:web itself) goes through a counting
+		// reverse proxy; the hold keeps serving on its own listener behind it.
+		proxyListener, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatalf("hold proxy listen: %v", err)
+		}
+		target, _ := url.Parse("http://" + holdAddr)
+		proxy := httputil.NewSingleHostReverseProxy(target)
+		proxySrv := &http.Server{Handler: o.tap("hold", proxy)}
+		go func() { _ = proxySrv.Serve(proxyListener) }()
+		t.Cleanup(func() { _ = proxySrv.Close() })
+		holdPublicURL = "http://" + proxyListener.Addr().String()
+	}
 	h.HoldURL = holdPublicURL
 	h.HoldDID = atprotodid.GenerateDIDFromURL(holdPublicURL)