at-container-registry/scanner/internal/e2e/protocol_test.go

package e2e

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	scanner "atcr.io/scanner"
	"atcr.io/scanner/internal/client"
	"atcr.io/scanner/internal/config"
	"atcr.io/scanner/internal/mockhold"
	"atcr.io/scanner/internal/queue"
)

// --- helpers ---------------------------------------------------------------

// silenceWindow is how long a test waits before concluding the scanner sent
// nothing back. It is short on purpose: the failure mode being pinned is
// "nothing, ever", and the hold's own ackTimeout is five minutes, so anything
// the scanner has not said within a second here it will not say at all.
const silenceWindow = 1 * time.Second

// synthJob builds a descriptor-only container image job. The digests point at
// nothing, which is fine for every scenario that either refuses the job before
// downloading or deliberately stalls the download.
func synthJob(repo string) *scanner.ScanJob {
	return &scanner.ScanJob{
		ManifestDigest: "sha256:" + strings.Repeat("a", 64),
		Repository:     repo,
		Tag:            "latest",
		Tier:           "deckhand",
		HoldDID:        "did:web:hold.example",
		Config: scanner.BlobDescriptor{
			Digest:    "sha256:" + strings.Repeat("c", 64),
			Size:      100,
			MediaType: "application/vnd.oci.image.config.v1+json",
		},
		Layers: []scanner.BlobDescriptor{{
			Digest:    "sha256:" + strings.Repeat("1", 64),
			Size:      200,
			MediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
		}},
	}
}

// helmJob returns a corpus Helm chart, the cheapest job that reaches a
// terminal message: the scanner refuses it on config media type alone, before
// any tmp dir or download work, so it is a reliable liveness probe.
func helmJob(t *testing.T) *scanner.ScanJob {
	t.Helper()
	charts, err := mockhold.CorpusByShape(mockhold.ShapeHelm)
	if err != nil {
		t.Fatalf("load corpus: %v", err)
	}
	if len(charts) == 0 {
		t.Fatal("corpus contains no helm manifests")
	}
	return charts[0].Job()
}

// rawJob renders a job frame from a map, so a test can omit a field, or give
// it a shape the scanner cannot decode.
func rawJob(t *testing.T, fields map[string]any) []byte {
	t.Helper()
	data, err := json.Marshal(fields)
	if err != nil {
		t.Fatalf("marshal raw job: %v", err)
	}
	return data
}

// expectSilence fails if any message matching match arrives within the window.
func expectSilence(t *testing.T, h *Harness, match func(mockhold.Message) bool, window time.Duration, what string) {
	t.Helper()
	msg, err := h.Hold.WaitForMessage(match, window)
	if err == nil {
		t.Fatalf("%s: expected no reply, got %s for seq %d", what, msg.Type, msg.Seq)
	}
}

// forSeq matches any message about one job.
func forSeq(seq int64) func(mockhold.Message) bool {
	return func(m mockhold.Message) bool { return m.Seq == seq }
}

// assertAlive proves the WebSocket survived whatever the previous step did to
// it, by pushing a job the scanner is guaranteed to answer.
func assertAlive(t *testing.T, h *Harness) {
	t.Helper()
	seq, err := h.Hold.SendJob(helmJob(t))
	if err != nil {
		t.Fatalf("connection did not survive: %v", err)
	}
	if msg := h.AwaitTerminal(t, seq, 30*time.Second); msg.Type != "skipped" {
		t.Fatalf("liveness probe: want skipped, got %s", msg.Type)
	}
}

// --- 1. frames the scanner cannot parse -------------------------------------

// TestUnparseableFramesAreAnsweredWithSkipped is the central protocol finding,
// and the shape of a nine-day outage.
//
// connectOnce decodes three things — the frame, then the config sub-document,
// then the layers sub-document — and every failure branch used to be
// slog.Error followed by continue, sending nothing back at all. The ack was
// sent only after both sub-document unmarshals, so a job whose config or
// layers did not decode was never even acknowledged.
//
// The hold has already written status='assigned' for that seq before it wrote
// the frame. Its only escape was the five-minute ackTimeout in
// reDispatchTimedOut, after which the row was re-offered — to the same
// scanner, which dropped it again for exactly the same reason, because a
// decoding disagreement is permanent. And because hasActiveJobs counts
// 'assigned' rows and dispatchLoop admits one proactive candidate at a time
// behind waitForCapacity, one such row meant no proactive scan was ever
// dispatched again, deployment-wide.
//
// The scanner now answers "skipped" for any frame that carries a usable seq.
// Skipped is the correct verdict rather than "error": the hold retries
// failures on the rescan interval, and no retry of an undecodable frame can
// ever succeed, whereas handleSkipped writes a terminal record and releases
// the row and the in-flight digest for good. The hold-side half is
// TestScanSkipped_RetiresAnUndecodableFrame in
// pkg/hold/pds/scan_broadcaster_stuck_test.go.
func TestUnparseableFramesAreAnsweredWithSkipped(t *testing.T) {
	h := Start(t, mockhold.NewMemory())

	base := func(seq int64) map[string]any {
		return map[string]any{
			"type":           "job",
			"seq":            seq,
			"manifestDigest": "sha256:" + strings.Repeat("a", 64),
			"repository":     "probe/unparseable",
			"tag":            "latest",
			"userDid":        "did:plc:probe",
			"holdDid":        "did:web:hold.example",
			"holdEndpoint":   h.Hold.URL(),
			"tier":           "deckhand",
			"config":         map[string]any{"digest": "sha256:" + strings.Repeat("c", 64), "size": 1, "mediaType": "application/vnd.oci.image.config.v1+json"},
			"layers":         []any{},
		}
	}

	cases := []struct {
		name  string
		frame func(seq int64) []byte
	}{
		{"config is a string", func(seq int64) []byte {
			f := base(seq)
			f["config"] = "not-an-object"
			return rawJob(t, f)
		}},
		{"config field absent", func(seq int64) []byte {
			f := base(seq)
			delete(f, "config")
			return rawJob(t, f)
		}},
		{"layers is an object", func(seq int64) []byte {
			f := base(seq)
			f["layers"] = map[string]any{"oops": 1}
			return rawJob(t, f)
		}},
		{"layers field absent", func(seq int64) []byte {
			f := base(seq)
			delete(f, "layers")
			return rawJob(t, f)
		}},
		// encoding/json records the first type error and keeps decoding the
		// rest of the object, so a frame the top-level unmarshal rejects can
		// still yield the seq that addresses the hold's row. That row must be
		// answered too, which is why the reply is keyed on the seq rather than
		// on which of the three decodes failed.
		{"frame has a type-mismatched field", func(seq int64) []byte {
			f := base(seq)
			f["tier"] = 12345
			return rawJob(t, f)
		}},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			seq := h.Hold.NextSeq()
			if err := h.Hold.SendRaw(tc.frame(seq)); err != nil {
				t.Fatalf("send raw frame: %v", err)
			}
			msg, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool {
				return m.Seq == seq && m.Type != "ack"
			}, 10*time.Second)
			if err != nil {
				t.Fatalf("%s: no reply for seq %d; the hold's row stays "+
					"assigned and is re-offered to this same scanner forever. "+
					"transcript: %s", tc.name, seq, describe(h.Hold.Transcript()))
			}
			if msg.Type != "skipped" {
				t.Fatalf("%s: reply for seq %d was %s (%s%s), want skipped: an "+
					"undecodable frame is a permanent condition and the hold "+
					"retries anything it records as a failure",
					tc.name, seq, msg.Type, msg.Error, msg.Reason)
			}
			if msg.Reason == "" {
				t.Errorf("%s: skipped with no reason; the hold stores it on the "+
					"scan record and it is all a user ever sees", tc.name)
			}
		})
	}

	// The connection survives all of it, which is what made the retry loop
	// infinite rather than self-limiting.
	assertAlive(t, h)
}

// TestFramesWithNoUsableSeqAreDroppedSilently is the deliberate exception.
//
// A frame that does not decode far enough to yield a seq addresses no job:
// there is no row to retire and no seq to put in a reply, so logging is the
// only thing left. This is safe in a way the config/layers case never was —
// the hold writes status='assigned' keyed by seq before it sends, so a frame
// whose seq never made it onto the wire cannot be the frame that stranded a
// row. The connection must survive, since the hold will keep using it.
func TestFramesWithNoUsableSeqAreDroppedSilently(t *testing.T) {
	h := Start(t, mockhold.NewMemory())

	for _, tc := range []struct {
		name  string
		frame []byte
	}{
		{"frame is not JSON", []byte("{ this is not json")},
		{"frame is a JSON array", []byte(`[1,2,3]`)},
		{"frame carries seq 0", rawJob(t, map[string]any{"type": "job", "seq": 0, "config": "nope"})},
	} {
		t.Run(tc.name, func(t *testing.T) {
			before := len(h.Hold.Transcript())
			if err := h.Hold.SendRaw(tc.frame); err != nil {
				t.Fatalf("send raw frame: %v", err)
			}
			expectSilence(t, h, func(m mockhold.Message) bool {
				return m.Seq == 0
			}, silenceWindow, tc.name)
			if got := len(h.Hold.Transcript()); got != before {
				t.Errorf("%s: scanner sent %d messages for an unaddressable frame",
					tc.name, got-before)
			}
		})
	}

	assertAlive(t, h)
}

// TestNullConfigIsAckedThenFailsRetryably covers the shape that *does* decode:
// a JSON null unmarshals into a zero BlobDescriptor without error, so the job
// is acked and enters the pipeline, then dies in buildOCILayout on the empty
// config digest. That is reported as "error", which the hold treats as
// transient and retries on the rescan interval forever, even though no retry
// can ever succeed: nothing about a null config will change.
func TestNullConfigIsAckedThenFailsRetryably(t *testing.T) {
	h := Start(t, mockhold.NewMemory())

	seq := h.Hold.NextSeq()
	frame := rawJob(t, map[string]any{
		"type":           "job",
		"seq":            seq,
		"manifestDigest": "sha256:" + strings.Repeat("b", 64),
		"repository":     "probe/null-config",
		"tag":            "latest",
		"userDid":        "did:plc:probe",
		"holdDid":        "did:web:hold.example",
		"holdEndpoint":   h.Hold.URL(),
		"tier":           "deckhand",
		"config":         nil,
		"layers": []any{map[string]any{
			"digest":    "sha256:" + strings.Repeat("1", 64),
			"size":      10,
			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
		}},
	})
	if err := h.Hold.SendRaw(frame); err != nil {
		t.Fatalf("send raw frame: %v", err)
	}

	if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool {
		return m.Seq == seq && m.Type == "ack"
	}, 10*time.Second); err != nil {
		t.Fatalf("null config was not acked: %v", err)
	}

	msg := h.AwaitTerminal(t, seq, 30*time.Second)
	if msg.Type != "error" {
		t.Fatalf("want the current retryable-error behaviour, got %s (%s%s)", msg.Type, msg.Error, msg.Reason)
	}
	if !strings.Contains(msg.Error, "empty digest") {
		t.Errorf("unexpected error text %q", msg.Error)
	}
	t.Logf("null config is a permanent condition reported as a retryable error: %q", msg.Error)
}

// TestUnknownMessageTypeIsIgnored pins the benign half: a frame the scanner
// does not recognise is logged and skipped, the connection survives, and
// nothing is sent back. Harmless today because the hold only ever sends
// "job", but it means any future message type is silently swallowed by an
// older scanner rather than refused.
func TestUnknownMessageTypeIsIgnored(t *testing.T) {
	h := Start(t, mockhold.NewMemory())

	seq := h.Hold.NextSeq()
	if err := h.Hold.SendRaw(rawJob(t, map[string]any{"type": "cancel", "seq": seq})); err != nil {
		t.Fatalf("send raw frame: %v", err)
	}
	expectSilence(t, h, forSeq(seq), silenceWindow, "unknown message type")
	assertAlive(t, h)
}

// --- 2. duplicate delivery --------------------------------------------------

// TestDuplicateSeqIsProcessedTwice shows the scanner has no idea it has seen a
// job before. Nothing dedupes on seq or on manifest digest: the job is acked
// twice, queued twice, and scanned twice.
//
// On the hold side the second ack is a no-op (handleAck's UPDATE is guarded by
// status='assigned', which the first ack already cleared) but the second
// terminal message is not: handleSkipped/handleResult/handleError re-run
// unconditionally, writing a second scan record to the PDS for the same
// manifest. For a real image this is also a full second download and Syft run.
func TestDuplicateSeqIsProcessedTwice(t *testing.T) {
	h := Start(t, mockhold.NewMemory())

	job := helmJob(t)
	job.Seq = h.Hold.NextSeq()
	for i := 0; i < 2; i++ {
		dup := *job
		if _, err := h.Hold.SendJob(&dup); err != nil {
			t.Fatalf("send job %d: %v", i, err)
		}
	}

	deadline := time.Now().Add(30 * time.Second)
	var acks, terminals int
	for time.Now().Before(deadline) {
		acks, terminals = 0, 0
		for _, m := range h.Hold.Transcript() {
			if m.Seq != job.Seq {
				continue
			}
			switch m.Type {
			case "ack":
				acks++
			case "result", "error", "skipped":
				terminals++
			}
		}
		if acks >= 2 && terminals >= 2 {
			break
		}
		time.Sleep(20 * time.Millisecond)
	}

	if acks != 2 || terminals != 2 {
		t.Fatalf("duplicate seq: got %d acks and %d terminal messages, want 2 and 2", acks, terminals)
	}
	t.Logf("seq %d was acked %d times and answered %d times: no dedup anywhere in the scanner",
		job.Seq, acks, terminals)
}

// --- 3. queue capacity ------------------------------------------------------

// TestQueueFullIsReportedAsRetryableError pins what happens past the queue's
// high-water mark: the job is acked (hold: assigned -> processing) and then
// immediately answered with "error: scanner queue full", which the hold
// records as a *failure*. Failures are retryable, so the same job comes back
// on the rescan interval and will overflow again for as long as the backlog
// persists. A capacity signal is being reported through the channel reserved
// for scan outcomes, and it lands in the user's scan history as a failed scan.
func TestQueueFullIsReportedAsRetryableError(t *testing.T) {
	const queueSize = 2
	h := Start(t, mockhold.NewMemory(),
		WithWorkers(0), // nothing drains the queue, so the Nth job is deterministic
		func(c *config.Config) { c.Scanner.QueueSize = queueSize })

	var seqs []int64
	for i := 0; i < queueSize+2; i++ {
		seq, err := h.Hold.SendJob(synthJob("probe/overflow"))
		if err != nil {
			t.Fatalf("send job %d: %v", i, err)
		}
		seqs = append(seqs, seq)
	}

	for _, seq := range seqs[queueSize:] {
		msg := h.AwaitTerminal(t, seq, 10*time.Second)
		if msg.Type != "error" || !strings.Contains(msg.Error, "queue full") {
			t.Fatalf("seq %d: want a queue-full error, got %s (%s%s)", seq, msg.Type, msg.Error, msg.Reason)
		}
	}

	// And the overflowed jobs were acked first, so the hold saw them go
	// assigned -> processing -> failed for a condition that never involved the
	// job at all.
	for _, seq := range seqs[queueSize:] {
		if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool {
			return m.Seq == seq && m.Type == "ack"
		}, time.Second); err != nil {
			t.Errorf("seq %d was rejected without ever being acked", seq)
		}
	}
	t.Log("queue overflow is reported as a retryable per-job failure, not as backpressure")
}

// TestZeroQueueSizeRejectsEveryJob is the configuration corner of the same
// path. scanner.queue_size = 0 passes validation, and NewJobQueue(0) then
// refuses every Enqueue, so a scanner that looks healthy (connected, health
// endpoint green, workers idle) fails 100% of jobs with "scanner queue full"
// and the hold retries all of them forever.
func TestZeroQueueSizeRejectsEveryJob(t *testing.T) {
	h := Start(t, mockhold.NewMemory(), func(c *config.Config) { c.Scanner.QueueSize = 0 })

	seq, err := h.Hold.SendJob(helmJob(t))
	if err != nil {
		t.Fatalf("send job: %v", err)
	}
	msg := h.AwaitTerminal(t, seq, 10*time.Second)
	if msg.Type != "error" || !strings.Contains(msg.Error, "queue full") {
		t.Fatalf("queue_size=0: want a queue-full error, got %s (%s%s)", msg.Type, msg.Error, msg.Reason)
	}
}

// --- 4. worker configuration ------------------------------------------------

// TestZeroWorkersAcksAndStrands: scanner.workers = 0 passes validation and
// starts a pool with no workers at all. The client still acks everything it
// receives, so the hold moves each job to 'processing' and then waits out the
// ten-minute processing timeout in reDispatchTimedOut before failing it. The
// scanner logs "Scanner worker pool started workers=0" once at boot and
// nothing else; there is no health signal that distinguishes this from idle.
func TestZeroWorkersAcksAndStrands(t *testing.T) {
	h := Start(t, mockhold.NewMemory(), WithWorkers(0))

	seq, err := h.Hold.SendJob(helmJob(t))
	if err != nil {
		t.Fatalf("send job: %v", err)
	}

	if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool {
		return m.Seq == seq && m.Type == "ack"
	}, 10*time.Second); err != nil {
		t.Fatalf("job was not even acked: %v", err)
	}

	expectSilence(t, h, func(m mockhold.Message) bool {
		return m.Seq == seq && (m.Type == "result" || m.Type == "error" || m.Type == "skipped")
	}, 2*time.Second, "workers=0")

	if n := h.Queue.Len(); n != 1 {
		t.Errorf("queue holds %d jobs, want 1 (acked and stranded)", n)
	}
}

// TestEmptyTmpDirFailsEveryJob: vuln.tmp_dir = "" is accepted by config
// loading, skips the TMPDIR export in WorkerPool.Start, and then fails every
// single job in processJob's ensureDir, because os.MkdirAll("") is an error.
// The failure is retryable, so every job in the deployment loops forever.
func TestEmptyTmpDirFailsEveryJob(t *testing.T) {
	h := Start(t, mockhold.NewMemory(), func(c *config.Config) { c.Vuln.TmpDir = "" })

	seq, err := h.Hold.SendJob(synthJob("probe/no-tmpdir"))
	if err != nil {
		t.Fatalf("send job: %v", err)
	}
	msg := h.AwaitTerminal(t, seq, 30*time.Second)
	if msg.Type != "error" || !strings.Contains(msg.Error, "tmp dir") {
		t.Fatalf("empty tmp_dir: want a tmp dir error, got %s (%s%s)", msg.Type, msg.Error, msg.Reason)
	}
	if n := len(h.Hold.BlobRequests()); n != 0 {
		t.Errorf("failed before download but still fetched %d blobs", n)
	}
	t.Logf("every job fails with %q, retryably", msg.Error)
}

// --- 5. priority ------------------------------------------------------------

// gate is an HTTP stand-in for a hold whose getBlob hangs. A job pointed at it
// occupies a worker for exactly as long as the test wants, which is how the
// backlog scenarios below build a queue without needing real image bytes.
type gate struct {
	srv     *httptest.Server
	entered chan struct{}
	release chan struct{}
}

func newGate(t *testing.T) *gate {
	t.Helper()
	g := &gate{entered: make(chan struct{}, 8), release: make(chan struct{})}
	g.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case g.entered <- struct{}{}:
		default:
		}
		<-g.release
		http.Error(w, "gate released", http.StatusNotFound)
	}))
	t.Cleanup(g.srv.Close)
	return g
}

func (g *gate) waitEntered(t *testing.T, timeout time.Duration) {
	t.Helper()
	select {
	case <-g.entered:
	case <-time.After(timeout):
		t.Fatal("worker never reached the stalled blob fetch")
	}
}

func (g *gate) open() { close(g.release) }

// TestHighTierJumpsQueuedBacklog confirms the priority heap does what it
// claims across the real client/queue/worker path: while one worker is busy,
// a later owner-tier job overtakes an earlier deckhand-tier one.
//
// It also shows the limit of that guarantee. Priority is consulted only at
// Dequeue, so a high-tier job that arrives while the single worker is inside
// a scan waits for that scan to finish plus the full JobCooldown. With the
// production 10s cooldown and multi-minute scans, "priority" means position in
// a queue, not preemption, and a saturated scanner starves the low tier
// entirely: every owner job admitted during a scan is dequeued before any
// deckhand job, no matter how long the deckhand job has waited.
func TestHighTierJumpsQueuedBacklog(t *testing.T) {
	h := Start(t, mockhold.NewMemory())
	g := newGate(t)

	// Occupy the single worker with a job whose blob fetch never returns.
	blocker := synthJob("probe/blocker")
	blocker.HoldEndpoint = g.srv.URL
	blockerSeq, err := h.Hold.SendJob(blocker)
	if err != nil {
		t.Fatalf("send blocker: %v", err)
	}
	g.waitEntered(t, 15*time.Second)

	// Queue a deckhand job first, then an owner job.
	low := helmJob(t)
	low.Tier = "deckhand"
	lowSeq, err := h.Hold.SendJob(low)
	if err != nil {
		t.Fatalf("send low: %v", err)
	}
	high := helmJob(t)
	high.Tier = "owner"
	highSeq, err := h.Hold.SendJob(high)
	if err != nil {
		t.Fatalf("send high: %v", err)
	}

	// Both must be in the queue before the worker is freed, or the test would
	// be measuring arrival order rather than priority.
	for _, seq := range []int64{lowSeq, highSeq} {
		if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool {
			return m.Seq == seq && m.Type == "ack"
		}, 10*time.Second); err != nil {
			t.Fatalf("seq %d never acked: %v", seq, err)
		}
	}
	if n := h.Queue.Len(); n != 2 {
		t.Fatalf("queue depth %d, want 2 backlogged jobs", n)
	}

	g.open()
	h.AwaitTerminal(t, blockerSeq, 30*time.Second)

	highMsg := h.AwaitTerminal(t, highSeq, 30*time.Second)
	lowMsg := h.AwaitTerminal(t, lowSeq, 30*time.Second)
	if !highMsg.At.Before(lowMsg.At) {
		t.Errorf("owner-tier job finished at %s, deckhand at %s: priority did not apply",
			highMsg.At, lowMsg.At)
	}
}

// --- 6. shutdown ------------------------------------------------------------

// TestQueueCloseDrainsRatherThanCancels documents queue.Close semantics, which
// are not what "close" suggests. Dequeue returns nil only when the queue is
// closed *and* empty, so a shutdown with a backlog hands every remaining job
// to a worker rather than dropping it. Combined with HoldClient.Close having
// already severed the socket, whatever those jobs produce is written into a
// dead connection and lost, while the hold sits on them until the ten-minute
// processing timeout.
func TestQueueCloseDrainsRatherThanCancels(t *testing.T) {
	q := queue.NewJobQueue(10)
	for i := 0; i < 3; i++ {
		if !q.Enqueue(&scanner.ScanJob{Seq: int64(i + 1), Tier: "deckhand"}) {
			t.Fatalf("enqueue %d refused", i)
		}
	}

	q.Close()

	var drained []int64
	for {
		job := q.Dequeue()
		if job == nil {
			break
		}
		drained = append(drained, job.Seq)
	}
	if len(drained) != 3 {
		t.Fatalf("Close() discarded the backlog: drained %v, want 3 jobs", drained)
	}
	t.Logf("Close() left %d jobs to be dequeued and scanned after shutdown began", len(drained))
}

// Shutdown and in-flight downloads used to be pinned here, by
// TestShutdownDoesNotInterruptInFlightDownload: the blob fetches built their
// requests with http.NewRequest and no context, so cancelling the pool context
// changed nothing and WorkerPool.Wait blocked until the HTTP client's own
// five-minute timeout expired. That is fixed, and the opposite is now asserted
// by TestShutdownAbortsAnInFlightDownload in deadline_test.go, which also
// checks the server sees the request hang up rather than the worker merely
// walking away from it.

// TestHoldClientCloseIsNotIdempotent pins a sharp edge rather than a live bug:
// HoldClient.Close closes c.done unconditionally, so a second call panics the
// process with "close of closed channel". cmd/scanner calls it exactly once
// today, which is the only reason this is not already an incident, and there
// is no guard if a future shutdown path (a health-check restart, a reconnect
// supervisor) calls it again.
func TestHoldClientCloseIsNotIdempotent(t *testing.T) {
	c := client.NewHoldClient("ws://127.0.0.1:1", "secret", queue.NewJobQueue(1))
	c.Close()

	defer func() {
		if recover() == nil {
			t.Fatal("Close is idempotent now; this finding is fixed and the test should be inverted")
		}
	}()
	c.Close()
	t.Fatal("unreachable")
}