at-container-registry/scanner/internal/scan/syft_test.go

package scan

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"testing"
	"time"
)

// The SBOM the scanner publishes must be a function of the image it describes
// and nothing else. It is content-addressed on the hold: the digest of these
// bytes becomes sbomDigest on the io.atcr.hold.scan record, and a digest that
// moves means every rescan of unchanged content uploads a fresh blob and
// orphans the one before it. The hold's stale-scan loop rescans on a schedule,
// so anything scan-local that leaks into these bytes accumulates storage
// forever.
//
// The scenario below is exactly what the stale loop does: the same image,
// scanned twice, from two different scratch directories. buildOCILayout makes
// a fresh os.MkdirTemp "scan-*" per job, so the two paths are never the same
// in production either.

// testLayoutFiles builds a minimal but real OCI image layout as an in-memory
// path -> bytes map, so the same bytes can be written into two different
// directories. No fixture download, no network: this runs in the default
// suite.
func testLayoutFiles(t *testing.T) map[string][]byte {
	t.Helper()

	// One gzipped tar layer. Everything is pinned (fixed modtimes, USTAR
	// headers, gzip with no embedded timestamp) so the bytes are the same on
	// every run, not merely the same between the two copies.
	var raw bytes.Buffer
	tw := tar.NewWriter(&raw)
	for _, f := range []struct{ name, body string }{
		{"etc/os-release", "NAME=\"Alpine Linux\"\nID=alpine\nVERSION_ID=3.19.0\nPRETTY_NAME=\"Alpine Linux v3.19\"\n"},
		{"bin/hello", "#!/bin/sh\necho hello\n"},
	} {
		hdr := &tar.Header{
			Typeflag: tar.TypeReg,
			Name:     f.name,
			Mode:     0o644,
			Size:     int64(len(f.body)),
			ModTime:  time.Unix(1000000000, 0).UTC(),
			Format:   tar.FormatUSTAR,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			t.Fatalf("tar header %s: %v", f.name, err)
		}
		if _, err := tw.Write([]byte(f.body)); err != nil {
			t.Fatalf("tar body %s: %v", f.name, err)
		}
	}
	if err := tw.Close(); err != nil {
		t.Fatalf("close tar: %v", err)
	}
	uncompressed := raw.Bytes()
	diffID := fmt.Sprintf("sha256:%x", sha256.Sum256(uncompressed))

	var gz bytes.Buffer
	zw := gzip.NewWriter(&gz)
	if _, err := zw.Write(uncompressed); err != nil {
		t.Fatalf("gzip write: %v", err)
	}
	if err := zw.Close(); err != nil {
		t.Fatalf("gzip close: %v", err)
	}
	layer := gz.Bytes()
	layerDigest := fmt.Sprintf("sha256:%x", sha256.Sum256(layer))

	config, err := json.Marshal(map[string]any{
		"architecture": "amd64",
		"os":           "linux",
		"config":       map[string]any{"Cmd": []string{"/bin/hello"}},
		"rootfs":       map[string]any{"type": "layers", "diff_ids": []string{diffID}},
	})
	if err != nil {
		t.Fatalf("marshal config: %v", err)
	}
	configDigest := fmt.Sprintf("sha256:%x", sha256.Sum256(config))

	manifest, err := json.Marshal(map[string]any{
		"schemaVersion": 2,
		"mediaType":     "application/vnd.oci.image.manifest.v1+json",
		"config": map[string]any{
			"mediaType": "application/vnd.oci.image.config.v1+json",
			"digest":    configDigest,
			"size":      len(config),
		},
		"layers": []any{map[string]any{
			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
			"digest":    layerDigest,
			"size":      len(layer),
		}},
	})
	if err != nil {
		t.Fatalf("marshal manifest: %v", err)
	}
	manifestDigest := fmt.Sprintf("sha256:%x", sha256.Sum256(manifest))

	index, err := json.Marshal(map[string]any{
		"schemaVersion": 2,
		"manifests": []any{map[string]any{
			"mediaType": "application/vnd.oci.image.manifest.v1+json",
			"digest":    manifestDigest,
			"size":      len(manifest),
		}},
	})
	if err != nil {
		t.Fatalf("marshal index: %v", err)
	}

	hex := func(d string) string { return d[len("sha256:"):] }
	return map[string][]byte{
		"oci-layout":                          []byte(`{"imageLayoutVersion":"1.0.0"}`),
		"index.json":                          index,
		"blobs/sha256/" + hex(manifestDigest): manifest,
		"blobs/sha256/" + hex(configDigest):   config,
		"blobs/sha256/" + hex(layerDigest):    layer,
	}
}

// writeTestLayout materialises the layout under a fresh scratch directory,
// standing in for one buildOCILayout call.
func writeTestLayout(t *testing.T, files map[string][]byte) string {
	t.Helper()
	dir := t.TempDir()
	for name, body := range files {
		path := filepath.Join(dir, filepath.FromSlash(name))
		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
			t.Fatalf("mkdir %s: %v", path, err)
		}
		if err := os.WriteFile(path, body, 0o644); err != nil {
			t.Fatalf("write %s: %v", path, err)
		}
	}
	return dir
}

// scanLocalResidue is the set of top-level SPDX keys that still move between
// two encodings of the same image, and why. Both come from Syft's SPDX
// encoder and neither is configurable:
//
//   - creationInfo carries "created", set from time.Now() in
//     syft/format/common/spdxhelpers.ToFormatModel;
//   - documentNamespace appends a fresh uuid.NewRandom() in
//     syft/format/internal/spdxutil/helpers.DocumentNamespace.
//
// So sbomDigest is still not stable across a rescan, even though nothing
// scan-local reaches the document any more. That is a separate defect with a
// separate cause, and this set is what keeps the two apart: if a third key
// starts moving, the assertions below fail rather than shrugging.
var scanLocalResidue = map[string]string{
	"creationInfo":      "creationInfo.created is time.Now() inside Syft's SPDX encoder",
	"documentNamespace": "documentNamespace embeds a random UUID inside Syft's SPDX encoder",
}

// alwaysVaries is the residue key that is a sound signal of the underlying
// defect. If these two keys ever stop moving, sbomDigest can become stable and
// this test should be tightened to a plain digest comparison — but only
// documentNamespace can say so, because its UUID is redrawn on every encode.
//
// creationInfo cannot: "created" has one-second granularity, so two scans that
// land inside the same second produce an identical value. Asserting that it
// varies makes the test a coin flip on how fast the machine is, which is
// exactly how it failed the first time the module was wired into `make test`
// (passing under -race, which is slow enough to straddle a second, and failing
// under -cover, which is not).
const alwaysVaries = "documentNamespace"

// TestSBOMIsStableAcrossScanDirectories is the requirement, as far as this
// package can carry it: the same image scanned twice from two different
// scratch directories must produce the same SBOM, apart from the two encoder
// fields named in scanLocalResidue.
//
// The scenario is what the hold's stale-scan loop does. buildOCILayout makes a
// fresh os.MkdirTemp("scan-*") per job, so the two paths are never equal in
// production either.
func TestSBOMIsStableAcrossScanDirectories(t *testing.T) {
	files := testLayoutFiles(t)
	dirA := writeTestLayout(t, files)
	dirB := writeTestLayout(t, files)

	ctx := context.Background()

	_, jsonA, _, err := generateSBOM(ctx, dirA, testManifestDigest)
	if err != nil {
		t.Fatalf("first scan: %v", err)
	}
	_, jsonB, _, err := generateSBOM(ctx, dirB, testManifestDigest)
	if err != nil {
		t.Fatalf("rescan: %v", err)
	}

	// 1. Nothing outside the known residue may move. This is the assertion
	//    that would have caught the original defect: documentName,
	//    documentNamespace, packages and relationships all carried the scratch
	//    path, and four of those five are now stable.
	for _, key := range topLevelDiff(t, jsonA, jsonB) {
		if _, known := scanLocalResidue[key]; !known {
			t.Errorf("SBOM key %q differs between two scans of identical content:\n    %s\n    %s",
				key, firstBytes(fieldOf(t, jsonA, key)), firstBytes(fieldOf(t, jsonB, key)))
		}
	}

	// 2. With the residue removed the two documents must be byte-identical, so
	//    this is a real digest comparison and not a field-by-field spot check.
	if a, b := digestWithout(t, jsonA, scanLocalResidue), digestWithout(t, jsonB, scanLocalResidue); a != b {
		t.Errorf("SBOMs differ beyond %v:\n first: %s\nsecond: %s", keysOf(scanLocalResidue), a, b)
	}

	// 3. The scratch path must appear nowhere at all. The diff above only sees
	//    keys that moved; a path baked identically into both documents would
	//    pass it and still be wrong.
	for _, dir := range []string{dirA, dirB} {
		for name, doc := range map[string][]byte{"first": jsonA, "second": jsonB} {
			if bytes.Contains(doc, []byte(dir)) {
				t.Errorf("%s SBOM still contains the scan directory %s", name, dir)
			}
		}
	}

	// 4. And it must be the manifest digest that took its place, which is what
	//    makes a stored SBOM say which image it describes.
	if got := fieldOf(t, jsonA, "name"); got != strconv.Quote(testManifestDigest) {
		t.Errorf("SBOM documentName is %s, want the manifest digest %q", got, testManifestDigest)
	}

	// 5. If the residue is ever fixed upstream or worked around here, this test
	//    should become a plain digest equality check. Only documentNamespace is
	//    asked, for the reason on alwaysVaries: a matching creationInfo means
	//    the clock did not tick, not that the encoder became deterministic.
	if fieldOf(t, jsonA, alwaysVaries) == fieldOf(t, jsonB, alwaysVaries) {
		t.Errorf("%q no longer varies between scans (%s); sbomDigest may now be stable, "+
			"so drop scanLocalResidue and assert digest equality directly",
			alwaysVaries, scanLocalResidue[alwaysVaries])
	}
}

// testManifestDigest stands in for job.ManifestDigest: the identity the hold
// keys the io.atcr.hold.scan record by, and now the SBOM's source reference.
const testManifestDigest = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"

// TestSBOMSourceReferenceIsTheManifestDigest pins the one field grype.go
// copies wholesale into the vulnerability report ("source": s.Source). The
// report has no timestamp and no UUID, so unlike the SBOM its digest is fully
// determined by this value plus the matches — which is what lets
// TestVulnDBScanIsDeterministic, in internal/e2e, assert vulnDigest equality
// outright.
func TestSBOMSourceReferenceIsTheManifestDigest(t *testing.T) {
	dir := writeTestLayout(t, testLayoutFiles(t))

	result, _, _, err := generateSBOM(context.Background(), dir, testManifestDigest)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	desc := result.Source
	if desc.Name != testManifestDigest {
		t.Errorf("source name is %q, want the manifest digest %q", desc.Name, testManifestDigest)
	}
	if desc.Version != testManifestDigest {
		t.Errorf("source version is %q, want the manifest digest %q", desc.Version, testManifestDigest)
	}

	// The report embeds the whole description, metadata included, so the
	// scratch path must not survive anywhere inside it.
	encoded, err := json.Marshal(desc)
	if err != nil {
		t.Fatalf("marshal source description: %v", err)
	}
	if bytes.Contains(encoded, []byte(dir)) {
		t.Errorf("the source description still carries the scan directory %s: %s", dir, encoded)
	}
}

// digestWithout hashes the document with the named top-level keys removed.
func digestWithout(t *testing.T, doc []byte, drop map[string]string) string {
	t.Helper()
	var m map[string]json.RawMessage
	if err := json.Unmarshal(doc, &m); err != nil {
		t.Fatalf("parse SBOM: %v", err)
	}
	for k := range drop {
		delete(m, k)
	}
	// json.Marshal of a map sorts keys, so this is stable.
	stripped, err := json.Marshal(m)
	if err != nil {
		t.Fatalf("re-encode SBOM: %v", err)
	}
	return fmt.Sprintf("sha256:%x", sha256.Sum256(stripped))
}

func keysOf(m map[string]string) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// topLevelDiff names the top-level JSON keys whose encoded values differ.
func topLevelDiff(t *testing.T, a, b []byte) []string {
	t.Helper()
	var ka, kb map[string]json.RawMessage
	if err := json.Unmarshal(a, &ka); err != nil {
		t.Fatalf("parse first SBOM: %v", err)
	}
	if err := json.Unmarshal(b, &kb); err != nil {
		t.Fatalf("parse second SBOM: %v", err)
	}
	var moved []string
	for k := range ka {
		if string(ka[k]) != string(kb[k]) {
			moved = append(moved, k)
		}
	}
	for k := range kb {
		if _, ok := ka[k]; !ok {
			moved = append(moved, k)
		}
	}
	sort.Strings(moved)
	return moved
}

func fieldOf(t *testing.T, doc []byte, key string) string {
	t.Helper()
	var m map[string]json.RawMessage
	if err := json.Unmarshal(doc, &m); err != nil {
		t.Fatalf("parse SBOM: %v", err)
	}
	return string(m[key])
}

func firstBytes(s string) string {
	if len(s) > 400 {
		return s[:400] + "..."
	}
	return s
}