at-container-registry/scanner/internal/scan/extractor.go

package scan

import (
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"

	scanner "atcr.io/scanner"
	"atcr.io/scanner/internal/client"
)

// OCI image layout types for constructing the layout on disk.
type ociDescriptor struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`
	Size      int64  `json:"size"`
}

type ociManifest struct {
	SchemaVersion int             `json:"schemaVersion"`
	MediaType     string          `json:"mediaType,omitempty"`
	Config        ociDescriptor   `json:"config"`
	Layers        []ociDescriptor `json:"layers"`
}

type ociIndex struct {
	SchemaVersion int             `json:"schemaVersion"`
	Manifests     []ociDescriptor `json:"manifests"`
}

// buildOCILayout downloads image blobs and constructs an OCI image layout directory.
// Instead of extracting layers to a rootfs (which requires decompression and causes
// permission/security issues), this writes compressed blobs directly and lets Syft's
// stereoscope handle layer processing internally.
//
// Layout structure:
//
//	scan-*/
//	├── oci-layout
//	├── index.json
//	└── blobs/sha256/
//	    ├── <manifest-hex>
//	    ├── <config-hex>
//	    └── <layer-hex>...
//
// Every blob is verified against its descriptor as it streams, and every
// digest is validated before it becomes a path, so the layout describes bytes
// that are on disk and hash to the names they are filed under.
//
// maxBytes is the ceiling on the total transferred for this job; zero or less
// means unlimited. It is spent down blob by blob, so it bounds the real bytes
// on disk rather than the numbers the manifest record claims.
func buildOCILayout(job *scanner.ScanJob, tmpDir, secret string, maxBytes int64) (string, func(), error) {
	scanDir, err := os.MkdirTemp(tmpDir, "scan-*")
	if err != nil {
		return "", nil, fmt.Errorf("failed to create temp directory: %w", err)
	}

	cleanup := func() {
		if err := os.RemoveAll(scanDir); err != nil {
			slog.Warn("Failed to clean up temp directory", "dir", scanDir, "error", err)
		}
	}

	blobsDir := filepath.Join(scanDir, "blobs", "sha256")
	if err := os.MkdirAll(blobsDir, 0755); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to create blobs directory: %w", err)
	}

	if job.Config.Digest == "" {
		cleanup()
		return "", nil, fmt.Errorf("config blob has empty digest, cannot download")
	}

	// Download every referenced blob and describe it in the same pass, so the
	// layout can only ever list a blob that arrived and was verified, at the
	// size it actually arrived in. Verification has already established that a
	// declared size, where the record gave one, matches; recording the
	// measured figure keeps the layout honest for the descriptors that
	// declared none.
	remaining := int64(-1) // -1 is unbounded
	if maxBytes > 0 {
		remaining = maxBytes
	}
	manifest := ociManifest{
		SchemaVersion: 2,
		MediaType:     "application/vnd.oci.image.manifest.v1+json",
		Layers:        make([]ociDescriptor, 0, len(job.Layers)),
	}
	for _, ref := range referencedBlobs(job) {
		digest, err := scanner.ParseDigest(ref.Descriptor.Digest)
		if err != nil {
			cleanup()
			return "", nil, &SkipError{Reason: fmt.Sprintf("%s: %v", ref.what(), err)}
		}
		slog.Info("Downloading blob", "blob", ref.what(), "digest", digest,
			"declaredSize", ref.Descriptor.Size, "mediaType", ref.Descriptor.MediaType)

		n, err := downloadBlob(job, digest, ref.Descriptor.Size, remaining, blobsDir, secret)
		if err != nil {
			cleanup()
			return "", nil, blobFailure(ref.what(), err)
		}
		if remaining >= 0 {
			remaining -= n
		}

		d := ociDescriptor{
			MediaType: defaultMediaType(ref.Descriptor.MediaType, ref.defaultMediaType()),
			Digest:    digest.String(),
			Size:      n,
		}
		if ref.isConfig() {
			manifest.Config = d
			continue
		}
		manifest.Layers = append(manifest.Layers, d)
	}

	// Write manifest blob
	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to marshal manifest: %w", err)
	}
	manifestHash := sha256.Sum256(manifestJSON)
	manifestDigest := fmt.Sprintf("sha256:%x", manifestHash)
	manifestPath := filepath.Join(blobsDir, fmt.Sprintf("%x", manifestHash))
	if err := os.WriteFile(manifestPath, manifestJSON, 0644); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to write manifest blob: %w", err)
	}

	// Write index.json
	index := ociIndex{
		SchemaVersion: 2,
		Manifests: []ociDescriptor{
			{
				MediaType: "application/vnd.oci.image.manifest.v1+json",
				Digest:    manifestDigest,
				Size:      int64(len(manifestJSON)),
			},
		},
	}
	indexJSON, err := json.Marshal(index)
	if err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to marshal index: %w", err)
	}
	if err := os.WriteFile(filepath.Join(scanDir, "index.json"), indexJSON, 0644); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to write index.json: %w", err)
	}

	// Write oci-layout file
	ociLayout := []byte(`{"imageLayoutVersion":"1.0.0"}`)
	if err := os.WriteFile(filepath.Join(scanDir, "oci-layout"), ociLayout, 0644); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("failed to write oci-layout: %w", err)
	}

	slog.Info("OCI layout built",
		"dir", scanDir,
		"layers", len(manifest.Layers),
		"manifestDigest", manifestDigest)

	return scanDir, cleanup, nil
}

// blobRef is one blob the layout will contain: the descriptor that named it,
// plus where it sits in the record so an error can say which blob it means.
type blobRef struct {
	Index      int // position in job.Layers; configIndex for the config blob
	Descriptor scanner.BlobDescriptor
}

const configIndex = -1

func (r blobRef) isConfig() bool { return r.Index == configIndex }

func (r blobRef) what() string {
	if r.isConfig() {
		return "config blob"
	}
	return fmt.Sprintf("layer %d", r.Index)
}

func (r blobRef) defaultMediaType() string {
	if r.isConfig() {
		return "application/vnd.oci.image.config.v1+json"
	}
	return "application/vnd.oci.image.layer.v1.tar+gzip"
}

// referencedBlobs returns every blob a scan of this job touches, in the order
// it touches them: the config first, then the layers that are not dropped.
//
// This is the single definition of "which blobs does this job reference".
// Digest validation, the download loop, the byte budget and the layout
// manifest all walk this list, so a layer dropped here is dropped everywhere
// and a layer kept here is one that has been checked.
func referencedBlobs(job *scanner.ScanJob) []blobRef {
	refs := make([]blobRef, 0, len(job.Layers)+1)
	if job.Config.Digest != "" {
		refs = append(refs, blobRef{Index: configIndex, Descriptor: job.Config})
	}
	for i, layer := range job.Layers {
		if layer.Digest == "" {
			continue
		}
		// Non-tar layers (cosign signatures, in-toto attestations) are not
		// something Syft can read, so they are never fetched or listed.
		if layer.MediaType != "" && !strings.Contains(layer.MediaType, "tar") {
			continue
		}
		refs = append(refs, blobRef{Index: i, Descriptor: layer})
	}
	return refs
}

// blobFailure labels a download error, converting the ones that can never
// succeed on a retry into a SkipError.
//
// The hold's stale-scan loop re-offers "error" records on every pass and never
// re-offers "skipped" ones, so a failure decided by the record itself or by
// bytes already stored belongs on the skip side. A transport fault (a 5xx, a
// dropped connection, an expired presigned URL) stays an error, because those
// really can succeed next time.
func blobFailure(what string, err error) error {
	if errors.Is(err, client.ErrBlobCorrupt) || errors.Is(err, client.ErrBlobTooLarge) {
		return &SkipError{Reason: fmt.Sprintf("%s: %v", what, err)}
	}
	return fmt.Errorf("failed to download %s: %w", what, err)
}

// downloadBlob fetches one validated blob into the blobs directory and returns
// how many bytes arrived. maxBytes is the remaining job budget, negative for
// unbounded.
func downloadBlob(job *scanner.ScanJob, digest scanner.Digest, declaredSize, maxBytes int64, blobsDir, secret string) (int64, error) {
	destPath := filepath.Join(blobsDir, digest.Hex)

	// The invariant, asserted rather than assumed. ParseDigest has already
	// constrained Hex to [0-9a-f], so this cannot fire; it is here so that any
	// future path that reaches os.Create with something less constrained fails
	// loudly instead of writing outside the scan directory.
	if filepath.Dir(filepath.Clean(destPath)) != filepath.Clean(blobsDir) {
		return 0, fmt.Errorf("refusing to write blob %s outside %s", digest, blobsDir)
	}

	presignedURL, err := client.GetBlobPresignedURL(job.HoldEndpoint, job.HoldDID, digest, secret)
	if err != nil {
		return 0, fmt.Errorf("failed to get presigned URL for %s: %w", digest, err)
	}
	return client.DownloadBlob(presignedURL, destPath, client.BlobExpectation{
		Digest:       digest,
		DeclaredSize: declaredSize,
		MaxBytes:     maxBytes,
	})
}

func defaultMediaType(mediaType, fallback string) string {
	if mediaType == "" {
		return fallback
	}
	return mediaType
}