improvements to how scanning works, and helmchart ui

cmd/hold/scan_backfill.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"atcr.io/pkg/atproto"
+	"atcr.io/pkg/hold"
+
+	"github.com/spf13/cobra"
+)
+
+// Media-type fragments that identify artifact types the scanner intentionally
+// skips. Keep this list in sync with scanner/internal/scan/worker.go's
+// unscannableConfigTypes — that map keys on config media types; here we look
+// at *layer* media types because the backfill walks the hold's layer index
+// (which has manifest AT-URIs we can join against scan records).
+//
+// Detection by layer media type is reliable: helm charts always have a single
+// layer with media type application/vnd.cncf.helm.chart.content.v1.tar+gzip;
+// in-toto / DSSE attestations use distinct layer types too.
+var unscannableLayerMediaSubstrings = []string{
+	"helm.chart.content",
+	"in-toto",
+	"dsse.envelope",
+}
+
+var scanBackfillConfigFile string
+
+var scanBackfillCmd = &cobra.Command{
+	Use:   "scan-backfill",
+	Short: "Rewrite legacy scan records to use the status field",
+	Long: `Walks every io.atcr.hold.scan record on this hold and assigns a status
+("skipped" or "failed") to records that pre-date the status field.
+
+A legacy record is one with an empty status, no SBOM blob, and zero vulnerability
+counts. The tool inspects each record's manifest's layers to decide:
+
+  - layer media type matches helm/in-toto/DSSE → status="skipped"
+  - everything else                            → status="failed"
+
+The tool is idempotent: records that already have a status are left alone.
+Run once per hold after upgrading.`,
+	Args: cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfg, err := hold.LoadConfig(scanBackfillConfigFile)
+		if err != nil {
+			return fmt.Errorf("load config: %w", err)
+		}
+
+		ctx := context.Background()
+		holdPDS, cleanup, err := openHoldPDS(ctx, cfg)
+		if err != nil {
+			return err
+		}
+		defer cleanup()
+
+		ri := holdPDS.RecordsIndex()
+		if ri == nil {
+			return fmt.Errorf("records index not available")
+		}
+
+		const batchSize = 200
+		var (
+			cursor      string
+			scanned     int
+			rewritten   int
+			markSkipped int
+			markFailed  int
+			alreadyOK   int
+		)
+
+		for {
+			records, nextCursor, err := ri.ListRecords(atproto.ScanCollection, batchSize, cursor, true)
+			if err != nil {
+				return fmt.Errorf("list scan records: %w", err)
+			}
+
+			for _, rec := range records {
+				scanned++
+				manifestDigest := "sha256:" + rec.Rkey
+
+				_, scanRecord, err := holdPDS.GetScanRecord(ctx, manifestDigest)
+				if err != nil {
+					fmt.Fprintf(cmd.ErrOrStderr(), "  skip rkey=%s: get failed: %v\n", rec.Rkey, err)
+					continue
+				}
+
+				// Already classified — nothing to do.
+				if scanRecord.Status != "" {
+					alreadyOK++
+					continue
+				}
+
+				// Only legacy records that signal failure (nil blob + zero
+				// counts) are candidates. Records with real data don't need
+				// rewriting; their absent status will be treated as "ok".
+				if scanRecord.SbomBlob != nil || scanRecord.Total != 0 {
+					alreadyOK++
+					continue
+				}
+
+				// Determine artifact type from layer media types.
+				layers, err := holdPDS.ListLayerRecordsForManifest(ctx, scanRecord.Manifest)
+				if err != nil {
+					fmt.Fprintf(cmd.ErrOrStderr(), "  skip rkey=%s: list layers failed: %v\n", rec.Rkey, err)
+					continue
+				}
+
+				skipped := false
+				for _, l := range layers {
+					for _, frag := range unscannableLayerMediaSubstrings {
+						if strings.Contains(l.MediaType, frag) {
+							skipped = true
+							break
+						}
+					}
+					if skipped {
+						break
+					}
+				}
+
+				var rewrite *atproto.ScanRecord
+				if skipped {
+					rewrite = atproto.NewSkippedScanRecord(
+						manifestDigest,
+						scanRecord.Repository,
+						scanRecord.UserDID,
+						"backfilled: unscannable artifact type",
+						scanRecord.ScannerVersion,
+					)
+					markSkipped++
+				} else {
+					rewrite = atproto.NewFailedScanRecord(
+						manifestDigest,
+						scanRecord.Repository,
+						scanRecord.UserDID,
+						"backfilled: legacy record (no SBOM and zero counts)",
+						scanRecord.ScannerVersion,
+					)
+					markFailed++
+				}
+				// Preserve the original ScannedAt — rewriting it would either
+				// reset the rescan timer or invalidate audit signals.
+				if scanRecord.ScannedAt != "" {
+					rewrite.ScannedAt = scanRecord.ScannedAt
+				}
+
+				if _, _, err := holdPDS.CreateScanRecord(ctx, rewrite); err != nil {
+					fmt.Fprintf(cmd.ErrOrStderr(), "  rewrite rkey=%s failed: %v\n", rec.Rkey, err)
+					continue
+				}
+				rewritten++
+			}
+
+			if nextCursor == "" || len(records) == 0 {
+				break
+			}
+			cursor = nextCursor
+		}
+
+		fmt.Fprintf(cmd.OutOrStdout(), "Backfill complete:\n")
+		fmt.Fprintf(cmd.OutOrStdout(), "  scanned:        %d\n", scanned)
+		fmt.Fprintf(cmd.OutOrStdout(), "  already-tagged: %d\n", alreadyOK)
+		fmt.Fprintf(cmd.OutOrStdout(), "  → skipped:      %d\n", markSkipped)
+		fmt.Fprintf(cmd.OutOrStdout(), "  → failed:       %d\n", markFailed)
+		fmt.Fprintf(cmd.OutOrStdout(), "  rewritten:      %d\n", rewritten)
+		return nil
+	},
+}
+
+func init() {
+	scanBackfillCmd.Flags().StringVarP(&scanBackfillConfigFile, "config", "c", "", "path to YAML configuration file")
+	rootCmd.AddCommand(scanBackfillCmd)
+}