mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-01 07:46:55 +00:00
improve admin tooling
This commit is contained in:
Evan Jarrett
+24
-132
@@ -3,45 +3,32 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/hold"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Media-type fragments that identify artifact types the scanner intentionally
|
||||
// skips. Keep this list in sync with scanner/internal/scan/worker.go's
|
||||
// unscannableConfigTypes — that map keys on config media types; here we look
|
||||
// at *layer* media types because the backfill walks the hold's layer index
|
||||
// (which has manifest AT-URIs we can join against scan records).
|
||||
//
|
||||
// Detection by layer media type is reliable: helm charts always have a single
|
||||
// layer with media type application/vnd.cncf.helm.chart.content.v1.tar+gzip;
|
||||
// in-toto / DSSE attestations use distinct layer types too.
|
||||
var unscannableLayerMediaSubstrings = []string{
|
||||
"helm.chart.content",
|
||||
"in-toto",
|
||||
"dsse.envelope",
|
||||
}
|
||||
|
||||
var scanBackfillConfigFile string
|
||||
|
||||
var scanBackfillCmd = &cobra.Command{
|
||||
Use: "scan-backfill",
|
||||
Short: "Rewrite legacy scan records to use the status field",
|
||||
Short: "Rewrite legacy scan records to use the status field (offline)",
|
||||
Long: `Walks every io.atcr.hold.scan record on this hold and assigns a status
|
||||
("skipped" or "failed") to records that pre-date the status field.
|
||||
|
||||
A legacy record is one with an empty status, no SBOM blob, and zero vulnerability
|
||||
counts. The tool inspects each record's manifest's layers to decide:
|
||||
A legacy record is one with an empty status, no SBOM blob, and zero
|
||||
vulnerability counts. Layer media types decide the rewrite:
|
||||
|
||||
- layer media type matches helm/in-toto/DSSE → status="skipped"
|
||||
- everything else → status="failed"
|
||||
- helm.chart.content / in-toto / dsse.envelope → status="skipped"
|
||||
- everything else → status="failed"
|
||||
|
||||
The tool is idempotent: records that already have a status are left alone.
|
||||
Run once per hold after upgrading.`,
|
||||
The tool is idempotent and preserves each record's original scannedAt.
|
||||
|
||||
This subcommand opens the hold's CAR store directly, so the running hold
|
||||
service must be stopped first (otherwise the embedded PDS holds an exclusive
|
||||
lock). For zero-downtime backfill on a production hold, hit the admin
|
||||
endpoint POST /admin/api/scan-backfill instead.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := hold.LoadConfig(scanBackfillConfigFile)
|
||||
@@ -56,116 +43,21 @@ Run once per hold after upgrading.`,
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ri := holdPDS.RecordsIndex()
|
||||
if ri == nil {
|
||||
return fmt.Errorf("records index not available")
|
||||
logf := func(format string, args ...any) {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " "+format+"\n", args...)
|
||||
}
|
||||
res, err := holdPDS.BackfillScanStatus(ctx, logf, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backfill: %w", err)
|
||||
}
|
||||
|
||||
const batchSize = 200
|
||||
var (
|
||||
cursor string
|
||||
scanned int
|
||||
rewritten int
|
||||
markSkipped int
|
||||
markFailed int
|
||||
alreadyOK int
|
||||
)
|
||||
|
||||
for {
|
||||
records, nextCursor, err := ri.ListRecords(atproto.ScanCollection, batchSize, cursor, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list scan records: %w", err)
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
scanned++
|
||||
manifestDigest := "sha256:" + rec.Rkey
|
||||
|
||||
_, scanRecord, err := holdPDS.GetScanRecord(ctx, manifestDigest)
|
||||
if err != nil {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " skip rkey=%s: get failed: %v\n", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Already classified — nothing to do.
|
||||
if scanRecord.Status != "" {
|
||||
alreadyOK++
|
||||
continue
|
||||
}
|
||||
|
||||
// Only legacy records that signal failure (nil blob + zero
|
||||
// counts) are candidates. Records with real data don't need
|
||||
// rewriting; their absent status will be treated as "ok".
|
||||
if scanRecord.SbomBlob != nil || scanRecord.Total != 0 {
|
||||
alreadyOK++
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine artifact type from layer media types.
|
||||
layers, err := holdPDS.ListLayerRecordsForManifest(ctx, scanRecord.Manifest)
|
||||
if err != nil {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " skip rkey=%s: list layers failed: %v\n", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
skipped := false
|
||||
for _, l := range layers {
|
||||
for _, frag := range unscannableLayerMediaSubstrings {
|
||||
if strings.Contains(l.MediaType, frag) {
|
||||
skipped = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skipped {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var rewrite *atproto.ScanRecord
|
||||
if skipped {
|
||||
rewrite = atproto.NewSkippedScanRecord(
|
||||
manifestDigest,
|
||||
scanRecord.Repository,
|
||||
scanRecord.UserDID,
|
||||
"backfilled: unscannable artifact type",
|
||||
scanRecord.ScannerVersion,
|
||||
)
|
||||
markSkipped++
|
||||
} else {
|
||||
rewrite = atproto.NewFailedScanRecord(
|
||||
manifestDigest,
|
||||
scanRecord.Repository,
|
||||
scanRecord.UserDID,
|
||||
"backfilled: legacy record (no SBOM and zero counts)",
|
||||
scanRecord.ScannerVersion,
|
||||
)
|
||||
markFailed++
|
||||
}
|
||||
// Preserve the original ScannedAt — rewriting it would either
|
||||
// reset the rescan timer or invalidate audit signals.
|
||||
if scanRecord.ScannedAt != "" {
|
||||
rewrite.ScannedAt = scanRecord.ScannedAt
|
||||
}
|
||||
|
||||
if _, _, err := holdPDS.CreateScanRecord(ctx, rewrite); err != nil {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " rewrite rkey=%s failed: %v\n", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
rewritten++
|
||||
}
|
||||
|
||||
if nextCursor == "" || len(records) == 0 {
|
||||
break
|
||||
}
|
||||
cursor = nextCursor
|
||||
}
|
||||
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Backfill complete:\n")
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " scanned: %d\n", scanned)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " already-tagged: %d\n", alreadyOK)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " → skipped: %d\n", markSkipped)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " → failed: %d\n", markFailed)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " rewritten: %d\n", rewritten)
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "Backfill complete:\n")
|
||||
fmt.Fprintf(out, " scanned: %d\n", res.Scanned)
|
||||
fmt.Fprintf(out, " already-tagged: %d\n", res.AlreadyTagged)
|
||||
fmt.Fprintf(out, " → skipped: %d\n", res.MarkedSkipped)
|
||||
fmt.Fprintf(out, " → failed: %d\n", res.MarkedFailed)
|
||||
fmt.Fprintf(out, " rewritten: %d\n", res.Rewritten)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"configJson": {
|
||||
"type": "string",
|
||||
"description": "Raw OCI image config JSON blob",
|
||||
"maxLength": 65536
|
||||
"maxLength": 1000000
|
||||
},
|
||||
"createdAt": {
|
||||
"type": "string",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{{ if eq .Error "upgrade_required" }}
|
||||
<div class="alert alert-info text-sm">
|
||||
{{ icon "sparkles" "size-4" }}
|
||||
<span>AI Image Advisor is a paid feature. <a href="/settings/billing" class="link link-primary font-medium">Upgrade your plan</a> to unlock image analysis.</span>
|
||||
<span>AI Image Advisor is a paid feature. <a href="/settings/billing" class="link link-hover font-semibold underline">Upgrade your plan</a> to unlock image analysis.</span>
|
||||
</div>
|
||||
{{ else if .Error }}
|
||||
<div class="alert alert-warning text-sm">
|
||||
|
||||
+163
-163
File diff suppressed because it is too large
Load Diff
@@ -25,8 +25,15 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Generate map-style encoders
|
||||
if err := cbg.WriteMapEncodersToFile("cbor_gen.go", "atproto",
|
||||
// MaxStringLength bumps the package-wide string read limit. cbor-gen v0.3.1
|
||||
// honors per-field `cborgen:"...,maxlen=N"` tags only on the write side; the
|
||||
// unmarshal template hard-codes the package default. ImageConfigRecord's
|
||||
// configJson holds raw OCI image configs that routinely exceed the stock
|
||||
// 8KB default (deep build histories, verbose created_by lines), so we lift
|
||||
// the ceiling for everyone. Per-field write caps still apply.
|
||||
gen := cbg.Gen{MaxStringLength: 1_000_000}
|
||||
|
||||
if err := gen.WriteMapEncodersToFile("cbor_gen.go", "atproto",
|
||||
atproto.CrewRecord{},
|
||||
atproto.CaptainRecord{},
|
||||
atproto.LayerRecord{},
|
||||
|
||||
@@ -932,9 +932,9 @@ func ScanRecordKey(manifestDigest string) string {
|
||||
// Stores the full OCI config JSON so the appview can display layer history including empty layers
|
||||
type ImageConfigRecord struct {
|
||||
Type string `json:"$type" cborgen:"$type"`
|
||||
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the manifest
|
||||
ConfigJSON string `json:"configJson" cborgen:"configJson"` // Raw OCI image config JSON
|
||||
CreatedAt string `json:"createdAt" cborgen:"createdAt"` // RFC3339 timestamp
|
||||
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the manifest
|
||||
ConfigJSON string `json:"configJson" cborgen:"configJson,maxlen=1000000"` // Raw OCI image config JSON. Cap mirrors lexicon maxLength so deep image histories (Bazel, multi-stage builds with verbose created_by lines) fit.
|
||||
CreatedAt string `json:"createdAt" cborgen:"createdAt"` // RFC3339 timestamp
|
||||
}
|
||||
|
||||
// NewImageConfigRecord creates a new image config record
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package atproto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -1385,3 +1386,72 @@ func TestRepoPageRecord_JSONSerialization(t *testing.T) {
|
||||
t.Errorf("Avatar.Ref.Link = %v, want %v", decoded.Avatar.Ref.Link, record.Avatar.Ref.Link)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageConfigRecord_CBORRoundTrip locks in that we can encode and decode
|
||||
// large OCI image configs without hitting cbor-gen's default 8KB string cap.
|
||||
// Real-world images with deep build histories (Bazel, multi-stage Dockerfiles)
|
||||
// routinely produce config blobs that blow past the default; if either side
|
||||
// regresses, the backfill silently drops records with "configJson was too
|
||||
// long" instead of populating the layer-history UI.
|
||||
func TestImageConfigRecord_CBORRoundTrip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
payloadSize int
|
||||
}{
|
||||
// Small payloads exercise the happy path — should always have worked.
|
||||
{"small", 1024},
|
||||
// 16KB is well past the old 8192 cborgen default. Pre-fix this would
|
||||
// have failed at marshal time.
|
||||
{"medium-16kb", 16 * 1024},
|
||||
// 200KB matches the upper end of pathological real configs (deep
|
||||
// Bazel histories with verbose created_by lines).
|
||||
{"large-200kb", 200 * 1024},
|
||||
// 900KB is just under our 1MB cap — the read side previously
|
||||
// hard-coded 8192 even with the per-field write tag, so this also
|
||||
// guards against the unmarshal regression.
|
||||
{"near-cap-900kb", 900 * 1024},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Use a repeating non-trivial pattern so any byte-level corruption
|
||||
// in encode/decode shows up as a mismatch rather than blending
|
||||
// into a sea of identical bytes.
|
||||
const chunk = "history entry: bazel build //pkg:foo # "
|
||||
payload := strings.Repeat(chunk, tc.payloadSize/len(chunk)+1)[:tc.payloadSize]
|
||||
|
||||
record := &ImageConfigRecord{
|
||||
Type: ImageConfigCollection,
|
||||
Manifest: "at://did:plc:test/io.atcr.manifest/abc",
|
||||
ConfigJSON: payload,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := record.MarshalCBOR(&buf); err != nil {
|
||||
t.Fatalf("MarshalCBOR(%d bytes): %v", tc.payloadSize, err)
|
||||
}
|
||||
|
||||
var decoded ImageConfigRecord
|
||||
if err := decoded.UnmarshalCBOR(&buf); err != nil {
|
||||
t.Fatalf("UnmarshalCBOR(%d bytes): %v", tc.payloadSize, err)
|
||||
}
|
||||
|
||||
if decoded.Type != record.Type {
|
||||
t.Errorf("Type = %q, want %q", decoded.Type, record.Type)
|
||||
}
|
||||
if decoded.Manifest != record.Manifest {
|
||||
t.Errorf("Manifest = %q, want %q", decoded.Manifest, record.Manifest)
|
||||
}
|
||||
if decoded.CreatedAt != record.CreatedAt {
|
||||
t.Errorf("CreatedAt = %q, want %q", decoded.CreatedAt, record.CreatedAt)
|
||||
}
|
||||
if len(decoded.ConfigJSON) != len(record.ConfigJSON) {
|
||||
t.Fatalf("ConfigJSON length = %d, want %d", len(decoded.ConfigJSON), len(record.ConfigJSON))
|
||||
}
|
||||
if decoded.ConfigJSON != record.ConfigJSON {
|
||||
t.Errorf("ConfigJSON content mismatch at length %d", len(record.ConfigJSON))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,21 @@ type AdminUI struct {
|
||||
// In-memory session storage (single user, no persistence needed)
|
||||
sessions map[string]*AdminSession
|
||||
sessionsMu sync.RWMutex
|
||||
|
||||
// scan-backfill state — runs as a background goroutine on click; the
|
||||
// status endpoint reads this for progress polling. Only one run at a
|
||||
// time (idempotent, so re-running is safe but pointless).
|
||||
scanBackfill scanBackfillState
|
||||
}
|
||||
|
||||
// scanBackfillState tracks the in-flight scan-status backfill run.
|
||||
type scanBackfillState struct {
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
startedAt time.Time
|
||||
current *pds.ScanBackfillResult // running totals (snapshot)
|
||||
result *pds.ScanBackfillResult // final result, set when running=false
|
||||
err string // last error (running ends with err set)
|
||||
}
|
||||
|
||||
// adminContextKey is used to store session data in request context
|
||||
@@ -532,6 +547,12 @@ func (ui *AdminUI) RegisterRoutes(r chi.Router) {
|
||||
r.Get("/admin/api/relay/status", ui.handleRelayStatus)
|
||||
r.Get("/admin/api/crew/member", ui.handleCrewMemberInfo)
|
||||
|
||||
// Scan-record backfill: kicks off a background run and returns a
|
||||
// progress fragment that polls /status. Use Accept:application/json
|
||||
// for a synchronous JSON response (curl-friendly).
|
||||
r.Post("/admin/api/scan-backfill", ui.handleScanBackfill)
|
||||
r.Get("/admin/api/scan-backfill/status", ui.handleScanBackfillStatus)
|
||||
|
||||
// Logout
|
||||
r.Post("/admin/auth/logout", ui.handleLogout)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/hold/pds"
|
||||
)
|
||||
|
||||
// handleScanBackfill kicks off a scan-status backfill in a background
|
||||
// goroutine and returns a progress fragment that polls
|
||||
// /admin/api/scan-backfill/status for updates. Idempotent — clicking again
|
||||
// while a run is in flight just shows the current progress.
|
||||
//
|
||||
// Why background: reverse proxies typically cap upstream HTTP timeouts at
|
||||
// 10–60s, which would cancel a synchronous request mid-loop. Detaching the
|
||||
// work from the request context lets it run to completion.
|
||||
//
|
||||
// JSON callers (Accept: application/json) get a synchronous run instead —
|
||||
// useful for curl + scripting.
|
||||
func (ui *AdminUI) handleScanBackfill(w http.ResponseWriter, r *http.Request) {
|
||||
session := getSessionFromContext(r.Context())
|
||||
wantJSON := strings.Contains(r.Header.Get("Accept"), "application/json")
|
||||
|
||||
if wantJSON {
|
||||
// Synchronous JSON path — caller decides their own timeout.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
res, err := ui.pds.BackfillScanStatus(ctx, scanBackfillLogger, nil)
|
||||
if err != nil {
|
||||
slog.Error("scan-backfill failed", "by", session.DID, "error", err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
slog.Info("scan-status backfill complete (sync)",
|
||||
"by", session.DID,
|
||||
"scanned", res.Scanned,
|
||||
"already_tagged", res.AlreadyTagged,
|
||||
"marked_skipped", res.MarkedSkipped,
|
||||
"marked_failed", res.MarkedFailed,
|
||||
"rewritten", res.Rewritten,
|
||||
)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
return
|
||||
}
|
||||
|
||||
// HTML path — kick off the background run if one isn't already going.
|
||||
st := &ui.scanBackfill
|
||||
st.mu.Lock()
|
||||
alreadyRunning := st.running
|
||||
if !alreadyRunning {
|
||||
st.running = true
|
||||
st.startedAt = time.Now()
|
||||
st.current = &pds.ScanBackfillResult{}
|
||||
st.result = nil
|
||||
st.err = ""
|
||||
}
|
||||
st.mu.Unlock()
|
||||
|
||||
if !alreadyRunning {
|
||||
slog.Info("scan-status backfill started via admin panel", "by", session.DID)
|
||||
go ui.runScanBackfill()
|
||||
} else {
|
||||
slog.Debug("scan-status backfill already in progress; returning current state")
|
||||
}
|
||||
|
||||
ui.renderTemplate(w, "partials/scan_backfill_progress.html", ui.snapshotScanBackfill())
|
||||
}
|
||||
|
||||
// handleScanBackfillStatus is polled by the progress fragment. Returns the
|
||||
// progress fragment again if running, the result fragment when done, or an
|
||||
// error fragment if something went wrong.
|
||||
func (ui *AdminUI) handleScanBackfillStatus(w http.ResponseWriter, r *http.Request) {
|
||||
snap := ui.snapshotScanBackfill()
|
||||
if snap.Running {
|
||||
ui.renderTemplate(w, "partials/scan_backfill_progress.html", snap)
|
||||
return
|
||||
}
|
||||
if snap.Error != "" {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{snap.Error})
|
||||
return
|
||||
}
|
||||
if snap.Result == nil {
|
||||
// Initial state, before any run — render an empty placeholder.
|
||||
_, _ = w.Write([]byte(""))
|
||||
return
|
||||
}
|
||||
ui.renderTemplate(w, "partials/scan_backfill_result.html", snap.Result)
|
||||
}
|
||||
|
||||
// runScanBackfill is the goroutine body. Updates the shared state as the
|
||||
// backfill progresses and stores the final result (or error) when it ends.
|
||||
func (ui *AdminUI) runScanBackfill() {
|
||||
st := &ui.scanBackfill
|
||||
defer func() {
|
||||
st.mu.Lock()
|
||||
st.running = false
|
||||
st.mu.Unlock()
|
||||
}()
|
||||
|
||||
// Generous independent timeout — the loop is single-threaded and large
|
||||
// holds with thousands of legacy records can take a few minutes.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
res, err := ui.pds.BackfillScanStatus(ctx, scanBackfillLogger, func(snap *pds.ScanBackfillResult) {
|
||||
// Copy so we don't keep a pointer the loop will mutate.
|
||||
c := *snap
|
||||
st.mu.Lock()
|
||||
st.current = &c
|
||||
st.mu.Unlock()
|
||||
})
|
||||
st.mu.Lock()
|
||||
if err != nil {
|
||||
st.err = err.Error()
|
||||
slog.Error("scan-status backfill failed", "error", err)
|
||||
} else {
|
||||
st.result = res
|
||||
slog.Info("scan-status backfill complete",
|
||||
"scanned", res.Scanned,
|
||||
"already_tagged", res.AlreadyTagged,
|
||||
"marked_skipped", res.MarkedSkipped,
|
||||
"marked_failed", res.MarkedFailed,
|
||||
"rewritten", res.Rewritten,
|
||||
)
|
||||
}
|
||||
st.mu.Unlock()
|
||||
}
|
||||
|
||||
// scanBackfillSnapshot is the shape exposed to templates.
|
||||
type scanBackfillSnapshot struct {
|
||||
Running bool
|
||||
StartedAt time.Time
|
||||
Current *pds.ScanBackfillResult // populated while running
|
||||
Result *pds.ScanBackfillResult // populated when complete
|
||||
Error string
|
||||
}
|
||||
|
||||
// snapshotScanBackfill returns a copy of the current state — safe to render
|
||||
// without holding the mutex.
|
||||
func (ui *AdminUI) snapshotScanBackfill() scanBackfillSnapshot {
|
||||
st := &ui.scanBackfill
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
|
||||
snap := scanBackfillSnapshot{
|
||||
Running: st.running,
|
||||
StartedAt: st.startedAt,
|
||||
Error: st.err,
|
||||
}
|
||||
if st.current != nil {
|
||||
c := *st.current
|
||||
snap.Current = &c
|
||||
}
|
||||
if st.result != nil {
|
||||
r := *st.result
|
||||
snap.Result = &r
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
// scanBackfillLogger formats the printf-style messages from BackfillScanStatus
|
||||
// into a single slog message — slog's variadic args are key/value pairs, not
|
||||
// printf operands.
|
||||
func scanBackfillLogger(format string, args ...any) {
|
||||
slog.Warn("scan-backfill: " + fmt.Sprintf(format, args...))
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{{define "partials/scan_backfill_progress.html"}}
|
||||
<div hx-get="/admin/api/scan-backfill/status"
|
||||
hx-trigger="load delay:1s"
|
||||
hx-swap="outerHTML"
|
||||
class="alert alert-info">
|
||||
<span class="loading loading-spinner loading-sm" aria-hidden="true"></span>
|
||||
<div class="text-sm">
|
||||
<p class="font-medium">Backfilling scan records...</p>
|
||||
{{ if .Current }}
|
||||
<p class="text-base-content/70 mt-1">
|
||||
Scanned <span class="font-mono">{{ .Current.Scanned }}</span> records ·
|
||||
rewrites: <span class="font-mono">{{ .Current.Rewritten }}</span>
|
||||
({{ .Current.MarkedSkipped }} skipped, {{ .Current.MarkedFailed }} failed)
|
||||
</p>
|
||||
{{ else }}
|
||||
<p class="text-base-content/70 mt-1">Starting...</p>
|
||||
{{ end }}
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -0,0 +1,15 @@
|
||||
{{define "partials/scan_backfill_result.html"}}
|
||||
<div class="alert alert-success">
|
||||
{{ icon "check-circle" "size-4 shrink-0" }}
|
||||
<div>
|
||||
<p class="font-medium">Scan-record backfill complete</p>
|
||||
<ul class="text-sm mt-1 space-y-0.5">
|
||||
<li>Scanned: <span class="font-mono">{{ .Scanned }}</span></li>
|
||||
<li>Already tagged: <span class="font-mono">{{ .AlreadyTagged }}</span></li>
|
||||
<li>→ Skipped (helm / in-toto / DSSE): <span class="font-mono">{{ .MarkedSkipped }}</span></li>
|
||||
<li>→ Failed (legacy errors): <span class="font-mono">{{ .MarkedFailed }}</span></li>
|
||||
<li>Total rewritten: <span class="font-mono">{{ .Rewritten }}</span></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -67,4 +67,31 @@
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
<!-- Scan-record maintenance -->
|
||||
<div class="card bg-base-100 shadow-sm mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg">Scan records</h2>
|
||||
<p class="text-sm text-base-content/70">
|
||||
Rewrite legacy scan records (created before the <code class="text-xs">status</code> field
|
||||
existed) so the appview can distinguish intentionally skipped artifacts (helm charts,
|
||||
in-toto, DSSE) from genuine failures. Idempotent — safe to re-run.
|
||||
</p>
|
||||
<div class="flex gap-3 mt-2">
|
||||
<button class="btn btn-outline gap-2"
|
||||
hx-post="/admin/api/scan-backfill"
|
||||
hx-target="#scan-backfill-result"
|
||||
hx-swap="innerHTML"
|
||||
hx-disabled-elt="this"
|
||||
hx-indicator="#scan-backfill-spinner">
|
||||
{{ icon "refresh-cw" "size-4" }}
|
||||
Backfill scan statuses
|
||||
</button>
|
||||
<span id="scan-backfill-spinner" class="htmx-indicator self-center" aria-hidden="true">
|
||||
<span class="loading loading-spinner loading-sm"></span>
|
||||
</span>
|
||||
</div>
|
||||
<div id="scan-backfill-result" class="mt-3" aria-live="polite"></div>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
@@ -3,11 +3,162 @@ package pds
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// unscannableLayerMediaSubstrings lists layer media-type fragments that
|
||||
// identify artifact types the scanner intentionally skips. Kept in sync with
|
||||
// scanner/internal/scan/worker.go's unscannableConfigTypes — that map keys on
|
||||
// config media types; here we look at *layer* media types because the
|
||||
// backfill walks the hold's layer index (manifest AT-URIs join layers to
|
||||
// scan records).
|
||||
var unscannableLayerMediaSubstrings = []string{
|
||||
"helm.chart.content",
|
||||
"in-toto",
|
||||
"dsse.envelope",
|
||||
}
|
||||
|
||||
// ScanBackfillResult summarizes what BackfillScanStatus did.
|
||||
type ScanBackfillResult struct {
|
||||
Scanned int // total scan records examined
|
||||
AlreadyTagged int // records with non-empty status (or non-failure shape) — left alone
|
||||
MarkedSkipped int // records rewritten as status=skipped
|
||||
MarkedFailed int // records rewritten as status=failed
|
||||
Rewritten int // total rewrites (MarkedSkipped + MarkedFailed) that succeeded
|
||||
}
|
||||
|
||||
// BackfillScanStatus walks every io.atcr.hold.scan record on this hold and
|
||||
// assigns a status ("skipped" or "failed") to legacy records that pre-date the
|
||||
// status field. Idempotent — records that already have a non-empty status are
|
||||
// left alone.
|
||||
//
|
||||
// A legacy record is one with empty status, no SBOM blob, and zero
|
||||
// vulnerability counts. Layer media types are inspected to choose the
|
||||
// rewrite: helm/in-toto/DSSE → skipped, otherwise → failed. ScannedAt is
|
||||
// preserved on rewrite so the rescan timer doesn't reset.
|
||||
//
|
||||
// Safe to call on a running hold — uses the existing repomgr APIs the same
|
||||
// way scan-job handling does.
|
||||
//
|
||||
// progress, if non-nil, is called periodically with a snapshot of the
|
||||
// running totals so callers can surface progress to the UI. The callback
|
||||
// must not retain the pointer past its return.
|
||||
func (p *HoldPDS) BackfillScanStatus(ctx context.Context, log func(format string, args ...any), progress func(*ScanBackfillResult)) (*ScanBackfillResult, error) {
|
||||
if log == nil {
|
||||
log = func(string, ...any) {}
|
||||
}
|
||||
if progress == nil {
|
||||
progress = func(*ScanBackfillResult) {}
|
||||
}
|
||||
ri := p.RecordsIndex()
|
||||
if ri == nil {
|
||||
return nil, fmt.Errorf("records index not available")
|
||||
}
|
||||
|
||||
const batchSize = 200
|
||||
res := &ScanBackfillResult{}
|
||||
var cursor string
|
||||
|
||||
for {
|
||||
records, nextCursor, err := ri.ListRecords(atproto.ScanCollection, batchSize, cursor, true)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("list scan records: %w", err)
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Scanned++
|
||||
manifestDigest := "sha256:" + rec.Rkey
|
||||
|
||||
_, scanRecord, err := p.GetScanRecord(ctx, manifestDigest)
|
||||
if err != nil {
|
||||
log("skip rkey=%s: get failed: %v", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Already classified — nothing to do.
|
||||
if scanRecord.Status != "" {
|
||||
res.AlreadyTagged++
|
||||
continue
|
||||
}
|
||||
// Real data present (legacy successful scan) — treat as ok, don't rewrite.
|
||||
if scanRecord.SbomBlob != nil || scanRecord.Total != 0 {
|
||||
res.AlreadyTagged++
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine artifact type from layer media types.
|
||||
layers, err := p.ListLayerRecordsForManifest(ctx, scanRecord.Manifest)
|
||||
if err != nil {
|
||||
log("skip rkey=%s: list layers failed: %v", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
skipped := false
|
||||
for _, l := range layers {
|
||||
for _, frag := range unscannableLayerMediaSubstrings {
|
||||
if strings.Contains(l.MediaType, frag) {
|
||||
skipped = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skipped {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var rewrite *atproto.ScanRecord
|
||||
if skipped {
|
||||
rewrite = atproto.NewSkippedScanRecord(
|
||||
manifestDigest,
|
||||
scanRecord.Repository,
|
||||
scanRecord.UserDID,
|
||||
"backfilled: unscannable artifact type",
|
||||
scanRecord.ScannerVersion,
|
||||
)
|
||||
res.MarkedSkipped++
|
||||
} else {
|
||||
rewrite = atproto.NewFailedScanRecord(
|
||||
manifestDigest,
|
||||
scanRecord.Repository,
|
||||
scanRecord.UserDID,
|
||||
"backfilled: legacy record (no SBOM and zero counts)",
|
||||
scanRecord.ScannerVersion,
|
||||
)
|
||||
res.MarkedFailed++
|
||||
}
|
||||
// Preserve original ScannedAt so the rescan timer doesn't reset
|
||||
// and we don't lose audit signal.
|
||||
if scanRecord.ScannedAt != "" {
|
||||
rewrite.ScannedAt = scanRecord.ScannedAt
|
||||
}
|
||||
|
||||
if _, _, err := p.CreateScanRecord(ctx, rewrite); err != nil {
|
||||
log("rewrite rkey=%s failed: %v", rec.Rkey, err)
|
||||
continue
|
||||
}
|
||||
res.Rewritten++
|
||||
}
|
||||
|
||||
// Report progress at every batch boundary — gives the UI smooth-ish
|
||||
// updates without locking the loop on every record.
|
||||
progress(res)
|
||||
|
||||
if nextCursor == "" || len(records) == 0 {
|
||||
break
|
||||
}
|
||||
cursor = nextCursor
|
||||
}
|
||||
|
||||
progress(res)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// CreateScanRecord creates or updates a scan result record in the hold's PDS
|
||||
// Uses a deterministic rkey based on the manifest digest, so re-scans upsert
|
||||
func (p *HoldPDS) CreateScanRecord(ctx context.Context, record *atproto.ScanRecord) (string, cid.Cid, error) {
|
||||
|
||||
Reference in New Issue
Block a user