From 13a793ca90f9aac0fba87d87ca3f2b17c4837071 Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Wed, 29 Apr 2026 10:37:36 -0500 Subject: [PATCH] improve admin tooling --- cmd/hold/scan_backfill.go | 156 ++------- lexicons/io/atcr/hold/image/config.json | 2 +- .../partials/image-advisor-results.html | 2 +- pkg/atproto/cbor_gen.go | 326 +++++++++--------- pkg/atproto/generate.go | 11 +- pkg/atproto/lexicon.go | 6 +- pkg/atproto/lexicon_test.go | 70 ++++ pkg/hold/admin/admin.go | 21 ++ pkg/hold/admin/handlers_scan.go | 173 ++++++++++ .../partials/scan_backfill_progress.html | 20 ++ .../partials/scan_backfill_result.html | 15 + .../admin/templates/partials/tab_storage.html | 27 ++ pkg/hold/pds/scan.go | 151 ++++++++ 13 files changed, 678 insertions(+), 302 deletions(-) create mode 100644 pkg/hold/admin/handlers_scan.go create mode 100644 pkg/hold/admin/templates/partials/scan_backfill_progress.html create mode 100644 pkg/hold/admin/templates/partials/scan_backfill_result.html diff --git a/cmd/hold/scan_backfill.go b/cmd/hold/scan_backfill.go index 0999f2e..b577c9a 100644 --- a/cmd/hold/scan_backfill.go +++ b/cmd/hold/scan_backfill.go @@ -3,45 +3,32 @@ package main import ( "context" "fmt" - "strings" - "atcr.io/pkg/atproto" "atcr.io/pkg/hold" "github.com/spf13/cobra" ) -// Media-type fragments that identify artifact types the scanner intentionally -// skips. Keep this list in sync with scanner/internal/scan/worker.go's -// unscannableConfigTypes — that map keys on config media types; here we look -// at *layer* media types because the backfill walks the hold's layer index -// (which has manifest AT-URIs we can join against scan records). -// -// Detection by layer media type is reliable: helm charts always have a single -// layer with media type application/vnd.cncf.helm.chart.content.v1.tar+gzip; -// in-toto / DSSE attestations use distinct layer types too. -var unscannableLayerMediaSubstrings = []string{ - "helm.chart.content", - "in-toto", - "dsse.envelope", -} - var scanBackfillConfigFile string var scanBackfillCmd = &cobra.Command{ Use: "scan-backfill", - Short: "Rewrite legacy scan records to use the status field", + Short: "Rewrite legacy scan records to use the status field (offline)", Long: `Walks every io.atcr.hold.scan record on this hold and assigns a status ("skipped" or "failed") to records that pre-date the status field. -A legacy record is one with an empty status, no SBOM blob, and zero vulnerability -counts. The tool inspects each record's manifest's layers to decide: +A legacy record is one with an empty status, no SBOM blob, and zero +vulnerability counts. Layer media types decide the rewrite: - - layer media type matches helm/in-toto/DSSE → status="skipped" - - everything else → status="failed" + - helm.chart.content / in-toto / dsse.envelope → status="skipped" + - everything else → status="failed" -The tool is idempotent: records that already have a status are left alone. -Run once per hold after upgrading.`, +The tool is idempotent and preserves each record's original scannedAt. + +This subcommand opens the hold's CAR store directly, so the running hold +service must be stopped first (otherwise the embedded PDS holds an exclusive +lock). For zero-downtime backfill on a production hold, hit the admin +endpoint POST /admin/api/scan-backfill instead.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := hold.LoadConfig(scanBackfillConfigFile) @@ -56,116 +43,21 @@ Run once per hold after upgrading.`, } defer cleanup() - ri := holdPDS.RecordsIndex() - if ri == nil { - return fmt.Errorf("records index not available") + logf := func(format string, args ...any) { + fmt.Fprintf(cmd.ErrOrStderr(), " "+format+"\n", args...) + } + res, err := holdPDS.BackfillScanStatus(ctx, logf, nil) + if err != nil { + return fmt.Errorf("backfill: %w", err) } - const batchSize = 200 - var ( - cursor string - scanned int - rewritten int - markSkipped int - markFailed int - alreadyOK int - ) - - for { - records, nextCursor, err := ri.ListRecords(atproto.ScanCollection, batchSize, cursor, true) - if err != nil { - return fmt.Errorf("list scan records: %w", err) - } - - for _, rec := range records { - scanned++ - manifestDigest := "sha256:" + rec.Rkey - - _, scanRecord, err := holdPDS.GetScanRecord(ctx, manifestDigest) - if err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), " skip rkey=%s: get failed: %v\n", rec.Rkey, err) - continue - } - - // Already classified — nothing to do. - if scanRecord.Status != "" { - alreadyOK++ - continue - } - - // Only legacy records that signal failure (nil blob + zero - // counts) are candidates. Records with real data don't need - // rewriting; their absent status will be treated as "ok". - if scanRecord.SbomBlob != nil || scanRecord.Total != 0 { - alreadyOK++ - continue - } - - // Determine artifact type from layer media types. - layers, err := holdPDS.ListLayerRecordsForManifest(ctx, scanRecord.Manifest) - if err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), " skip rkey=%s: list layers failed: %v\n", rec.Rkey, err) - continue - } - - skipped := false - for _, l := range layers { - for _, frag := range unscannableLayerMediaSubstrings { - if strings.Contains(l.MediaType, frag) { - skipped = true - break - } - } - if skipped { - break - } - } - - var rewrite *atproto.ScanRecord - if skipped { - rewrite = atproto.NewSkippedScanRecord( - manifestDigest, - scanRecord.Repository, - scanRecord.UserDID, - "backfilled: unscannable artifact type", - scanRecord.ScannerVersion, - ) - markSkipped++ - } else { - rewrite = atproto.NewFailedScanRecord( - manifestDigest, - scanRecord.Repository, - scanRecord.UserDID, - "backfilled: legacy record (no SBOM and zero counts)", - scanRecord.ScannerVersion, - ) - markFailed++ - } - // Preserve the original ScannedAt — rewriting it would either - // reset the rescan timer or invalidate audit signals. - if scanRecord.ScannedAt != "" { - rewrite.ScannedAt = scanRecord.ScannedAt - } - - if _, _, err := holdPDS.CreateScanRecord(ctx, rewrite); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), " rewrite rkey=%s failed: %v\n", rec.Rkey, err) - continue - } - rewritten++ - } - - if nextCursor == "" || len(records) == 0 { - break - } - cursor = nextCursor - } - - fmt.Fprintf(cmd.OutOrStdout(), "Backfill complete:\n") - fmt.Fprintf(cmd.OutOrStdout(), " scanned: %d\n", scanned) - fmt.Fprintf(cmd.OutOrStdout(), " already-tagged: %d\n", alreadyOK) - fmt.Fprintf(cmd.OutOrStdout(), " → skipped: %d\n", markSkipped) - fmt.Fprintf(cmd.OutOrStdout(), " → failed: %d\n", markFailed) - fmt.Fprintf(cmd.OutOrStdout(), " rewritten: %d\n", rewritten) + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Backfill complete:\n") + fmt.Fprintf(out, " scanned: %d\n", res.Scanned) + fmt.Fprintf(out, " already-tagged: %d\n", res.AlreadyTagged) + fmt.Fprintf(out, " → skipped: %d\n", res.MarkedSkipped) + fmt.Fprintf(out, " → failed: %d\n", res.MarkedFailed) + fmt.Fprintf(out, " rewritten: %d\n", res.Rewritten) return nil }, } diff --git a/lexicons/io/atcr/hold/image/config.json b/lexicons/io/atcr/hold/image/config.json index bfaf8e7..fbe739e 100644 --- a/lexicons/io/atcr/hold/image/config.json +++ b/lexicons/io/atcr/hold/image/config.json @@ -18,7 +18,7 @@ "configJson": { "type": "string", "description": "Raw OCI image config JSON blob", - "maxLength": 65536 + "maxLength": 1000000 }, "createdAt": { "type": "string", diff --git a/pkg/appview/templates/partials/image-advisor-results.html b/pkg/appview/templates/partials/image-advisor-results.html index fdab1b5..cc7f500 100644 --- a/pkg/appview/templates/partials/image-advisor-results.html +++ b/pkg/appview/templates/partials/image-advisor-results.html @@ -2,7 +2,7 @@ {{ if eq .Error "upgrade_required" }}
{{ icon "sparkles" "size-4" }} - AI Image Advisor is a paid feature. Upgrade your plan to unlock image analysis. + AI Image Advisor is a paid feature. Upgrade your plan to unlock image analysis.
{{ else if .Error }}
diff --git a/pkg/atproto/cbor_gen.go b/pkg/atproto/cbor_gen.go index 9e00218..ba7fd14 100644 --- a/pkg/atproto/cbor_gen.go +++ b/pkg/atproto/cbor_gen.go @@ -37,7 +37,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.Role (string) (string) - if len("role") > 8192 { + if len("role") > 1000000 { return xerrors.Errorf("Value in field \"role\" was too long") } @@ -48,7 +48,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Role) > 8192 { + if len(t.Role) > 1000000 { return xerrors.Errorf("Value in field t.Role was too long") } @@ -62,7 +62,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { // t.Tier (string) (string) if t.Tier != "" { - if len("tier") > 8192 { + if len("tier") > 1000000 { return xerrors.Errorf("Value in field \"tier\" was too long") } @@ -73,7 +73,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Tier) > 8192 { + if len(t.Tier) > 1000000 { return xerrors.Errorf("Value in field t.Tier was too long") } @@ -86,7 +86,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -97,7 +97,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -109,7 +109,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.Member (string) (string) - if len("member") > 8192 { + if len("member") > 1000000 { return xerrors.Errorf("Value in field \"member\" was too long") } @@ -120,7 +120,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Member) > 8192 { + if len(t.Member) > 1000000 { return xerrors.Errorf("Value in field t.Member was too long") } @@ -132,7 +132,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.AddedAt (string) (string) - if len("addedAt") > 8192 { + if len("addedAt") > 1000000 { return xerrors.Errorf("Value in field \"addedAt\" was too long") } @@ -143,7 +143,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.AddedAt) > 8192 { + if len(t.AddedAt) > 1000000 { return xerrors.Errorf("Value in field t.AddedAt was too long") } @@ -155,7 +155,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.Plankowner (bool) (bool) - if len("plankowner") > 8192 { + if len("plankowner") > 1000000 { return xerrors.Errorf("Value in field \"plankowner\" was too long") } @@ -171,7 +171,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { } // t.Permissions ([]string) (slice) - if len("permissions") > 8192 { + if len("permissions") > 1000000 { return xerrors.Errorf("Value in field \"permissions\" was too long") } @@ -190,7 +190,7 @@ func (t *CrewRecord) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.Permissions { - if len(v) > 8192 { + if len(v) > 1000000 { return xerrors.Errorf("Value in field v was too long") } @@ -232,7 +232,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -250,7 +250,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { case "role": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -261,7 +261,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { case "tier": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -272,7 +272,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -283,7 +283,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { case "member": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -294,7 +294,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { case "addedAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -349,7 +349,7 @@ func (t *CrewRecord) UnmarshalCBOR(r io.Reader) (err error) { _ = err { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -392,7 +392,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -403,7 +403,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -415,7 +415,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.Owner (string) (string) - if len("owner") > 8192 { + if len("owner") > 1000000 { return xerrors.Errorf("Value in field \"owner\" was too long") } @@ -426,7 +426,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Owner) > 8192 { + if len(t.Owner) > 1000000 { return xerrors.Errorf("Value in field t.Owner was too long") } @@ -438,7 +438,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.Public (bool) (bool) - if len("public") > 8192 { + if len("public") > 1000000 { return xerrors.Errorf("Value in field \"public\" was too long") } @@ -456,7 +456,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { // t.Region (string) (string) if t.Region != "" { - if len("region") > 8192 { + if len("region") > 1000000 { return xerrors.Errorf("Value in field \"region\" was too long") } @@ -467,7 +467,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Region) > 8192 { + if len(t.Region) > 1000000 { return xerrors.Errorf("Value in field t.Region was too long") } @@ -482,7 +482,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { // t.Successor (string) (string) if t.Successor != "" { - if len("successor") > 8192 { + if len("successor") > 1000000 { return xerrors.Errorf("Value in field \"successor\" was too long") } @@ -493,7 +493,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Successor) > 8192 { + if len(t.Successor) > 1000000 { return xerrors.Errorf("Value in field t.Successor was too long") } @@ -506,7 +506,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.DeployedAt (string) (string) - if len("deployedAt") > 8192 { + if len("deployedAt") > 1000000 { return xerrors.Errorf("Value in field \"deployedAt\" was too long") } @@ -517,7 +517,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.DeployedAt) > 8192 { + if len(t.DeployedAt) > 1000000 { return xerrors.Errorf("Value in field t.DeployedAt was too long") } @@ -529,7 +529,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.AllowAllCrew (bool) (bool) - if len("allowAllCrew") > 8192 { + if len("allowAllCrew") > 1000000 { return xerrors.Errorf("Value in field \"allowAllCrew\" was too long") } @@ -545,7 +545,7 @@ func (t *CaptainRecord) MarshalCBOR(w io.Writer) error { } // t.EnableBlueskyPosts (bool) (bool) - if len("enableBlueskyPosts") > 8192 { + if len("enableBlueskyPosts") > 1000000 { return xerrors.Errorf("Value in field \"enableBlueskyPosts\" was too long") } @@ -589,7 +589,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 18) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -607,7 +607,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -618,7 +618,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { case "owner": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -647,7 +647,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { case "region": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -658,7 +658,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { case "successor": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -669,7 +669,7 @@ func (t *CaptainRecord) UnmarshalCBOR(r io.Reader) (err error) { case "deployedAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -736,7 +736,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.Size (int64) (int64) - if len("size") > 8192 { + if len("size") > 1000000 { return xerrors.Errorf("Value in field \"size\" was too long") } @@ -758,7 +758,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -769,7 +769,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -781,7 +781,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.Digest (string) (string) - if len("digest") > 8192 { + if len("digest") > 1000000 { return xerrors.Errorf("Value in field \"digest\" was too long") } @@ -792,7 +792,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Digest) > 8192 { + if len(t.Digest) > 1000000 { return xerrors.Errorf("Value in field t.Digest was too long") } @@ -804,7 +804,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.UserDID (string) (string) - if len("userDid") > 8192 { + if len("userDid") > 1000000 { return xerrors.Errorf("Value in field \"userDid\" was too long") } @@ -815,7 +815,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.UserDID) > 8192 { + if len(t.UserDID) > 1000000 { return xerrors.Errorf("Value in field t.UserDID was too long") } @@ -827,7 +827,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.Manifest (string) (string) - if len("manifest") > 8192 { + if len("manifest") > 1000000 { return xerrors.Errorf("Value in field \"manifest\" was too long") } @@ -838,7 +838,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Manifest) > 8192 { + if len(t.Manifest) > 1000000 { return xerrors.Errorf("Value in field t.Manifest was too long") } @@ -850,7 +850,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.CreatedAt (string) (string) - if len("createdAt") > 8192 { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } @@ -861,7 +861,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.CreatedAt) > 8192 { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } @@ -873,7 +873,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { } // t.MediaType (string) (string) - if len("mediaType") > 8192 { + if len("mediaType") > 1000000 { return xerrors.Errorf("Value in field \"mediaType\" was too long") } @@ -884,7 +884,7 @@ func (t *LayerRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.MediaType) > 8192 { + if len(t.MediaType) > 1000000 { return xerrors.Errorf("Value in field t.MediaType was too long") } @@ -924,7 +924,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -968,7 +968,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -979,7 +979,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "digest": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -990,7 +990,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "userDid": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1001,7 +1001,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "manifest": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1012,7 +1012,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "createdAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1023,7 +1023,7 @@ func (t *LayerRecord) UnmarshalCBOR(r io.Reader) (err error) { case "mediaType": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1054,7 +1054,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -1065,7 +1065,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -1077,7 +1077,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Links ([]string) (slice) - if len("links") > 8192 { + if len("links") > 1000000 { return xerrors.Errorf("Value in field \"links\" was too long") } @@ -1096,7 +1096,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.Links { - if len(v) > 8192 { + if len(v) > 1000000 { return xerrors.Errorf("Value in field v was too long") } @@ -1110,7 +1110,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Stats ([]string) (slice) - if len("stats") > 8192 { + if len("stats") > 1000000 { return xerrors.Errorf("Value in field \"stats\" was too long") } @@ -1129,7 +1129,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.Stats { - if len(v) > 8192 { + if len(v) > 1000000 { return xerrors.Errorf("Value in field v was too long") } @@ -1143,7 +1143,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Bluesky (bool) (bool) - if len("bluesky") > 8192 { + if len("bluesky") > 1000000 { return xerrors.Errorf("Value in field \"bluesky\" was too long") } @@ -1159,7 +1159,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Location (string) (string) - if len("location") > 8192 { + if len("location") > 1000000 { return xerrors.Errorf("Value in field \"location\" was too long") } @@ -1170,7 +1170,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Location) > 8192 { + if len(t.Location) > 1000000 { return xerrors.Errorf("Value in field t.Location was too long") } @@ -1182,7 +1182,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.Description (string) (string) - if len("description") > 8192 { + if len("description") > 1000000 { return xerrors.Errorf("Value in field \"description\" was too long") } @@ -1193,7 +1193,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Description) > 8192 { + if len(t.Description) > 1000000 { return xerrors.Errorf("Value in field t.Description was too long") } @@ -1205,7 +1205,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { } // t.PinnedRepositories ([]string) (slice) - if len("pinnedRepositories") > 8192 { + if len("pinnedRepositories") > 1000000 { return xerrors.Errorf("Value in field \"pinnedRepositories\" was too long") } @@ -1224,7 +1224,7 @@ func (t *TangledProfileRecord) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.PinnedRepositories { - if len(v) > 8192 { + if len(v) > 1000000 { return xerrors.Errorf("Value in field v was too long") } @@ -1266,7 +1266,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 18) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -1284,7 +1284,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1321,7 +1321,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { _ = err { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1361,7 +1361,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { _ = err { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1393,7 +1393,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { case "location": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1404,7 +1404,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { case "description": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1441,7 +1441,7 @@ func (t *TangledProfileRecord) UnmarshalCBOR(r io.Reader) (err error) { _ = err { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1484,7 +1484,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -1495,7 +1495,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -1509,7 +1509,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { // t.LastPull (string) (string) if t.LastPull != "" { - if len("lastPull") > 8192 { + if len("lastPull") > 1000000 { return xerrors.Errorf("Value in field \"lastPull\" was too long") } @@ -1520,7 +1520,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.LastPull) > 8192 { + if len(t.LastPull) > 1000000 { return xerrors.Errorf("Value in field t.LastPull was too long") } @@ -1535,7 +1535,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { // t.LastPush (string) (string) if t.LastPush != "" { - if len("lastPush") > 8192 { + if len("lastPush") > 1000000 { return xerrors.Errorf("Value in field \"lastPush\" was too long") } @@ -1546,7 +1546,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.LastPush) > 8192 { + if len(t.LastPush) > 1000000 { return xerrors.Errorf("Value in field t.LastPush was too long") } @@ -1559,7 +1559,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.OwnerDID (string) (string) - if len("ownerDid") > 8192 { + if len("ownerDid") > 1000000 { return xerrors.Errorf("Value in field \"ownerDid\" was too long") } @@ -1570,7 +1570,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.OwnerDID) > 8192 { + if len(t.OwnerDID) > 1000000 { return xerrors.Errorf("Value in field t.OwnerDID was too long") } @@ -1582,7 +1582,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.PullCount (int64) (int64) - if len("pullCount") > 8192 { + if len("pullCount") > 1000000 { return xerrors.Errorf("Value in field \"pullCount\" was too long") } @@ -1604,7 +1604,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.PushCount (int64) (int64) - if len("pushCount") > 8192 { + if len("pushCount") > 1000000 { return xerrors.Errorf("Value in field \"pushCount\" was too long") } @@ -1626,7 +1626,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.UpdatedAt (string) (string) - if len("updatedAt") > 8192 { + if len("updatedAt") > 1000000 { return xerrors.Errorf("Value in field \"updatedAt\" was too long") } @@ -1637,7 +1637,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.UpdatedAt) > 8192 { + if len(t.UpdatedAt) > 1000000 { return xerrors.Errorf("Value in field t.UpdatedAt was too long") } @@ -1649,7 +1649,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { } // t.Repository (string) (string) - if len("repository") > 8192 { + if len("repository") > 1000000 { return xerrors.Errorf("Value in field \"repository\" was too long") } @@ -1660,7 +1660,7 @@ func (t *StatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Repository) > 8192 { + if len(t.Repository) > 1000000 { return xerrors.Errorf("Value in field t.Repository was too long") } @@ -1700,7 +1700,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -1718,7 +1718,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1729,7 +1729,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "lastPull": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1740,7 +1740,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "lastPush": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1751,7 +1751,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "ownerDid": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1814,7 +1814,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "updatedAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1825,7 +1825,7 @@ func (t *StatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "repository": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -1856,7 +1856,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.Date (string) (string) - if len("date") > 8192 { + if len("date") > 1000000 { return xerrors.Errorf("Value in field \"date\" was too long") } @@ -1867,7 +1867,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Date) > 8192 { + if len(t.Date) > 1000000 { return xerrors.Errorf("Value in field t.Date was too long") } @@ -1879,7 +1879,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -1890,7 +1890,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -1902,7 +1902,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.OwnerDID (string) (string) - if len("ownerDid") > 8192 { + if len("ownerDid") > 1000000 { return xerrors.Errorf("Value in field \"ownerDid\" was too long") } @@ -1913,7 +1913,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.OwnerDID) > 8192 { + if len(t.OwnerDID) > 1000000 { return xerrors.Errorf("Value in field t.OwnerDID was too long") } @@ -1925,7 +1925,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.PullCount (int64) (int64) - if len("pullCount") > 8192 { + if len("pullCount") > 1000000 { return xerrors.Errorf("Value in field \"pullCount\" was too long") } @@ -1947,7 +1947,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.PushCount (int64) (int64) - if len("pushCount") > 8192 { + if len("pushCount") > 1000000 { return xerrors.Errorf("Value in field \"pushCount\" was too long") } @@ -1969,7 +1969,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.UpdatedAt (string) (string) - if len("updatedAt") > 8192 { + if len("updatedAt") > 1000000 { return xerrors.Errorf("Value in field \"updatedAt\" was too long") } @@ -1980,7 +1980,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.UpdatedAt) > 8192 { + if len(t.UpdatedAt) > 1000000 { return xerrors.Errorf("Value in field t.UpdatedAt was too long") } @@ -1992,7 +1992,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { } // t.Repository (string) (string) - if len("repository") > 8192 { + if len("repository") > 1000000 { return xerrors.Errorf("Value in field \"repository\" was too long") } @@ -2003,7 +2003,7 @@ func (t *DailyStatsRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Repository) > 8192 { + if len(t.Repository) > 1000000 { return xerrors.Errorf("Value in field t.Repository was too long") } @@ -2043,7 +2043,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -2061,7 +2061,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "date": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2072,7 +2072,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2083,7 +2083,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "ownerDid": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2146,7 +2146,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "updatedAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2157,7 +2157,7 @@ func (t *DailyStatsRecord) UnmarshalCBOR(r io.Reader) (err error) { case "repository": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2197,7 +2197,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Low (int64) (int64) - if len("low") > 8192 { + if len("low") > 1000000 { return xerrors.Errorf("Value in field \"low\" was too long") } @@ -2219,7 +2219,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.High (int64) (int64) - if len("high") > 8192 { + if len("high") > 1000000 { return xerrors.Errorf("Value in field \"high\" was too long") } @@ -2241,7 +2241,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -2252,7 +2252,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -2264,7 +2264,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Total (int64) (int64) - if len("total") > 8192 { + if len("total") > 1000000 { return xerrors.Errorf("Value in field \"total\" was too long") } @@ -2286,7 +2286,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Medium (int64) (int64) - if len("medium") > 8192 { + if len("medium") > 1000000 { return xerrors.Errorf("Value in field \"medium\" was too long") } @@ -2310,7 +2310,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { // t.Reason (string) (string) if t.Reason != "" { - if len("reason") > 8192 { + if len("reason") > 1000000 { return xerrors.Errorf("Value in field \"reason\" was too long") } @@ -2321,7 +2321,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Reason) > 8192 { + if len(t.Reason) > 1000000 { return xerrors.Errorf("Value in field t.Reason was too long") } @@ -2336,7 +2336,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { // t.Status (string) (string) if t.Status != "" { - if len("status") > 8192 { + if len("status") > 1000000 { return xerrors.Errorf("Value in field \"status\" was too long") } @@ -2347,7 +2347,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Status) > 8192 { + if len(t.Status) > 1000000 { return xerrors.Errorf("Value in field t.Status was too long") } @@ -2360,7 +2360,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.UserDID (string) (string) - if len("userDid") > 8192 { + if len("userDid") > 1000000 { return xerrors.Errorf("Value in field \"userDid\" was too long") } @@ -2371,7 +2371,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.UserDID) > 8192 { + if len(t.UserDID) > 1000000 { return xerrors.Errorf("Value in field t.UserDID was too long") } @@ -2383,7 +2383,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Critical (int64) (int64) - if len("critical") > 8192 { + if len("critical") > 1000000 { return xerrors.Errorf("Value in field \"critical\" was too long") } @@ -2405,7 +2405,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Manifest (string) (string) - if len("manifest") > 8192 { + if len("manifest") > 1000000 { return xerrors.Errorf("Value in field \"manifest\" was too long") } @@ -2416,7 +2416,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Manifest) > 8192 { + if len(t.Manifest) > 1000000 { return xerrors.Errorf("Value in field t.Manifest was too long") } @@ -2428,7 +2428,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.SbomBlob (util.LexBlob) (struct) - if len("sbomBlob") > 8192 { + if len("sbomBlob") > 1000000 { return xerrors.Errorf("Value in field \"sbomBlob\" was too long") } @@ -2444,7 +2444,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.ScannedAt (string) (string) - if len("scannedAt") > 8192 { + if len("scannedAt") > 1000000 { return xerrors.Errorf("Value in field \"scannedAt\" was too long") } @@ -2455,7 +2455,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.ScannedAt) > 8192 { + if len(t.ScannedAt) > 1000000 { return xerrors.Errorf("Value in field t.ScannedAt was too long") } @@ -2467,7 +2467,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.Repository (string) (string) - if len("repository") > 8192 { + if len("repository") > 1000000 { return xerrors.Errorf("Value in field \"repository\" was too long") } @@ -2478,7 +2478,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Repository) > 8192 { + if len(t.Repository) > 1000000 { return xerrors.Errorf("Value in field t.Repository was too long") } @@ -2490,7 +2490,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.ScannerVersion (string) (string) - if len("scannerVersion") > 8192 { + if len("scannerVersion") > 1000000 { return xerrors.Errorf("Value in field \"scannerVersion\" was too long") } @@ -2501,7 +2501,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.ScannerVersion) > 8192 { + if len(t.ScannerVersion) > 1000000 { return xerrors.Errorf("Value in field t.ScannerVersion was too long") } @@ -2513,7 +2513,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error { } // t.VulnReportBlob (util.LexBlob) (struct) - if len("vulnReportBlob") > 8192 { + if len("vulnReportBlob") > 1000000 { return xerrors.Errorf("Value in field \"vulnReportBlob\" was too long") } @@ -2557,7 +2557,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 14) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -2627,7 +2627,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2690,7 +2690,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "reason": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2701,7 +2701,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "status": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2712,7 +2712,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "userDid": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2749,7 +2749,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "manifest": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2780,7 +2780,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "scannedAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2791,7 +2791,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "repository": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2802,7 +2802,7 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { case "scannerVersion": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -2853,7 +2853,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { } // t.Type (string) (string) - if len("$type") > 8192 { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } @@ -2864,7 +2864,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Type) > 8192 { + if len(t.Type) > 1000000 { return xerrors.Errorf("Value in field t.Type was too long") } @@ -2876,7 +2876,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { } // t.Manifest (string) (string) - if len("manifest") > 8192 { + if len("manifest") > 1000000 { return xerrors.Errorf("Value in field \"manifest\" was too long") } @@ -2887,7 +2887,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.Manifest) > 8192 { + if len(t.Manifest) > 1000000 { return xerrors.Errorf("Value in field t.Manifest was too long") } @@ -2899,7 +2899,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { } // t.CreatedAt (string) (string) - if len("createdAt") > 8192 { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } @@ -2910,7 +2910,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.CreatedAt) > 8192 { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } @@ -2922,7 +2922,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { } // t.ConfigJSON (string) (string) - if len("configJson") > 8192 { + if len("configJson") > 1000000 { return xerrors.Errorf("Value in field \"configJson\" was too long") } @@ -2933,7 +2933,7 @@ func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { return err } - if len(t.ConfigJSON) > 8192 { + if len(t.ConfigJSON) > 1000000 { return xerrors.Errorf("Value in field t.ConfigJSON was too long") } @@ -2973,7 +2973,7 @@ func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { return err } @@ -2991,7 +2991,7 @@ func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -3002,7 +3002,7 @@ func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { case "manifest": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -3013,7 +3013,7 @@ func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { case "createdAt": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -3024,7 +3024,7 @@ func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { case "configJson": { - sval, err := cbg.ReadStringWithMax(cr, 8192) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } diff --git a/pkg/atproto/generate.go b/pkg/atproto/generate.go index 8058af5..bc68509 100644 --- a/pkg/atproto/generate.go +++ b/pkg/atproto/generate.go @@ -25,8 +25,15 @@ import ( ) func main() { - // Generate map-style encoders - if err := cbg.WriteMapEncodersToFile("cbor_gen.go", "atproto", + // MaxStringLength bumps the package-wide string read limit. cbor-gen v0.3.1 + // honors per-field `cborgen:"...,maxlen=N"` tags only on the write side; the + // unmarshal template hard-codes the package default. ImageConfigRecord's + // configJson holds raw OCI image configs that routinely exceed the stock + // 8KB default (deep build histories, verbose created_by lines), so we lift + // the ceiling for everyone. Per-field write caps still apply. + gen := cbg.Gen{MaxStringLength: 1_000_000} + + if err := gen.WriteMapEncodersToFile("cbor_gen.go", "atproto", atproto.CrewRecord{}, atproto.CaptainRecord{}, atproto.LayerRecord{}, diff --git a/pkg/atproto/lexicon.go b/pkg/atproto/lexicon.go index 360ed82..54e7170 100644 --- a/pkg/atproto/lexicon.go +++ b/pkg/atproto/lexicon.go @@ -932,9 +932,9 @@ func ScanRecordKey(manifestDigest string) string { // Stores the full OCI config JSON so the appview can display layer history including empty layers type ImageConfigRecord struct { Type string `json:"$type" cborgen:"$type"` - Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the manifest - ConfigJSON string `json:"configJson" cborgen:"configJson"` // Raw OCI image config JSON - CreatedAt string `json:"createdAt" cborgen:"createdAt"` // RFC3339 timestamp + Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the manifest + ConfigJSON string `json:"configJson" cborgen:"configJson,maxlen=1000000"` // Raw OCI image config JSON. Cap mirrors lexicon maxLength so deep image histories (Bazel, multi-stage builds with verbose created_by lines) fit. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` // RFC3339 timestamp } // NewImageConfigRecord creates a new image config record diff --git a/pkg/atproto/lexicon_test.go b/pkg/atproto/lexicon_test.go index b7f855b..5a1f4d4 100644 --- a/pkg/atproto/lexicon_test.go +++ b/pkg/atproto/lexicon_test.go @@ -1,6 +1,7 @@ package atproto import ( + "bytes" "encoding/json" "strings" "testing" @@ -1385,3 +1386,72 @@ func TestRepoPageRecord_JSONSerialization(t *testing.T) { t.Errorf("Avatar.Ref.Link = %v, want %v", decoded.Avatar.Ref.Link, record.Avatar.Ref.Link) } } + +// TestImageConfigRecord_CBORRoundTrip locks in that we can encode and decode +// large OCI image configs without hitting cbor-gen's default 8KB string cap. +// Real-world images with deep build histories (Bazel, multi-stage Dockerfiles) +// routinely produce config blobs that blow past the default; if either side +// regresses, the backfill silently drops records with "configJson was too +// long" instead of populating the layer-history UI. +func TestImageConfigRecord_CBORRoundTrip(t *testing.T) { + tests := []struct { + name string + payloadSize int + }{ + // Small payloads exercise the happy path — should always have worked. + {"small", 1024}, + // 16KB is well past the old 8192 cborgen default. Pre-fix this would + // have failed at marshal time. + {"medium-16kb", 16 * 1024}, + // 200KB matches the upper end of pathological real configs (deep + // Bazel histories with verbose created_by lines). + {"large-200kb", 200 * 1024}, + // 900KB is just under our 1MB cap — the read side previously + // hard-coded 8192 even with the per-field write tag, so this also + // guards against the unmarshal regression. + {"near-cap-900kb", 900 * 1024}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Use a repeating non-trivial pattern so any byte-level corruption + // in encode/decode shows up as a mismatch rather than blending + // into a sea of identical bytes. + const chunk = "history entry: bazel build //pkg:foo # " + payload := strings.Repeat(chunk, tc.payloadSize/len(chunk)+1)[:tc.payloadSize] + + record := &ImageConfigRecord{ + Type: ImageConfigCollection, + Manifest: "at://did:plc:test/io.atcr.manifest/abc", + ConfigJSON: payload, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + + var buf bytes.Buffer + if err := record.MarshalCBOR(&buf); err != nil { + t.Fatalf("MarshalCBOR(%d bytes): %v", tc.payloadSize, err) + } + + var decoded ImageConfigRecord + if err := decoded.UnmarshalCBOR(&buf); err != nil { + t.Fatalf("UnmarshalCBOR(%d bytes): %v", tc.payloadSize, err) + } + + if decoded.Type != record.Type { + t.Errorf("Type = %q, want %q", decoded.Type, record.Type) + } + if decoded.Manifest != record.Manifest { + t.Errorf("Manifest = %q, want %q", decoded.Manifest, record.Manifest) + } + if decoded.CreatedAt != record.CreatedAt { + t.Errorf("CreatedAt = %q, want %q", decoded.CreatedAt, record.CreatedAt) + } + if len(decoded.ConfigJSON) != len(record.ConfigJSON) { + t.Fatalf("ConfigJSON length = %d, want %d", len(decoded.ConfigJSON), len(record.ConfigJSON)) + } + if decoded.ConfigJSON != record.ConfigJSON { + t.Errorf("ConfigJSON content mismatch at length %d", len(record.ConfigJSON)) + } + }) + } +} diff --git a/pkg/hold/admin/admin.go b/pkg/hold/admin/admin.go index 80e2c23..3f90c45 100644 --- a/pkg/hold/admin/admin.go +++ b/pkg/hold/admin/admin.go @@ -90,6 +90,21 @@ type AdminUI struct { // In-memory session storage (single user, no persistence needed) sessions map[string]*AdminSession sessionsMu sync.RWMutex + + // scan-backfill state — runs as a background goroutine on click; the + // status endpoint reads this for progress polling. Only one run at a + // time (idempotent, so re-running is safe but pointless). + scanBackfill scanBackfillState +} + +// scanBackfillState tracks the in-flight scan-status backfill run. +type scanBackfillState struct { + mu sync.Mutex + running bool + startedAt time.Time + current *pds.ScanBackfillResult // running totals (snapshot) + result *pds.ScanBackfillResult // final result, set when running=false + err string // last error (running ends with err set) } // adminContextKey is used to store session data in request context @@ -532,6 +547,12 @@ func (ui *AdminUI) RegisterRoutes(r chi.Router) { r.Get("/admin/api/relay/status", ui.handleRelayStatus) r.Get("/admin/api/crew/member", ui.handleCrewMemberInfo) + // Scan-record backfill: kicks off a background run and returns a + // progress fragment that polls /status. Use Accept:application/json + // for a synchronous JSON response (curl-friendly). + r.Post("/admin/api/scan-backfill", ui.handleScanBackfill) + r.Get("/admin/api/scan-backfill/status", ui.handleScanBackfillStatus) + // Logout r.Post("/admin/auth/logout", ui.handleLogout) }) diff --git a/pkg/hold/admin/handlers_scan.go b/pkg/hold/admin/handlers_scan.go new file mode 100644 index 0000000..c06dcb5 --- /dev/null +++ b/pkg/hold/admin/handlers_scan.go @@ -0,0 +1,173 @@ +package admin + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "atcr.io/pkg/hold/pds" +) + +// handleScanBackfill kicks off a scan-status backfill in a background +// goroutine and returns a progress fragment that polls +// /admin/api/scan-backfill/status for updates. Idempotent — clicking again +// while a run is in flight just shows the current progress. +// +// Why background: reverse proxies typically cap upstream HTTP timeouts at +// 10–60s, which would cancel a synchronous request mid-loop. Detaching the +// work from the request context lets it run to completion. +// +// JSON callers (Accept: application/json) get a synchronous run instead — +// useful for curl + scripting. +func (ui *AdminUI) handleScanBackfill(w http.ResponseWriter, r *http.Request) { + session := getSessionFromContext(r.Context()) + wantJSON := strings.Contains(r.Header.Get("Accept"), "application/json") + + if wantJSON { + // Synchronous JSON path — caller decides their own timeout. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + res, err := ui.pds.BackfillScanStatus(ctx, scanBackfillLogger, nil) + if err != nil { + slog.Error("scan-backfill failed", "by", session.DID, "error", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + slog.Info("scan-status backfill complete (sync)", + "by", session.DID, + "scanned", res.Scanned, + "already_tagged", res.AlreadyTagged, + "marked_skipped", res.MarkedSkipped, + "marked_failed", res.MarkedFailed, + "rewritten", res.Rewritten, + ) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(res) + return + } + + // HTML path — kick off the background run if one isn't already going. + st := &ui.scanBackfill + st.mu.Lock() + alreadyRunning := st.running + if !alreadyRunning { + st.running = true + st.startedAt = time.Now() + st.current = &pds.ScanBackfillResult{} + st.result = nil + st.err = "" + } + st.mu.Unlock() + + if !alreadyRunning { + slog.Info("scan-status backfill started via admin panel", "by", session.DID) + go ui.runScanBackfill() + } else { + slog.Debug("scan-status backfill already in progress; returning current state") + } + + ui.renderTemplate(w, "partials/scan_backfill_progress.html", ui.snapshotScanBackfill()) +} + +// handleScanBackfillStatus is polled by the progress fragment. Returns the +// progress fragment again if running, the result fragment when done, or an +// error fragment if something went wrong. +func (ui *AdminUI) handleScanBackfillStatus(w http.ResponseWriter, r *http.Request) { + snap := ui.snapshotScanBackfill() + if snap.Running { + ui.renderTemplate(w, "partials/scan_backfill_progress.html", snap) + return + } + if snap.Error != "" { + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{snap.Error}) + return + } + if snap.Result == nil { + // Initial state, before any run — render an empty placeholder. + _, _ = w.Write([]byte("")) + return + } + ui.renderTemplate(w, "partials/scan_backfill_result.html", snap.Result) +} + +// runScanBackfill is the goroutine body. Updates the shared state as the +// backfill progresses and stores the final result (or error) when it ends. +func (ui *AdminUI) runScanBackfill() { + st := &ui.scanBackfill + defer func() { + st.mu.Lock() + st.running = false + st.mu.Unlock() + }() + + // Generous independent timeout — the loop is single-threaded and large + // holds with thousands of legacy records can take a few minutes. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + res, err := ui.pds.BackfillScanStatus(ctx, scanBackfillLogger, func(snap *pds.ScanBackfillResult) { + // Copy so we don't keep a pointer the loop will mutate. + c := *snap + st.mu.Lock() + st.current = &c + st.mu.Unlock() + }) + st.mu.Lock() + if err != nil { + st.err = err.Error() + slog.Error("scan-status backfill failed", "error", err) + } else { + st.result = res + slog.Info("scan-status backfill complete", + "scanned", res.Scanned, + "already_tagged", res.AlreadyTagged, + "marked_skipped", res.MarkedSkipped, + "marked_failed", res.MarkedFailed, + "rewritten", res.Rewritten, + ) + } + st.mu.Unlock() +} + +// scanBackfillSnapshot is the shape exposed to templates. +type scanBackfillSnapshot struct { + Running bool + StartedAt time.Time + Current *pds.ScanBackfillResult // populated while running + Result *pds.ScanBackfillResult // populated when complete + Error string +} + +// snapshotScanBackfill returns a copy of the current state — safe to render +// without holding the mutex. +func (ui *AdminUI) snapshotScanBackfill() scanBackfillSnapshot { + st := &ui.scanBackfill + st.mu.Lock() + defer st.mu.Unlock() + + snap := scanBackfillSnapshot{ + Running: st.running, + StartedAt: st.startedAt, + Error: st.err, + } + if st.current != nil { + c := *st.current + snap.Current = &c + } + if st.result != nil { + r := *st.result + snap.Result = &r + } + return snap +} + +// scanBackfillLogger formats the printf-style messages from BackfillScanStatus +// into a single slog message — slog's variadic args are key/value pairs, not +// printf operands. +func scanBackfillLogger(format string, args ...any) { + slog.Warn("scan-backfill: " + fmt.Sprintf(format, args...)) +} diff --git a/pkg/hold/admin/templates/partials/scan_backfill_progress.html b/pkg/hold/admin/templates/partials/scan_backfill_progress.html new file mode 100644 index 0000000..edcf72d --- /dev/null +++ b/pkg/hold/admin/templates/partials/scan_backfill_progress.html @@ -0,0 +1,20 @@ +{{define "partials/scan_backfill_progress.html"}} +
+ +
+

Backfilling scan records...

+ {{ if .Current }} +

+ Scanned {{ .Current.Scanned }} records · + rewrites: {{ .Current.Rewritten }} + ({{ .Current.MarkedSkipped }} skipped, {{ .Current.MarkedFailed }} failed) +

+ {{ else }} +

Starting...

+ {{ end }} +
+
+{{end}} diff --git a/pkg/hold/admin/templates/partials/scan_backfill_result.html b/pkg/hold/admin/templates/partials/scan_backfill_result.html new file mode 100644 index 0000000..73dcd65 --- /dev/null +++ b/pkg/hold/admin/templates/partials/scan_backfill_result.html @@ -0,0 +1,15 @@ +{{define "partials/scan_backfill_result.html"}} +
+ {{ icon "check-circle" "size-4 shrink-0" }} +
+

Scan-record backfill complete

+
    +
  • Scanned: {{ .Scanned }}
    • +
  • Already tagged: {{ .AlreadyTagged }}
    • +
  • → Skipped (helm / in-toto / DSSE): {{ .MarkedSkipped }}
    • +
  • → Failed (legacy errors): {{ .MarkedFailed }}
    • +
  • Total rewritten: {{ .Rewritten }}
    • +
+
+
+{{end}} diff --git a/pkg/hold/admin/templates/partials/tab_storage.html b/pkg/hold/admin/templates/partials/tab_storage.html index 7ace056..07b132f 100644 --- a/pkg/hold/admin/templates/partials/tab_storage.html +++ b/pkg/hold/admin/templates/partials/tab_storage.html @@ -67,4 +67,31 @@
{{end}} + + +
+
+

Scan records

+

+ Rewrite legacy scan records (created before the status field + existed) so the appview can distinguish intentionally skipped artifacts (helm charts, + in-toto, DSSE) from genuine failures. Idempotent — safe to re-run. +

+
+ + +
+
+
+
{{end}} diff --git a/pkg/hold/pds/scan.go b/pkg/hold/pds/scan.go index 77c464f..e3008c2 100644 --- a/pkg/hold/pds/scan.go +++ b/pkg/hold/pds/scan.go @@ -3,11 +3,162 @@ package pds import ( "context" "fmt" + "strings" "atcr.io/pkg/atproto" "github.com/ipfs/go-cid" ) +// unscannableLayerMediaSubstrings lists layer media-type fragments that +// identify artifact types the scanner intentionally skips. Kept in sync with +// scanner/internal/scan/worker.go's unscannableConfigTypes — that map keys on +// config media types; here we look at *layer* media types because the +// backfill walks the hold's layer index (manifest AT-URIs join layers to +// scan records). +var unscannableLayerMediaSubstrings = []string{ + "helm.chart.content", + "in-toto", + "dsse.envelope", +} + +// ScanBackfillResult summarizes what BackfillScanStatus did. +type ScanBackfillResult struct { + Scanned int // total scan records examined + AlreadyTagged int // records with non-empty status (or non-failure shape) — left alone + MarkedSkipped int // records rewritten as status=skipped + MarkedFailed int // records rewritten as status=failed + Rewritten int // total rewrites (MarkedSkipped + MarkedFailed) that succeeded +} + +// BackfillScanStatus walks every io.atcr.hold.scan record on this hold and +// assigns a status ("skipped" or "failed") to legacy records that pre-date the +// status field. Idempotent — records that already have a non-empty status are +// left alone. +// +// A legacy record is one with empty status, no SBOM blob, and zero +// vulnerability counts. Layer media types are inspected to choose the +// rewrite: helm/in-toto/DSSE → skipped, otherwise → failed. ScannedAt is +// preserved on rewrite so the rescan timer doesn't reset. +// +// Safe to call on a running hold — uses the existing repomgr APIs the same +// way scan-job handling does. +// +// progress, if non-nil, is called periodically with a snapshot of the +// running totals so callers can surface progress to the UI. The callback +// must not retain the pointer past its return. +func (p *HoldPDS) BackfillScanStatus(ctx context.Context, log func(format string, args ...any), progress func(*ScanBackfillResult)) (*ScanBackfillResult, error) { + if log == nil { + log = func(string, ...any) {} + } + if progress == nil { + progress = func(*ScanBackfillResult) {} + } + ri := p.RecordsIndex() + if ri == nil { + return nil, fmt.Errorf("records index not available") + } + + const batchSize = 200 + res := &ScanBackfillResult{} + var cursor string + + for { + records, nextCursor, err := ri.ListRecords(atproto.ScanCollection, batchSize, cursor, true) + if err != nil { + return res, fmt.Errorf("list scan records: %w", err) + } + + for _, rec := range records { + if err := ctx.Err(); err != nil { + return res, err + } + res.Scanned++ + manifestDigest := "sha256:" + rec.Rkey + + _, scanRecord, err := p.GetScanRecord(ctx, manifestDigest) + if err != nil { + log("skip rkey=%s: get failed: %v", rec.Rkey, err) + continue + } + + // Already classified — nothing to do. + if scanRecord.Status != "" { + res.AlreadyTagged++ + continue + } + // Real data present (legacy successful scan) — treat as ok, don't rewrite. + if scanRecord.SbomBlob != nil || scanRecord.Total != 0 { + res.AlreadyTagged++ + continue + } + + // Determine artifact type from layer media types. + layers, err := p.ListLayerRecordsForManifest(ctx, scanRecord.Manifest) + if err != nil { + log("skip rkey=%s: list layers failed: %v", rec.Rkey, err) + continue + } + + skipped := false + for _, l := range layers { + for _, frag := range unscannableLayerMediaSubstrings { + if strings.Contains(l.MediaType, frag) { + skipped = true + break + } + } + if skipped { + break + } + } + + var rewrite *atproto.ScanRecord + if skipped { + rewrite = atproto.NewSkippedScanRecord( + manifestDigest, + scanRecord.Repository, + scanRecord.UserDID, + "backfilled: unscannable artifact type", + scanRecord.ScannerVersion, + ) + res.MarkedSkipped++ + } else { + rewrite = atproto.NewFailedScanRecord( + manifestDigest, + scanRecord.Repository, + scanRecord.UserDID, + "backfilled: legacy record (no SBOM and zero counts)", + scanRecord.ScannerVersion, + ) + res.MarkedFailed++ + } + // Preserve original ScannedAt so the rescan timer doesn't reset + // and we don't lose audit signal. + if scanRecord.ScannedAt != "" { + rewrite.ScannedAt = scanRecord.ScannedAt + } + + if _, _, err := p.CreateScanRecord(ctx, rewrite); err != nil { + log("rewrite rkey=%s failed: %v", rec.Rkey, err) + continue + } + res.Rewritten++ + } + + // Report progress at every batch boundary — gives the UI smooth-ish + // updates without locking the loop on every record. + progress(res) + + if nextCursor == "" || len(records) == 0 { + break + } + cursor = nextCursor + } + + progress(res) + return res, nil +} + // CreateScanRecord creates or updates a scan result record in the hold's PDS // Uses a deterministic rkey based on the manifest digest, so re-scans upsert func (p *HoldPDS) CreateScanRecord(ctx context.Context, record *atproto.ScanRecord) (string, cid.Cid, error) {