Files
at-container-registry/pkg/hold/pds/profile.go
T
Evan JarrettandClaude Opus 5 61deadc124 hold: reuse stored scan blobs when a rescan finds nothing new
The stale-scan loop rescans an image on a schedule and handleResult uploaded
both artifacts every time, so each pass wrote a fresh SBOM blob and orphaned
the one before it. Nothing reclaims those, because GC walks only the OCI blob
prefix and scan artifacts live under /repos.

Keep the stored blobs when the record already on file was written by this same
scanner version and carries a vulnerability-report reference equal to the one
this report would be stored under, then keep each blob individually and only if
it is still in S3.

The report reference alone is not quite enough evidence on its own: the report
lists matched packages, not every package, so an SBOM could gain a package with
no known CVEs and leave the report byte-identical. Content is pinned by the
manifest digest, so that can only happen across a scanner or Syft change, which
is what the version clause closes. It is inert until that constant moves, and
one comparison is cheaper than remembering to add it at the moment it first
matters.

Every failure path declines to reuse, which costs an upload and never costs
correctness, including the reuse check's own timeout, which is carved out of
the upload budget rather than given its own so a slow read cannot eat the
deadline it stands in for. A record whose earlier upload failed heals on the
next rescan, keeping the report and rewriting only the missing SBOM.

scannedAt still advances. runStalePass selects on it, so freezing it would make
every deduplicated record permanently stale and rescanned forever, which costs
far more than the bytes saved.

A scanner running with vulnerability scanning off sends no report, so there is
no stable digest to compare and its SBOM still churns. Closing that means
either parsing SPDX in the hold to normalise a timestamp and a UUID, or a
lexicon change; a test pins the gap rather than leaving it to be rediscovered.

Note for anyone extending GC to /repos later: a live SBOM object used to be
rewritten on every rescan and so was always young. It now keeps its original
mtime for the life of the content, so an age-based rule there would delete
referenced blobs.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
2026-09-05 15:53:57 -05:00

230 lines
7.6 KiB
Go

package pds
import (
"context"
"crypto/sha256"
"fmt"
"io"
"log/slog"
"net/http"
"time"
"atcr.io/pkg/atproto"
"atcr.io/pkg/s3"
bsky "github.com/bluesky-social/indigo/api/bsky"
lexutil "github.com/bluesky-social/indigo/lex/util"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
const (
// ProfileRkey is the fixed rkey for the profile record (singleton)
ProfileRkey = "self"
// ProfileCollection is the collection name for Bluesky actor profiles
ProfileCollection = "app.bsky.actor.profile"
// TangledProfileRkey is the fixed rkey for the tangled profile record (singleton)
TangledProfileRkey = "self"
// TangledProfileCollection is the collection name for Tangled actor profiles
TangledProfileCollection = "sh.tangled.actor.profile"
)
// downloadImage downloads an image from a URL and returns the data and content type
func downloadImage(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, "", fmt.Errorf("failed to create request: %w", err)
}
client := &http.Client{
Timeout: 30 * time.Second,
}
resp, err := client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("failed to download image: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("download failed with status %d", resp.StatusCode)
}
// Read image data
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("failed to read image data: %w", err)
}
// Get content type from response header
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
}
return data, contentType, nil
}
// blobRefForBytes computes the blob reference these bytes would be stored
// under, without touching S3. Blob storage is content-addressed, so this is
// also the answer to "is this payload already uploaded" — see
// reusableScanBlobs, which compares a fresh scan result against the reference
// on the record already on file. It is split out of uploadBlobToStorage rather
// than reimplemented so the two can never disagree about the CID.
func blobRefForBytes(data []byte, mimeType string) (*lexutil.LexBlob, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty blob data")
}
// Compute SHA-256 hash
hash := sha256.Sum256(data)
// Create CIDv1 with SHA-256 multihash
mh, err := multihash.EncodeName(hash[:], "sha2-256")
if err != nil {
return nil, fmt.Errorf("failed to encode multihash: %w", err)
}
// Create CIDv1 with raw codec (0x55)
// ATProto uses CIDv1 with raw codec for blobs
blobCID := cid.NewCidV1(0x55, mh)
// Create blob reference in the format expected by bsky.ActorProfile
return &lexutil.LexBlob{
Ref: lexutil.LexLink(blobCID),
MimeType: mimeType,
Size: int64(len(data)),
}, nil
}
// uploadBlobToStorage uploads a blob to the hold's S3 storage and returns a blob reference.
// This stores the blob at the ATProto path for the hold's DID.
func uploadBlobToStorage(ctx context.Context, s3svc *s3.S3Service, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
blob, err := blobRefForBytes(data, mimeType)
if err != nil {
return nil, err
}
// Store blob via S3 at ATProto path
path := atprotoBlobPath(did, blob.Ref.String())
if err := s3svc.PutBytes(ctx, path, data, mimeType); err != nil {
return nil, fmt.Errorf("failed to put blob: %w", err)
}
return blob, nil
}
// CreateProfileRecord creates the app.bsky.actor.profile record for the hold
// This will FAIL if the profile record already exists.
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, s3svc *s3.S3Service, displayName, description, avatarURL string) (cid.Cid, error) {
// Create profile struct
profile := &bsky.ActorProfile{
DisplayName: &displayName,
Description: &description,
}
// Download and upload avatar if URL is provided
if avatarURL != "" {
slog.Debug("Downloading avatar", "url", avatarURL)
imageData, mimeType, err := downloadImage(ctx, avatarURL)
if err != nil {
return cid.Undef, fmt.Errorf("failed to download avatar: %w", err)
}
slog.Debug("Uploading avatar blob",
"size", len(imageData),
"mimeType", mimeType)
avatarBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, imageData, mimeType)
if err != nil {
return cid.Undef, fmt.Errorf("failed to upload avatar blob: %w", err)
}
profile.Avatar = avatarBlob
slog.Info("Avatar uploaded successfully", "ref", avatarBlob.Ref.String())
}
// Use repomgr.PutRecord - creates with explicit rkey, fails if already exists
recordPath, recordCID, err := p.repomgr.PutRecord(ctx, p.uid, ProfileCollection, ProfileRkey, profile)
if err != nil {
return cid.Undef, fmt.Errorf("failed to create profile record: %w", err)
}
slog.Info("Created profile record",
"path", recordPath,
"cid", recordCID.String())
return recordCID, nil
}
// UpdateProfileRecord updates the existing app.bsky.actor.profile record.
// Callers should GetProfileRecord first, modify fields, then pass the updated record.
func (p *HoldPDS) UpdateProfileRecord(ctx context.Context, record *bsky.ActorProfile) (cid.Cid, error) {
recordCID, err := p.repomgr.UpdateRecord(ctx, p.uid, ProfileCollection, ProfileRkey, record)
if err != nil {
return cid.Undef, fmt.Errorf("failed to update profile record: %w", err)
}
return recordCID, nil
}
// GetProfileRecord retrieves the app.bsky.actor.profile record
func (p *HoldPDS) GetProfileRecord(ctx context.Context) (cid.Cid, *bsky.ActorProfile, error) {
// Use repomgr.GetRecord
recordCID, val, err := p.repomgr.GetRecord(ctx, p.uid, ProfileCollection, ProfileRkey, cid.Undef)
if err != nil {
return cid.Undef, nil, fmt.Errorf("failed to get profile record: %w", err)
}
// Type assert to bsky.ActorProfile
profileRecord, ok := val.(*bsky.ActorProfile)
if !ok {
return cid.Undef, nil, fmt.Errorf("unexpected type for profile record: %T", val)
}
return recordCID, profileRecord, nil
}
// CreateTangledProfileRecord creates the sh.tangled.actor.profile record for the hold
// This will FAIL if the tangled profile record already exists.
func (p *HoldPDS) CreateTangledProfileRecord(ctx context.Context, links []string, description string) (cid.Cid, error) {
// Create tangled profile struct
profile := &atproto.TangledProfileRecord{
Type: atproto.TangledProfileCollection,
Links: links,
Stats: []string{}, // Empty for now
Bluesky: true,
Location: "",
Description: description,
PinnedRepositories: []string{}, // Empty for now
}
// Use repomgr.PutRecord - creates with explicit rkey, fails if already exists
recordPath, recordCID, err := p.repomgr.PutRecord(ctx, p.uid, TangledProfileCollection, TangledProfileRkey, profile)
if err != nil {
return cid.Undef, fmt.Errorf("failed to create tangled profile record: %w", err)
}
slog.Info("Created tangled profile record",
"path", recordPath,
"cid", recordCID.String())
return recordCID, nil
}
// GetTangledProfileRecord retrieves the sh.tangled.actor.profile record
func (p *HoldPDS) GetTangledProfileRecord(ctx context.Context) (cid.Cid, *atproto.TangledProfileRecord, error) {
// Use repomgr.GetRecord
recordCID, val, err := p.repomgr.GetRecord(ctx, p.uid, TangledProfileCollection, TangledProfileRkey, cid.Undef)
if err != nil {
return cid.Undef, nil, fmt.Errorf("failed to get tangled profile record: %w", err)
}
// Type assert to TangledProfileRecord
profileRecord, ok := val.(*atproto.TangledProfileRecord)
if !ok {
return cid.Undef, nil, fmt.Errorf("unexpected type for tangled profile record: %T", val)
}
return recordCID, profileRecord, nil
}