mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-26 04:04:15 +00:00
The stale-scan loop rescans an image on a schedule and handleResult uploaded both artifacts every time, so each pass wrote a fresh SBOM blob and orphaned the one before it. Nothing reclaims those, because GC walks only the OCI blob prefix and scan artifacts live under /repos. Keep the stored blobs when the record already on file was written by this same scanner version and carries a vulnerability-report reference equal to the one this report would be stored under, then keep each blob individually and only if it is still in S3. The report reference alone is not quite enough evidence on its own: the report lists matched packages, not every package, so an SBOM could gain a package with no known CVEs and leave the report byte-identical. Content is pinned by the manifest digest, so that can only happen across a scanner or Syft change, which is what the version clause closes. It is inert until that constant moves, and one comparison is cheaper than remembering to add it at the moment it first matters. Every failure path declines to reuse, which costs an upload and never costs correctness, including the reuse check's own timeout, which is carved out of the upload budget rather than given its own so a slow read cannot eat the deadline it stands in for. A record whose earlier upload failed heals on the next rescan, keeping the report and rewriting only the missing SBOM. scannedAt still advances. runStalePass selects on it, so freezing it would make every deduplicated record permanently stale and rescanned forever, which costs far more than the bytes saved. A scanner running with vulnerability scanning off sends no report, so there is no stable digest to compare and its SBOM still churns. Closing that means either parsing SPDX in the hold to normalise a timestamp and a UUID, or a lexicon change; a test pins the gap rather than leaving it to be rediscovered. Note for anyone extending GC to /repos later: a live SBOM object used to be rewritten on every rescan and so was always young. It now keeps its original mtime for the life of the content, so an age-based rule there would delete referenced blobs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
230 lines
7.6 KiB
Go
230 lines
7.6 KiB
Go
package pds
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"time"
|
|
|
|
"atcr.io/pkg/atproto"
|
|
"atcr.io/pkg/s3"
|
|
bsky "github.com/bluesky-social/indigo/api/bsky"
|
|
lexutil "github.com/bluesky-social/indigo/lex/util"
|
|
"github.com/ipfs/go-cid"
|
|
"github.com/multiformats/go-multihash"
|
|
)
|
|
|
|
const (
|
|
// ProfileRkey is the fixed rkey for the profile record (singleton)
|
|
ProfileRkey = "self"
|
|
|
|
// ProfileCollection is the collection name for Bluesky actor profiles
|
|
ProfileCollection = "app.bsky.actor.profile"
|
|
|
|
// TangledProfileRkey is the fixed rkey for the tangled profile record (singleton)
|
|
TangledProfileRkey = "self"
|
|
|
|
// TangledProfileCollection is the collection name for Tangled actor profiles
|
|
TangledProfileCollection = "sh.tangled.actor.profile"
|
|
)
|
|
|
|
// downloadImage downloads an image from a URL and returns the data and content type
|
|
func downloadImage(ctx context.Context, url string) ([]byte, string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
client := &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("failed to download image: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, "", fmt.Errorf("download failed with status %d", resp.StatusCode)
|
|
}
|
|
|
|
// Read image data
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("failed to read image data: %w", err)
|
|
}
|
|
|
|
// Get content type from response header
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
|
|
return data, contentType, nil
|
|
}
|
|
|
|
// blobRefForBytes computes the blob reference these bytes would be stored
|
|
// under, without touching S3. Blob storage is content-addressed, so this is
|
|
// also the answer to "is this payload already uploaded" — see
|
|
// reusableScanBlobs, which compares a fresh scan result against the reference
|
|
// on the record already on file. It is split out of uploadBlobToStorage rather
|
|
// than reimplemented so the two can never disagree about the CID.
|
|
func blobRefForBytes(data []byte, mimeType string) (*lexutil.LexBlob, error) {
|
|
if len(data) == 0 {
|
|
return nil, fmt.Errorf("empty blob data")
|
|
}
|
|
|
|
// Compute SHA-256 hash
|
|
hash := sha256.Sum256(data)
|
|
|
|
// Create CIDv1 with SHA-256 multihash
|
|
mh, err := multihash.EncodeName(hash[:], "sha2-256")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to encode multihash: %w", err)
|
|
}
|
|
|
|
// Create CIDv1 with raw codec (0x55)
|
|
// ATProto uses CIDv1 with raw codec for blobs
|
|
blobCID := cid.NewCidV1(0x55, mh)
|
|
|
|
// Create blob reference in the format expected by bsky.ActorProfile
|
|
return &lexutil.LexBlob{
|
|
Ref: lexutil.LexLink(blobCID),
|
|
MimeType: mimeType,
|
|
Size: int64(len(data)),
|
|
}, nil
|
|
}
|
|
|
|
// uploadBlobToStorage uploads a blob to the hold's S3 storage and returns a blob reference.
|
|
// This stores the blob at the ATProto path for the hold's DID.
|
|
func uploadBlobToStorage(ctx context.Context, s3svc *s3.S3Service, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
|
|
blob, err := blobRefForBytes(data, mimeType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Store blob via S3 at ATProto path
|
|
path := atprotoBlobPath(did, blob.Ref.String())
|
|
|
|
if err := s3svc.PutBytes(ctx, path, data, mimeType); err != nil {
|
|
return nil, fmt.Errorf("failed to put blob: %w", err)
|
|
}
|
|
|
|
return blob, nil
|
|
}
|
|
|
|
// CreateProfileRecord creates the app.bsky.actor.profile record for the hold
|
|
// This will FAIL if the profile record already exists.
|
|
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, s3svc *s3.S3Service, displayName, description, avatarURL string) (cid.Cid, error) {
|
|
// Create profile struct
|
|
profile := &bsky.ActorProfile{
|
|
DisplayName: &displayName,
|
|
Description: &description,
|
|
}
|
|
|
|
// Download and upload avatar if URL is provided
|
|
if avatarURL != "" {
|
|
slog.Debug("Downloading avatar", "url", avatarURL)
|
|
imageData, mimeType, err := downloadImage(ctx, avatarURL)
|
|
if err != nil {
|
|
return cid.Undef, fmt.Errorf("failed to download avatar: %w", err)
|
|
}
|
|
|
|
slog.Debug("Uploading avatar blob",
|
|
"size", len(imageData),
|
|
"mimeType", mimeType)
|
|
avatarBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, imageData, mimeType)
|
|
if err != nil {
|
|
return cid.Undef, fmt.Errorf("failed to upload avatar blob: %w", err)
|
|
}
|
|
|
|
profile.Avatar = avatarBlob
|
|
slog.Info("Avatar uploaded successfully", "ref", avatarBlob.Ref.String())
|
|
}
|
|
|
|
// Use repomgr.PutRecord - creates with explicit rkey, fails if already exists
|
|
recordPath, recordCID, err := p.repomgr.PutRecord(ctx, p.uid, ProfileCollection, ProfileRkey, profile)
|
|
if err != nil {
|
|
return cid.Undef, fmt.Errorf("failed to create profile record: %w", err)
|
|
}
|
|
|
|
slog.Info("Created profile record",
|
|
"path", recordPath,
|
|
"cid", recordCID.String())
|
|
return recordCID, nil
|
|
}
|
|
|
|
// UpdateProfileRecord updates the existing app.bsky.actor.profile record.
|
|
// Callers should GetProfileRecord first, modify fields, then pass the updated record.
|
|
func (p *HoldPDS) UpdateProfileRecord(ctx context.Context, record *bsky.ActorProfile) (cid.Cid, error) {
|
|
recordCID, err := p.repomgr.UpdateRecord(ctx, p.uid, ProfileCollection, ProfileRkey, record)
|
|
if err != nil {
|
|
return cid.Undef, fmt.Errorf("failed to update profile record: %w", err)
|
|
}
|
|
return recordCID, nil
|
|
}
|
|
|
|
// GetProfileRecord retrieves the app.bsky.actor.profile record
|
|
func (p *HoldPDS) GetProfileRecord(ctx context.Context) (cid.Cid, *bsky.ActorProfile, error) {
|
|
// Use repomgr.GetRecord
|
|
recordCID, val, err := p.repomgr.GetRecord(ctx, p.uid, ProfileCollection, ProfileRkey, cid.Undef)
|
|
if err != nil {
|
|
return cid.Undef, nil, fmt.Errorf("failed to get profile record: %w", err)
|
|
}
|
|
|
|
// Type assert to bsky.ActorProfile
|
|
profileRecord, ok := val.(*bsky.ActorProfile)
|
|
if !ok {
|
|
return cid.Undef, nil, fmt.Errorf("unexpected type for profile record: %T", val)
|
|
}
|
|
|
|
return recordCID, profileRecord, nil
|
|
}
|
|
|
|
// CreateTangledProfileRecord creates the sh.tangled.actor.profile record for the hold
|
|
// This will FAIL if the tangled profile record already exists.
|
|
func (p *HoldPDS) CreateTangledProfileRecord(ctx context.Context, links []string, description string) (cid.Cid, error) {
|
|
// Create tangled profile struct
|
|
profile := &atproto.TangledProfileRecord{
|
|
Type: atproto.TangledProfileCollection,
|
|
Links: links,
|
|
Stats: []string{}, // Empty for now
|
|
Bluesky: true,
|
|
Location: "",
|
|
Description: description,
|
|
PinnedRepositories: []string{}, // Empty for now
|
|
}
|
|
|
|
// Use repomgr.PutRecord - creates with explicit rkey, fails if already exists
|
|
recordPath, recordCID, err := p.repomgr.PutRecord(ctx, p.uid, TangledProfileCollection, TangledProfileRkey, profile)
|
|
if err != nil {
|
|
return cid.Undef, fmt.Errorf("failed to create tangled profile record: %w", err)
|
|
}
|
|
|
|
slog.Info("Created tangled profile record",
|
|
"path", recordPath,
|
|
"cid", recordCID.String())
|
|
return recordCID, nil
|
|
}
|
|
|
|
// GetTangledProfileRecord retrieves the sh.tangled.actor.profile record
|
|
func (p *HoldPDS) GetTangledProfileRecord(ctx context.Context) (cid.Cid, *atproto.TangledProfileRecord, error) {
|
|
// Use repomgr.GetRecord
|
|
recordCID, val, err := p.repomgr.GetRecord(ctx, p.uid, TangledProfileCollection, TangledProfileRkey, cid.Undef)
|
|
if err != nil {
|
|
return cid.Undef, nil, fmt.Errorf("failed to get tangled profile record: %w", err)
|
|
}
|
|
|
|
// Type assert to TangledProfileRecord
|
|
profileRecord, ok := val.(*atproto.TangledProfileRecord)
|
|
if !ok {
|
|
return cid.Undef, nil, fmt.Errorf("unexpected type for tangled profile record: %T", val)
|
|
}
|
|
|
|
return recordCID, profileRecord, nil
|
|
}
|