diff --git a/CLAUDE.md b/CLAUDE.md index 2f8802a..f26d98c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,6 +109,7 @@ The hold's embedded PDS stores all operational data as ATProto records in a CAR | `io.atcr.hold.layer` | Per-layer | Layer metadata (digest, size, media type) | | `io.atcr.hold.stats` | Per-repo | Push/pull counts per owner+repository | | `io.atcr.hold.scan` | Per-scan | Vulnerability scan results | +| `io.atcr.hold.image.config` | Per-manifest | OCI image config (history, env, entrypoint, labels) | | `app.bsky.feed.post` | Status posts | Online/offline status, push notifications | | `sh.tangled.actor.profile` | Singleton | Hold profile (name, description, avatar) | diff --git a/docs/HOLD_XRPC_ENDPOINTS.md b/docs/HOLD_XRPC_ENDPOINTS.md index ed82288..77bb8b2 100644 --- a/docs/HOLD_XRPC_ENDPOINTS.md +++ b/docs/HOLD_XRPC_ENDPOINTS.md @@ -80,6 +80,8 @@ All require `blob:write` permission via service token: | `/xrpc/io.atcr.hold.requestCrew` | POST | auth | Request crew membership | | `/xrpc/io.atcr.hold.exportUserData` | GET | auth | GDPR data export | | `/xrpc/io.atcr.hold.getQuota` | GET | none | Get user quota info | +| `/xrpc/io.atcr.hold.getLayersForManifest` | GET | none | Get layer records for a manifest AT-URI | +| `/xrpc/io.atcr.hold.image.getConfig` | GET | none | Get OCI image config record for a manifest digest | | `/xrpc/io.atcr.hold.listTiers` | GET | none | List hold's available tiers with quotas and features (scanOnPush) | | `/xrpc/io.atcr.hold.updateCrewTier` | POST | appview token | Update crew member's tier | diff --git a/pkg/appview/db/delete.go b/pkg/appview/db/delete.go index 47eebe6..0e3e7ff 100644 --- a/pkg/appview/db/delete.go +++ b/pkg/appview/db/delete.go @@ -36,7 +36,7 @@ func DeleteUserDataFull(db DBTX, oauthStore *OAuthStore, did string) error { } // 3. Delete user (cascades to manifests, tags, stars, annotations, etc.) - if err := DeleteUserData(db, did); err != nil { + if _, err := DeleteUserData(db, did); err != nil { slog.Error("Failed to delete user data", "did", did, "error", err) return fmt.Errorf("failed to delete user data: %w", err) } diff --git a/pkg/appview/db/models.go b/pkg/appview/db/models.go index b2f6acd..062024b 100644 --- a/pkg/appview/db/models.go +++ b/pkg/appview/db/models.go @@ -124,12 +124,13 @@ func SetRegistryURL(cards []RepoCardData, registryURL string) { // PlatformInfo represents platform information (OS/Architecture) type PlatformInfo struct { - OS string - Architecture string - Variant string - OSVersion string - Digest string // child platform manifest digest (for manifest lists) - HoldEndpoint string // hold endpoint for this platform manifest + OS string + Architecture string + Variant string + OSVersion string + Digest string // child platform manifest digest (for manifest lists) + HoldEndpoint string // hold endpoint for this platform manifest + CompressedSize int64 // sum of layer sizes (compressed) } // TagWithPlatforms extends Tag with platform information @@ -140,6 +141,7 @@ type TagWithPlatforms struct { IsMultiArch bool HasAttestations bool // true if manifest list contains attestation references ArtifactType string // container-image, helm-chart, unknown + CompressedSize int64 // sum of layer sizes for single-arch tags } // ManifestWithMetadata extends Manifest with tags and platform information @@ -155,6 +157,21 @@ type ManifestWithMetadata struct { // Note: ArtifactType is available via embedded Manifest struct } +// ManifestEntry is a unified view model for the tags tab. +// Every entry is a manifest — labeled by tag name or digest. +type ManifestEntry struct { + Label string // tag name, or digest if untagged + Digest string // manifest digest + IsTagged bool + CreatedAt time.Time + HoldEndpoint string + Platforms []PlatformInfo + IsMultiArch bool + HasAttestations bool + ArtifactType string + CompressedSize int64 // for single-arch +} + // AttestationDetail represents an attestation manifest and its layers type AttestationDetail struct { Digest string diff --git a/pkg/appview/db/queries.go b/pkg/appview/db/queries.go index 9d358ef..01c9b8e 100644 --- a/pkg/appview/db/queries.go +++ b/pkg/appview/db/queries.go @@ -692,12 +692,59 @@ func DeleteTag(db DBTX, did, repository, tag string) error { return err } -// GetTagsWithPlatforms returns all tags for a repository with platform information +// LatestTagInfo holds the most recent tag name and its artifact type. +type LatestTagInfo struct { + Tag string + ArtifactType string +} + +// RepositoryExists checks if any manifests exist for a given repository. +func RepositoryExists(db DBTX, did, repository string) (bool, error) { + var count int + err := db.QueryRow(`SELECT COUNT(*) FROM manifests WHERE did = ? AND repository = ? LIMIT 1`, did, repository).Scan(&count) + if err != nil { + return false, err + } + return count > 0, nil +} + +// GetLatestTag returns the most recently created tag and its artifact type for a repository. +// Returns nil if no tags exist. +func GetLatestTag(db DBTX, did, repository string) (*LatestTagInfo, error) { + var info LatestTagInfo + err := db.QueryRow(` + SELECT t.tag, COALESCE(m.artifact_type, 'container-image') + FROM tags t + JOIN manifests m ON t.digest = m.digest AND t.did = m.did AND t.repository = m.repository + WHERE t.did = ? AND t.repository = ? + ORDER BY t.created_at DESC LIMIT 1 + `, did, repository).Scan(&info.Tag, &info.ArtifactType) + if err != nil { + return nil, nil // no tags is not an error + } + return &info, nil +} + +// CountTags returns the total number of tags for a repository. +func CountTags(db DBTX, did, repository string) (int, error) { + var count int + err := db.QueryRow(`SELECT COUNT(*) FROM tags WHERE did = ? AND repository = ?`, did, repository).Scan(&count) + return count, err +} + +// GetTagsWithPlatforms returns tags for a repository with platform information // Only multi-arch tags (manifest lists) have platform info in manifest_references // Single-arch tags will have empty Platforms slice (platform is obvious for single-arch) // Attestation references (unknown/unknown platforms) are filtered out but tracked via HasAttestations -func GetTagsWithPlatforms(db DBTX, did, repository string) ([]TagWithPlatforms, error) { +func GetTagsWithPlatforms(db DBTX, did, repository string, limit, offset int) ([]TagWithPlatforms, error) { rows, err := db.Query(` + WITH paged_tags AS ( + SELECT id, did, repository, tag, digest, created_at + FROM tags + WHERE did = ? AND repository = ? + ORDER BY created_at DESC + LIMIT ? OFFSET ? + ) SELECT t.id, t.did, @@ -714,14 +761,14 @@ func GetTagsWithPlatforms(db DBTX, did, repository string) ([]TagWithPlatforms, COALESCE(mr.platform_os_version, '') as platform_os_version, COALESCE(mr.is_attestation, 0) as is_attestation, COALESCE(mr.digest, '') as child_digest, - COALESCE(child_m.hold_endpoint, m.hold_endpoint, '') as child_hold_endpoint - FROM tags t + COALESCE(child_m.hold_endpoint, m.hold_endpoint, '') as child_hold_endpoint, + COALESCE((SELECT SUM(l.size) FROM layers l WHERE l.manifest_id = COALESCE(child_m.id, m.id)), 0) as compressed_size + FROM paged_tags t JOIN manifests m ON t.digest = m.digest AND t.did = m.did AND t.repository = m.repository LEFT JOIN manifest_references mr ON m.id = mr.manifest_id LEFT JOIN manifests child_m ON mr.digest = child_m.digest AND child_m.did = t.did AND child_m.repository = t.repository - WHERE t.did = ? AND t.repository = ? ORDER BY t.created_at DESC, mr.reference_index - `, did, repository) + `, did, repository, limit, offset) if err != nil { return nil, err @@ -738,11 +785,12 @@ func GetTagsWithPlatforms(db DBTX, did, repository string) ([]TagWithPlatforms, var platformOS, platformArch, platformVariant, platformOSVersion string var isAttestation bool var childDigest, childHoldEndpoint string + var compressedSize int64 if err := rows.Scan(&t.ID, &t.DID, &t.Repository, &t.Tag, &t.Digest, &t.CreatedAt, &mediaType, &artifactType, &holdEndpoint, &platformOS, &platformArch, &platformVariant, &platformOSVersion, - &isAttestation, &childDigest, &childHoldEndpoint); err != nil { + &isAttestation, &childDigest, &childHoldEndpoint, &compressedSize); err != nil { return nil, err } @@ -750,10 +798,11 @@ func GetTagsWithPlatforms(db DBTX, did, repository string) ([]TagWithPlatforms, tagKey := t.Tag if _, exists := tagMap[tagKey]; !exists { tagMap[tagKey] = &TagWithPlatforms{ - Tag: t, - HoldEndpoint: holdEndpoint, - Platforms: []PlatformInfo{}, - ArtifactType: artifactType, + Tag: t, + HoldEndpoint: holdEndpoint, + Platforms: []PlatformInfo{}, + ArtifactType: artifactType, + CompressedSize: compressedSize, // for single-arch (no manifest_references row) } tagOrder = append(tagOrder, tagKey) } @@ -768,12 +817,13 @@ func GetTagsWithPlatforms(db DBTX, did, repository string) ([]TagWithPlatforms, // Add platform info if present (only for multi-arch manifest lists) if platformOS != "" || platformArch != "" { tagMap[tagKey].Platforms = append(tagMap[tagKey].Platforms, PlatformInfo{ - OS: platformOS, - Architecture: platformArch, - Variant: platformVariant, - OSVersion: platformOSVersion, - Digest: childDigest, - HoldEndpoint: childHoldEndpoint, + OS: platformOS, + Architecture: platformArch, + Variant: platformVariant, + OSVersion: platformOSVersion, + Digest: childDigest, + HoldEndpoint: childHoldEndpoint, + CompressedSize: compressedSize, }) } } @@ -809,19 +859,14 @@ func DeleteManifest(db DBTX, did, repository, digest string) error { // // Due to ON DELETE CASCADE in the schema, deleting from users will automatically // cascade to: manifests, tags, layers, references, annotations, stars, repo_pages, etc. -func DeleteUserData(db DBTX, did string) error { +func DeleteUserData(db DBTX, did string) (bool, error) { result, err := db.Exec(`DELETE FROM users WHERE did = ?`, did) if err != nil { - return fmt.Errorf("failed to delete user: %w", err) + return false, fmt.Errorf("failed to delete user: %w", err) } rowsAffected, _ := result.RowsAffected() - if rowsAffected == 0 { - // User didn't exist, nothing to delete - return nil - } - - return nil + return rowsAffected > 0, nil } // GetManifest fetches a single manifest by digest @@ -1086,15 +1131,19 @@ func GetTopLevelManifests(db DBTX, did, repository string, limit, offset int) ([ if manifests[i].IsManifestList { platformRows, err := db.Query(` SELECT - mr.platform_os, - mr.platform_architecture, - mr.platform_variant, - mr.platform_os_version, - COALESCE(mr.is_attestation, 0) as is_attestation + COALESCE(mr.platform_os, '') as platform_os, + COALESCE(mr.platform_architecture, '') as platform_architecture, + COALESCE(mr.platform_variant, '') as platform_variant, + COALESCE(mr.platform_os_version, '') as platform_os_version, + COALESCE(mr.is_attestation, 0) as is_attestation, + COALESCE(mr.digest, '') as child_digest, + COALESCE(child_m.hold_endpoint, '') as child_hold_endpoint, + COALESCE((SELECT SUM(l.size) FROM layers l WHERE l.manifest_id = child_m.id), 0) as compressed_size FROM manifest_references mr + LEFT JOIN manifests child_m ON mr.digest = child_m.digest AND child_m.did = ? AND child_m.repository = ? WHERE mr.manifest_id = ? ORDER BY mr.reference_index - `, manifests[i].ID) + `, manifests[i].DID, manifests[i].Repository, manifests[i].ID) if err != nil { return nil, err @@ -1103,10 +1152,10 @@ func GetTopLevelManifests(db DBTX, did, repository string, limit, offset int) ([ manifests[i].Platforms = []PlatformInfo{} for platformRows.Next() { var p PlatformInfo - var os, arch, variant, osVersion sql.NullString var isAttestation bool - if err := platformRows.Scan(&os, &arch, &variant, &osVersion, &isAttestation); err != nil { + if err := platformRows.Scan(&p.OS, &p.Architecture, &p.Variant, &p.OSVersion, + &isAttestation, &p.Digest, &p.HoldEndpoint, &p.CompressedSize); err != nil { platformRows.Close() return nil, err } @@ -1114,23 +1163,9 @@ func GetTopLevelManifests(db DBTX, did, repository string, limit, offset int) ([ // Track if manifest list has attestations if isAttestation { manifests[i].HasAttestations = true - // Skip attestation references in platform display continue } - if os.Valid { - p.OS = os.String - } - if arch.Valid { - p.Architecture = arch.String - } - if variant.Valid { - p.Variant = variant.String - } - if osVersion.Valid { - p.OSVersion = osVersion.String - } - manifests[i].Platforms = append(manifests[i].Platforms, p) } platformRows.Close() @@ -1190,19 +1225,23 @@ func GetManifestDetail(db DBTX, did, repository, digest string) (*ManifestWithMe // Determine if manifest list m.IsManifestList = strings.Contains(m.MediaType, "index") || strings.Contains(m.MediaType, "manifest.list") - // If this is a manifest list, get platform details + // If this is a manifest list, get platform details with child digests and sizes if m.IsManifestList { platforms, err := db.Query(` SELECT - mr.platform_os, - mr.platform_architecture, - mr.platform_variant, - mr.platform_os_version, - COALESCE(mr.is_attestation, 0) as is_attestation + COALESCE(mr.platform_os, '') as platform_os, + COALESCE(mr.platform_architecture, '') as platform_architecture, + COALESCE(mr.platform_variant, '') as platform_variant, + COALESCE(mr.platform_os_version, '') as platform_os_version, + COALESCE(mr.is_attestation, 0) as is_attestation, + COALESCE(mr.digest, '') as child_digest, + COALESCE(child_m.hold_endpoint, '') as child_hold_endpoint, + COALESCE((SELECT SUM(l.size) FROM layers l WHERE l.manifest_id = child_m.id), 0) as compressed_size FROM manifest_references mr + LEFT JOIN manifests child_m ON mr.digest = child_m.digest AND child_m.did = ? AND child_m.repository = ? WHERE mr.manifest_id = ? ORDER BY mr.reference_index - `, m.ID) + `, m.DID, m.Repository, m.ID) if err != nil { return nil, err @@ -1212,33 +1251,18 @@ func GetManifestDetail(db DBTX, did, repository, digest string) (*ManifestWithMe m.Platforms = []PlatformInfo{} for platforms.Next() { var p PlatformInfo - var os, arch, variant, osVersion sql.NullString var isAttestation bool - if err := platforms.Scan(&os, &arch, &variant, &osVersion, &isAttestation); err != nil { + if err := platforms.Scan(&p.OS, &p.Architecture, &p.Variant, &p.OSVersion, + &isAttestation, &p.Digest, &p.HoldEndpoint, &p.CompressedSize); err != nil { return nil, err } - // Track if manifest list has attestations if isAttestation { m.HasAttestations = true - // Skip attestation references in platform display continue } - if os.Valid { - p.OS = os.String - } - if arch.Valid { - p.Architecture = arch.String - } - if variant.Valid { - p.Variant = variant.String - } - if osVersion.Valid { - p.OSVersion = osVersion.String - } - m.Platforms = append(m.Platforms, p) } diff --git a/pkg/appview/db/queries_test.go b/pkg/appview/db/queries_test.go index 9abb392..8577b3f 100644 --- a/pkg/appview/db/queries_test.go +++ b/pkg/appview/db/queries_test.go @@ -882,7 +882,7 @@ func TestGetTagsWithPlatforms(t *testing.T) { t.Fatalf("Failed to insert single-arch tag: %v", err) } - tagsWithPlatforms, err := GetTagsWithPlatforms(db, testUser.DID, "myapp") + tagsWithPlatforms, err := GetTagsWithPlatforms(db, testUser.DID, "myapp", 100, 0) if err != nil { t.Fatalf("Failed to get tags with platforms: %v", err) } @@ -951,7 +951,7 @@ func TestGetTagsWithPlatforms(t *testing.T) { t.Fatalf("Failed to insert multi-arch tag: %v", err) } - multiTagsWithPlatforms, err := GetTagsWithPlatforms(db, testUser.DID, "multiapp") + multiTagsWithPlatforms, err := GetTagsWithPlatforms(db, testUser.DID, "multiapp", 100, 0) if err != nil { t.Fatalf("Failed to get multi-arch tags with platforms: %v", err) } @@ -1280,7 +1280,7 @@ func TestDeleteUserData(t *testing.T) { } // Delete user data - if err := DeleteUserData(db, testUser.DID); err != nil { + if _, err := DeleteUserData(db, testUser.DID); err != nil { t.Fatalf("Failed to delete user data: %v", err) } @@ -1303,8 +1303,10 @@ func TestDeleteUserData(t *testing.T) { } // Test idempotency - deleting non-existent user should not error - if err := DeleteUserData(db, testUser.DID); err != nil { + if deleted, err := DeleteUserData(db, testUser.DID); err != nil { t.Errorf("Deleting non-existent user should not error, got: %v", err) + } else if deleted { + t.Errorf("Deleting non-existent user should return false, got true") } } diff --git a/pkg/appview/handlers/repository.go b/pkg/appview/handlers/repository.go index ce12a81..afaab1e 100644 --- a/pkg/appview/handlers/repository.go +++ b/pkg/appview/handlers/repository.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/http" "net/url" + "strconv" "strings" "sync" "time" @@ -50,97 +51,41 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request owner.Handle = resolvedHandle } - // Fetch tags with platform information - tagsWithPlatforms, err := db.GetTagsWithPlatforms(h.ReadOnlyDB, owner.DID, repository) + // Check if repository exists + exists, err := db.RepositoryExists(h.ReadOnlyDB, owner.DID, repository) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - - // Fetch top-level manifests (filters out platform-specific manifests) - manifests, err := db.GetTopLevelManifests(h.ReadOnlyDB, owner.DID, repository, 50, 0) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Check health status for each manifest's hold endpoint (concurrent with 1s timeout) - if h.HealthChecker != nil { - // Create context with 1 second deadline for fast-fail - ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second) - defer cancel() - - var wg sync.WaitGroup - var mu sync.Mutex - - for i := range manifests { - if manifests[i].HoldEndpoint == "" { - // No hold endpoint, mark as unreachable - manifests[i].Reachable = false - manifests[i].Pending = false - continue - } - - wg.Go(func() { - endpoint := manifests[i].HoldEndpoint - - // Try to get cached status first (instant) - if cached := h.HealthChecker.GetCachedStatus(endpoint); cached != nil { - mu.Lock() - manifests[i].Reachable = cached.Reachable - manifests[i].Pending = false - mu.Unlock() - return - } - - // Perform health check with timeout context - reachable, err := h.HealthChecker.CheckHealth(ctx, endpoint) - - mu.Lock() - if ctx.Err() == context.DeadlineExceeded { - // Timeout - mark as pending for HTMX polling - manifests[i].Reachable = false - manifests[i].Pending = true - } else if err != nil { - // Error - mark as unreachable - manifests[i].Reachable = false - manifests[i].Pending = false - } else { - // Success - manifests[i].Reachable = reachable - manifests[i].Pending = false - } - mu.Unlock() - }) - } - - // Wait for all checks to complete or timeout - wg.Wait() - } else { - // If no health checker, assume all are reachable (backward compatibility) - for i := range manifests { - manifests[i].Reachable = true - manifests[i].Pending = false - } - } - - if len(tagsWithPlatforms) == 0 && len(manifests) == 0 { + if !exists { RenderNotFound(w, r, &h.BaseUIHandler) return } + // Fetch latest tag for pull command + latestTag, err := db.GetLatestTag(h.ReadOnlyDB, owner.DID, repository) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Determine artifact type from latest tag + artifactType := "container-image" + latestTagName := "" + if latestTag != nil { + latestTagName = latestTag.Tag + artifactType = latestTag.ArtifactType + } + // Create repository summary repo := &db.Repository{ - Name: repository, - TagCount: len(tagsWithPlatforms), - ManifestCount: len(manifests), + Name: repository, } // Fetch repository metadata from annotations table metadata, err := db.GetRepositoryMetadata(h.ReadOnlyDB, owner.DID, repository) if err != nil { slog.Warn("Failed to fetch repository metadata", "error", err) - // Continue without metadata on error } else { repo.Title = metadata["org.opencontainers.image.title"] repo.Description = metadata["org.opencontainers.image.description"] @@ -156,7 +101,6 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request stats, err := db.GetRepositoryStats(h.ReadOnlyDB, owner.DID, repository) if err != nil { slog.Warn("Failed to fetch repository stats", "error", err) - // Continue with zero stats on error stats = &db.RepositoryStats{StarCount: 0} } @@ -164,10 +108,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request isStarred := false user := middleware.GetUser(r) if user != nil && h.Refresher != nil && h.Directory != nil { - // Create ATProto client with session provider (uses DoWithSession for DPoP nonce safety) pdsClient := atproto.NewClientWithSessionProvider(user.PDSEndpoint, user.DID, h.Refresher) - - // Check if star record exists rkey := atproto.StarRecordKey(owner.DID, repository) _, err := pdsClient.GetRecord(r.Context(), atproto.StarCollection, rkey) isStarred = (err == nil) @@ -182,14 +123,11 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request // Fetch README content from repo page record or annotations var readmeHTML template.HTML - // Try repo page record from database (synced from PDS via Jetstream) repoPage, err := db.GetRepoPage(h.ReadOnlyDB, owner.DID, repository) if err == nil && repoPage != nil { - // Use repo page avatar if present if repoPage.AvatarCID != "" { repo.IconURL = atproto.BlobCDNURL(owner.DID, repoPage.AvatarCID) } - // Render description as markdown if present if repoPage.Description != "" && h.ReadmeFetcher != nil { html, err := h.ReadmeFetcher.RenderMarkdown([]byte(repoPage.Description)) if err != nil { @@ -199,12 +137,9 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request } } } - // Fall back to fetching README from URL annotations if no description in repo page if readmeHTML == "" && h.ReadmeFetcher != nil { - // Fall back to fetching from URL annotations readmeURL := repo.ReadmeURL if readmeURL == "" && repo.SourceURL != "" { - // Try to derive README URL from source URL readmeURL = readme.DeriveReadmeURL(repo.SourceURL, "main") if readmeURL == "" { readmeURL = readme.DeriveReadmeURL(repo.SourceURL, "master") @@ -220,44 +155,6 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request } } - // Determine artifact type for header section from first tag - // This is used for the "Pull this image/chart" header command - artifactType := "container-image" - if len(tagsWithPlatforms) > 0 { - artifactType = tagsWithPlatforms[0].ArtifactType - } else if len(manifests) > 0 { - // Fallback to manifests if no tags - artifactType = manifests[0].ArtifactType - } - - // Collect digests for batch scan-result requests, grouped by hold endpoint - holdDigests := make(map[string][]string) // holdEndpoint → []hexDigest - seen := make(map[string]bool) // dedup digests - for _, t := range tagsWithPlatforms { - if len(t.Platforms) > 0 { - // Multi-arch: collect each platform's child digest - for _, p := range t.Platforms { - if p.Digest != "" && p.HoldEndpoint != "" && !seen[p.Digest] { - seen[p.Digest] = true - hex := strings.TrimPrefix(p.Digest, "sha256:") - holdDigests[p.HoldEndpoint] = append(holdDigests[p.HoldEndpoint], hex) - } - } - } else if t.HoldEndpoint != "" { - // Single-arch: use tag's own digest - if !seen[t.Digest] { - seen[t.Digest] = true - hex := strings.TrimPrefix(t.Digest, "sha256:") - holdDigests[t.HoldEndpoint] = append(holdDigests[t.HoldEndpoint], hex) - } - } - } - var scanBatchParams []template.HTML - for hold, digests := range holdDigests { - scanBatchParams = append(scanBatchParams, template.HTML( - "holdEndpoint="+url.QueryEscape(hold)+"&digests="+strings.Join(digests, ","))) - } - // Build page meta title := owner.Handle + "/" + repository + " - " + h.ClientShortName if repo.Title != "" { @@ -284,32 +181,28 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request data := struct { PageData - Meta *PageMeta - Owner *db.User // Repository owner - Repository *db.Repository // Repository summary - Tags []db.TagWithPlatforms // Tags with platform info - Manifests []db.ManifestWithMetadata // Top-level manifests only - StarCount int - PullCount int - IsStarred bool - IsOwner bool // Whether current user owns this repository - ReadmeHTML template.HTML - ArtifactType string // Dominant artifact type: container-image, helm-chart, unknown - ScanBatchParams []template.HTML // Pre-encoded query strings for batch scan-result endpoint (one per hold) + Meta *PageMeta + Owner *db.User + Repository *db.Repository + LatestTag string + StarCount int + PullCount int + IsStarred bool + IsOwner bool + ReadmeHTML template.HTML + ArtifactType string }{ - PageData: NewPageData(r, &h.BaseUIHandler), - Meta: meta, - Owner: owner, - Repository: repo, - Tags: tagsWithPlatforms, - Manifests: manifests, - StarCount: stats.StarCount, - PullCount: stats.PullCount, - IsStarred: isStarred, - IsOwner: isOwner, - ReadmeHTML: readmeHTML, - ArtifactType: artifactType, - ScanBatchParams: scanBatchParams, + PageData: NewPageData(r, &h.BaseUIHandler), + Meta: meta, + Owner: owner, + Repository: repo, + LatestTag: latestTagName, + StarCount: stats.StarCount, + PullCount: stats.PullCount, + IsStarred: isStarred, + IsOwner: isOwner, + ReadmeHTML: readmeHTML, + ArtifactType: artifactType, } if err := h.Templates.ExecuteTemplate(w, "repository", data); err != nil { @@ -317,3 +210,233 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request return } } + +// RepositoryTagsHandler returns the tags+manifests HTMX partial for a repository +type RepositoryTagsHandler struct { + BaseUIHandler +} + +func (h *RepositoryTagsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + identifier := chi.URLParam(r, "handle") + repository := strings.TrimPrefix(chi.URLParam(r, "*"), "/") + + did, _, _, err := atproto.ResolveIdentity(r.Context(), identifier) + if err != nil { + http.Error(w, "Not found", http.StatusNotFound) + return + } + + owner, err := db.GetUserByDID(h.ReadOnlyDB, did) + if err != nil || owner == nil { + http.Error(w, "Not found", http.StatusNotFound) + return + } + + // Parse pagination + const pageSize = 50 + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if parsed, err := strconv.Atoi(offsetStr); err == nil && parsed > 0 { + offset = parsed + } + } + + // Count total tags for pagination + totalTags, err := db.CountTags(h.ReadOnlyDB, owner.DID, repository) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Fetch tags with platform information and compressed sizes + tagsWithPlatforms, err := db.GetTagsWithPlatforms(h.ReadOnlyDB, owner.DID, repository, pageSize, offset) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Fetch untagged manifests only on first page + var manifests []db.ManifestWithMetadata + if offset == 0 { + manifests, err = db.GetTopLevelManifests(h.ReadOnlyDB, owner.DID, repository, 50, 0) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + + // Check health status for each manifest's hold endpoint + if h.HealthChecker != nil { + ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second) + defer cancel() + + var wg sync.WaitGroup + var mu sync.Mutex + + for i := range manifests { + if manifests[i].HoldEndpoint == "" { + manifests[i].Reachable = false + manifests[i].Pending = false + continue + } + + wg.Go(func() { + endpoint := manifests[i].HoldEndpoint + + if cached := h.HealthChecker.GetCachedStatus(endpoint); cached != nil { + mu.Lock() + manifests[i].Reachable = cached.Reachable + manifests[i].Pending = false + mu.Unlock() + return + } + + reachable, err := h.HealthChecker.CheckHealth(ctx, endpoint) + + mu.Lock() + if ctx.Err() == context.DeadlineExceeded { + manifests[i].Reachable = false + manifests[i].Pending = true + } else if err != nil { + manifests[i].Reachable = false + manifests[i].Pending = false + } else { + manifests[i].Reachable = reachable + manifests[i].Pending = false + } + mu.Unlock() + }) + } + + wg.Wait() + } else { + for i := range manifests { + manifests[i].Reachable = true + manifests[i].Pending = false + } + } + + // Check if current user is the repository owner + isOwner := false + user := middleware.GetUser(r) + if user != nil { + isOwner = (user.DID == owner.DID) + } + + // Build unified entries list: tagged first, then untagged. + // Single-arch entries get a one-element Platforms slice so the template + // can always just range over .Platforms without branching. + var entries []db.ManifestEntry + for _, t := range tagsWithPlatforms { + platforms := t.Platforms + if len(platforms) == 0 { + platforms = []db.PlatformInfo{{ + Digest: t.Digest, + HoldEndpoint: t.HoldEndpoint, + CompressedSize: t.CompressedSize, + }} + } + entries = append(entries, db.ManifestEntry{ + Label: t.Tag.Tag, + Digest: t.Digest, + IsTagged: true, + CreatedAt: t.CreatedAt, + HoldEndpoint: t.HoldEndpoint, + Platforms: platforms, + IsMultiArch: t.IsMultiArch, + HasAttestations: t.HasAttestations, + ArtifactType: t.ArtifactType, + }) + } + for _, m := range manifests { + if len(m.Tags) > 0 { + continue + } + platforms := m.Platforms + if len(platforms) == 0 { + platforms = []db.PlatformInfo{{ + Digest: m.Digest, + HoldEndpoint: m.HoldEndpoint, + }} + } + entries = append(entries, db.ManifestEntry{ + Label: m.Digest, + Digest: m.Digest, + IsTagged: false, + CreatedAt: m.CreatedAt, + HoldEndpoint: m.HoldEndpoint, + Platforms: platforms, + IsMultiArch: m.IsManifestList, + HasAttestations: m.HasAttestations, + ArtifactType: m.ArtifactType, + }) + } + + // Collect digests for batch scan-result requests + holdDigests := make(map[string][]string) + seen := make(map[string]bool) + for _, e := range entries { + if len(e.Platforms) > 0 { + for _, p := range e.Platforms { + if p.Digest != "" && p.HoldEndpoint != "" && !seen[p.Digest] { + seen[p.Digest] = true + hex := strings.TrimPrefix(p.Digest, "sha256:") + holdDigests[p.HoldEndpoint] = append(holdDigests[p.HoldEndpoint], hex) + } + } + } else if e.HoldEndpoint != "" { + if !seen[e.Digest] { + seen[e.Digest] = true + hex := strings.TrimPrefix(e.Digest, "sha256:") + holdDigests[e.HoldEndpoint] = append(holdDigests[e.HoldEndpoint], hex) + } + } + } + var scanBatchParams []template.HTML + for hold, digests := range holdDigests { + // Chunk into batches of 50 to match the batch handler's limit + for i := 0; i < len(digests); i += 50 { + end := i + 50 + if end > len(digests) { + end = len(digests) + } + scanBatchParams = append(scanBatchParams, template.HTML( + "holdEndpoint="+url.QueryEscape(hold)+"&digests="+strings.Join(digests[i:end], ","))) + } + } + + hasMore := offset+pageSize < totalTags + isFirstPage := offset == 0 + + data := struct { + Owner *db.User + Repository *db.Repository + Entries []db.ManifestEntry + IsOwner bool + ScanBatchParams []template.HTML + RegistryURL string + HasMore bool + NextOffset int + IsFirstPage bool + }{ + Owner: owner, + Repository: &db.Repository{Name: repository}, + Entries: entries, + IsOwner: isOwner, + ScanBatchParams: scanBatchParams, + RegistryURL: h.RegistryURL, + HasMore: hasMore, + NextOffset: offset + pageSize, + IsFirstPage: isFirstPage, + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + templateName := "repo-tags" + if !isFirstPage { + templateName = "repo-tags-page" + } + if err := h.Templates.ExecuteTemplate(w, templateName, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} diff --git a/pkg/appview/handlers/scan_result.go b/pkg/appview/handlers/scan_result.go index 1ac52aa..fb9653a 100644 --- a/pkg/appview/handlers/scan_result.go +++ b/pkg/appview/handlers/scan_result.go @@ -49,30 +49,14 @@ func (h *ScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Resolve hold identity: holdEndpoint may be a DID or URL - holdDID, err := atproto.ResolveHoldDID(r.Context(), holdEndpoint) + hold, err := ResolveHold(r.Context(), h.ReadOnlyDB, holdEndpoint) if err != nil { - slog.Debug("Failed to resolve hold DID", "holdEndpoint", holdEndpoint, "error", err) - h.renderBadge(w, vulnBadgeData{Error: true}) - return - } - - // Check if this hold has a successor — scan records may live there instead - resolvedHoldDID := resolveHoldSuccessor(h.ReadOnlyDB, holdDID) - - // Resolve to HTTP endpoint URL. If successor redirected, resolve the new DID; - // otherwise use the original holdEndpoint (which may already be a URL). - holdURLTarget := holdEndpoint - if resolvedHoldDID != holdDID { - holdDID = resolvedHoldDID - holdURLTarget = resolvedHoldDID - } - holdURL, err := atproto.ResolveHoldURL(r.Context(), holdURLTarget) - if err != nil { - slog.Debug("Failed to resolve hold URL", "holdEndpoint", holdEndpoint, "error", err) + slog.Debug("Failed to resolve hold", "holdEndpoint", holdEndpoint, "error", err) h.renderBadge(w, vulnBadgeData{Error: true}) return } + holdDID := hold.DID + holdURL := hold.URL // Compute rkey from digest (strip sha256: prefix) rkey := strings.TrimPrefix(digest, "sha256:") @@ -226,36 +210,17 @@ func (h *BatchScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques digests = digests[:50] } - holdDID, err := atproto.ResolveHoldDID(r.Context(), holdEndpoint) + hold, err := ResolveHold(r.Context(), h.ReadOnlyDB, holdEndpoint) if err != nil { - // Can't resolve hold — render empty OOB spans - slog.Debug("Failed to resolve hold DID for batch scan", "holdEndpoint", holdEndpoint, "error", err) - w.Header().Set("Content-Type", "text/html") - for _, d := range digests { - fmt.Fprintf(w, ``, template.HTMLEscapeString(d)) - } - return - } - - // Check if this hold has a successor — scan records may live there instead - resolvedHoldDID := resolveHoldSuccessor(h.ReadOnlyDB, holdDID) - - // Resolve to HTTP endpoint URL. If successor redirected, resolve the new DID; - // otherwise use the original holdEndpoint (which may already be a URL). - holdURLTarget := holdEndpoint - if resolvedHoldDID != holdDID { - holdDID = resolvedHoldDID - holdURLTarget = resolvedHoldDID - } - holdURL, err := atproto.ResolveHoldURL(r.Context(), holdURLTarget) - if err != nil { - slog.Debug("Failed to resolve hold URL for batch scan", "holdEndpoint", holdEndpoint, "error", err) + slog.Debug("Failed to resolve hold for batch scan", "holdEndpoint", holdEndpoint, "error", err) w.Header().Set("Content-Type", "text/html") for _, d := range digests { fmt.Fprintf(w, ``, template.HTMLEscapeString(d)) } return } + holdDID := hold.DID + holdURL := hold.URL // Fetch scan records concurrently with a semaphore to limit parallelism type result struct { @@ -294,21 +259,36 @@ func (h *BatchScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques } } -// resolveHoldSuccessor checks if a hold has a successor in the cached captain records. -// Returns the successor DID if set, otherwise returns the original holdDID. -// Single-hop only — does not follow chains. -func resolveHoldSuccessor(database *sql.DB, holdDID string) string { - if database == nil { - return holdDID +// ResolvedHold contains the resolved DID and URL for a hold endpoint, +// after following any successor chain. +type ResolvedHold struct { + DID string + URL string +} + +// ResolveHold resolves a hold endpoint (DID, URL, or hostname) to its final +// DID and URL, following a single successor hop if one exists in the captain records. +func ResolveHold(ctx context.Context, database *sql.DB, holdEndpoint string) (*ResolvedHold, error) { + holdDID, err := atproto.ResolveHoldDID(ctx, holdEndpoint) + if err != nil { + return nil, fmt.Errorf("resolve hold DID: %w", err) } - captain, err := db.GetCaptainRecord(database, holdDID) - if err != nil || captain == nil { - return holdDID + + // Check for successor + resolveTarget := holdEndpoint + if database != nil { + captain, err := db.GetCaptainRecord(database, holdDID) + if err == nil && captain != nil && captain.Successor != "" { + slog.Debug("Following hold successor", "from", holdDID, "to", captain.Successor) + holdDID = captain.Successor + resolveTarget = captain.Successor + } } - if captain.Successor != "" { - slog.Debug("Scan result: following hold successor", - "from", holdDID, "to", captain.Successor) - return captain.Successor + + holdURL, err := atproto.ResolveHoldURL(ctx, resolveTarget) + if err != nil { + return nil, fmt.Errorf("resolve hold URL: %w", err) } - return holdDID + + return &ResolvedHold{DID: holdDID, URL: holdURL}, nil } diff --git a/pkg/appview/handlers/scan_result_test.go b/pkg/appview/handlers/scan_result_test.go index 6fde643..f3434de 100644 --- a/pkg/appview/handlers/scan_result_test.go +++ b/pkg/appview/handlers/scan_result_test.go @@ -110,9 +110,9 @@ func TestScanResult_WithVulnerabilities(t *testing.T) { if !strings.Contains(body, `data-tip="Low">3<`) { t.Error("Expected low count of 3") } - // Should be clickable (has openVulnDetails) - if !strings.Contains(body, "openVulnDetails") { - t.Error("Expected body to contain openVulnDetails click handler") + // Should show vulnerability strip with tooltip + if !strings.Contains(body, "vuln-strip") { + t.Error("Expected body to contain vuln-strip class") } } @@ -141,9 +141,9 @@ func TestScanResult_Clean(t *testing.T) { if !strings.Contains(body, "badge-success") { t.Error("Expected body to contain badge-success for clean scan") } - // Should NOT be clickable - if strings.Contains(body, "openVulnDetails") { - t.Error("Clean badge should not have openVulnDetails click handler") + // Clean badge should not have vuln-strip + if strings.Contains(body, "vuln-strip") { + t.Error("Clean badge should not have vuln-strip") } } @@ -415,6 +415,65 @@ func TestBatchScanResult_HoldUnreachable(t *testing.T) { } } +// --- ResolveHold tests --- + +func TestResolveHold_DirectURL(t *testing.T) { + // Mock hold that serves DID resolution + hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/atproto-did" { + w.Write([]byte("did:web:hold.example.com")) + return + } + http.Error(w, "not found", http.StatusNotFound) + })) + defer hold.Close() + + resolved, err := handlers.ResolveHold(t.Context(), nil, hold.URL) + if err != nil { + t.Fatalf("ResolveHold failed: %v", err) + } + if resolved.DID != "did:web:hold.example.com" { + t.Errorf("DID = %q, want %q", resolved.DID, "did:web:hold.example.com") + } + if resolved.URL != hold.URL { + t.Errorf("URL = %q, want %q", resolved.URL, hold.URL) + } +} + +func TestResolveHold_NilDB_NoSuccessor(t *testing.T) { + // With nil DB, successor check is skipped + hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/atproto-did" { + w.Write([]byte("did:web:hold.example.com")) + return + } + http.Error(w, "not found", http.StatusNotFound) + })) + defer hold.Close() + + resolved, err := handlers.ResolveHold(t.Context(), nil, hold.URL) + if err != nil { + t.Fatalf("ResolveHold failed: %v", err) + } + // Should resolve to the original hold since no DB to check successor + if resolved.DID != "did:web:hold.example.com" { + t.Errorf("DID = %q, want %q", resolved.DID, "did:web:hold.example.com") + } + if resolved.URL != hold.URL { + t.Errorf("URL = %q, want %q", resolved.URL, hold.URL) + } +} + +func TestResolveHold_Unreachable(t *testing.T) { + hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + hold.Close() + + _, err := handlers.ResolveHold(t.Context(), nil, hold.URL) + if err == nil { + t.Error("Expected error for unreachable hold") + } +} + func TestBatchScanResult_SingleDigest(t *testing.T) { hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if handleMockDID(w, r) { diff --git a/pkg/appview/handlers/vuln_details.go b/pkg/appview/handlers/vuln_details.go index 00f3c91..c2fca64 100644 --- a/pkg/appview/handlers/vuln_details.go +++ b/pkg/appview/handlers/vuln_details.go @@ -97,19 +97,14 @@ func (h *VulnDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - holdDID, err := atproto.ResolveHoldDID(r.Context(), holdEndpoint) - if err != nil { - slog.Debug("Failed to resolve hold DID", "holdEndpoint", holdEndpoint, "error", err) - h.renderDetails(w, vulnDetailsData{Error: "Could not resolve hold identity"}) - return - } - - // Resolve to HTTP endpoint URL (handles DID, URL, or hostname) - holdURL, err := atproto.ResolveHoldURL(r.Context(), holdEndpoint) + hold, err := ResolveHold(r.Context(), h.ReadOnlyDB, holdEndpoint) if err != nil { + slog.Debug("Failed to resolve hold", "holdEndpoint", holdEndpoint, "error", err) h.renderDetails(w, vulnDetailsData{Error: "Could not resolve hold endpoint"}) return } + holdDID := hold.DID + holdURL := hold.URL rkey := strings.TrimPrefix(digest, "sha256:") @@ -255,3 +250,142 @@ func (h *VulnDetailsHandler) renderDetails(w http.ResponseWriter, data vulnDetai slog.Warn("Failed to render vuln details", "error", err) } } + +// FetchVulnDetails fetches vulnerability scan details for a digest from a hold. +// This is the shared logic used by both VulnDetailsHandler and DigestDetailHandler. +// holdEndpoint should already be resolved (successor-aware) before calling this. +func FetchVulnDetails(ctx context.Context, holdEndpoint, digest string) vulnDetailsData { + holdDID, err := atproto.ResolveHoldDID(ctx, holdEndpoint) + if err != nil { + return vulnDetailsData{Error: "Could not resolve hold identity"} + } + + holdURL, err := atproto.ResolveHoldURL(ctx, holdEndpoint) + if err != nil { + return vulnDetailsData{Error: "Could not resolve hold endpoint"} + } + + rkey := strings.TrimPrefix(digest, "sha256:") + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + // Fetch the scan record + scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", + holdURL, + url.QueryEscape(holdDID), + url.QueryEscape(atproto.ScanCollection), + url.QueryEscape(rkey), + ) + + req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil) + if err != nil { + return vulnDetailsData{Error: "Failed to build request"} + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return vulnDetailsData{Error: "Hold service unreachable"} + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return vulnDetailsData{Error: "No scan record found"} + } + + var envelope struct { + Value json.RawMessage `json:"value"` + } + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + return vulnDetailsData{Error: "Failed to parse scan record"} + } + + var scanRecord atproto.ScanRecord + if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil { + return vulnDetailsData{Error: "Failed to parse scan record"} + } + + summary := vulnSummary{ + Critical: scanRecord.Critical, + High: scanRecord.High, + Medium: scanRecord.Medium, + Low: scanRecord.Low, + Total: scanRecord.Total, + } + + // Fetch the vulnerability report blob + if scanRecord.VulnReportBlob == nil || scanRecord.VulnReportBlob.Ref.String() == "" { + return vulnDetailsData{ + Summary: summary, + ScannedAt: scanRecord.ScannedAt, + Error: "No detailed vulnerability report available. Only summary counts were recorded.", + } + } + + blobCID := scanRecord.VulnReportBlob.Ref.String() + blobURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", + holdURL, + url.QueryEscape(holdDID), + url.QueryEscape(blobCID), + ) + + blobReq, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil) + if err != nil { + return vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to build blob request"} + } + + blobResp, err := http.DefaultClient.Do(blobReq) + if err != nil { + return vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to fetch vulnerability report"} + } + defer blobResp.Body.Close() + + if blobResp.StatusCode != http.StatusOK { + return vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Vulnerability report not accessible"} + } + + var report grypeReport + if err := json.NewDecoder(blobResp.Body).Decode(&report); err != nil { + return vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to parse vulnerability report"} + } + + matches := make([]vulnMatch, 0, len(report.Matches)) + for _, m := range report.Matches { + fixedIn := "" + if len(m.Vulnerability.Fix.Versions) > 0 { + fixedIn = strings.Join(m.Vulnerability.Fix.Versions, ", ") + } + + cveURL := "" + if strings.HasPrefix(m.Vulnerability.ID, "CVE-") { + cveURL = "https://nvd.nist.gov/vuln/detail/" + m.Vulnerability.ID + } else if strings.HasPrefix(m.Vulnerability.ID, "GHSA-") { + cveURL = "https://github.com/advisories/" + m.Vulnerability.ID + } + + matches = append(matches, vulnMatch{ + CVEID: m.Vulnerability.ID, + CVEURL: cveURL, + Severity: m.Vulnerability.Metadata.Severity, + Package: m.Package.Name, + Version: m.Package.Version, + FixedIn: fixedIn, + Type: m.Package.Type, + }) + } + + sort.Slice(matches, func(i, j int) bool { + oi := severityOrder[matches[i].Severity] + oj := severityOrder[matches[j].Severity] + if oi != oj { + return oi < oj + } + return matches[i].CVEID < matches[j].CVEID + }) + + return vulnDetailsData{ + Matches: matches, + Summary: summary, + ScannedAt: scanRecord.ScannedAt, + } +} diff --git a/pkg/appview/handlers/vuln_details_test.go b/pkg/appview/handlers/vuln_details_test.go index 7520880..bc7a7f7 100644 --- a/pkg/appview/handlers/vuln_details_test.go +++ b/pkg/appview/handlers/vuln_details_test.go @@ -222,9 +222,9 @@ func TestVulnDetails_FullReport(t *testing.T) { t.Error("Expected body to contain fix version '1.2.4'") } - // Should contain "No fix" for unfixed vuln - if !strings.Contains(body, "No fix") { - t.Error("Expected body to contain 'No fix' for unfixed vulnerability") + // Should contain "-" placeholder for unfixed vuln + if !strings.Contains(body, `opacity-40`) { + t.Error("Expected body to contain opacity-40 placeholder for unfixed vulnerability") } // Should contain a table diff --git a/pkg/appview/jetstream/backfill.go b/pkg/appview/jetstream/backfill.go index a519eff..9403c65 100644 --- a/pkg/appview/jetstream/backfill.go +++ b/pkg/appview/jetstream/backfill.go @@ -151,9 +151,10 @@ func (b *BackfillWorker) backfillCollection(ctx context.Context, collection stri strings.Contains(errStr, "Could not find repo") || strings.Contains(errStr, "status 400") || strings.Contains(errStr, "status 404") { - if delErr := db.DeleteUserData(b.db, repo.DID); delErr != nil { + deleted, delErr := db.DeleteUserData(b.db, repo.DID) + if delErr != nil { slog.Warn("Backfill failed to delete data for removed repo", "did", repo.DID, "error", delErr) - } else { + } else if deleted { slog.Info("Backfill cleaned up data for deleted/deactivated repo", "did", repo.DID) } } else { @@ -183,8 +184,9 @@ func (b *BackfillWorker) backfillCollection(ctx context.Context, collection stri } // backfillRepo backfills all records for a single repo/DID. -// Per-record processing is wrapped in a single SQL transaction to batch writes -// (one commit per repo instead of per-statement). +// Records are fetched from PDS first, then network-dependent caches are warmed, +// and finally DB writes happen in chunked transactions to batch writes while +// staying under the remote SQLite transaction timeout (~5s on Bunny Database). func (b *BackfillWorker) backfillRepo(ctx context.Context, did, collection string) (int, error) { // Resolve DID to get user's PDS endpoint pdsEndpoint, err := atproto.ResolveDIDToPDS(ctx, did) @@ -193,37 +195,24 @@ func (b *BackfillWorker) backfillRepo(ctx context.Context, did, collection strin } // Create a client for this user's PDS with the user's DID - // This allows GetRecord to work properly with the repo parameter pdsClient := atproto.NewClient(pdsEndpoint, did, "") - // Begin transaction for per-record processing (batches all writes into one commit) - tx, err := b.db.Begin() - if err != nil { - return 0, fmt.Errorf("failed to begin transaction: %w", err) - } - defer tx.Rollback() - - // Create a transactional processor — all DB writes go through this tx - txProcessor := NewProcessor(tx, false, b.processor.statsCache) - var recordCursor string - recordCount := 0 // Track which records exist on the PDS for reconciliation var foundManifestDigests []string var foundTags []struct{ Repository, Tag string } foundStars := make(map[string]time.Time) // key: "ownerDID/repository", value: createdAt - // Paginate through all records for this repo + // Phase 1: Collect all records from PDS (network I/O, no transaction) + var allRecords []atproto.Record for { records, cursor, err := pdsClient.ListRecordsForRepo(ctx, did, collection, 100, recordCursor) if err != nil { - return recordCount, fmt.Errorf("failed to list records: %w", err) + return 0, fmt.Errorf("failed to list records: %w", err) } - // Process each record for _, record := range records { - // Track what we found for deletion reconciliation switch collection { case atproto.ManifestCollection: var manifestRecord atproto.ManifestRecord @@ -248,24 +237,65 @@ func (b *BackfillWorker) backfillRepo(ctx context.Context, did, collection strin } } - if err := b.processRecordWith(ctx, txProcessor, did, collection, &record); err != nil { - slog.Warn("Backfill failed to process record", "uri", record.URI, "error", err) + allRecords = append(allRecords, record) + } + + if cursor == "" { + break + } + recordCursor = cursor + } + + // Phase 2: Pre-warm caches outside any transaction so that ProcessRecord + // inside transactions hits only DB (no network I/O that could cause timeouts). + + // Ensure user exists in DB (resolves DID → handle/PDS, fetches profile) + switch collection { + case atproto.SailorProfileCollection: + if err := b.processor.EnsureUser(ctx, did); err != nil { + slog.Warn("Backfill failed to pre-ensure user", "did", did, "error", err) + } + case atproto.ManifestCollection, atproto.TagCollection, atproto.StarCollection, atproto.RepoPageCollection: + if err := b.processor.EnsureUserExists(ctx, did); err != nil { + slog.Warn("Backfill failed to pre-ensure user", "did", did, "error", err) + } + } + + // Pre-cache hold DIDs and captain records referenced in records. + // ProcessSailorProfile calls ResolveHoldDID + queryCaptainFn, + // ProcessManifest calls ResolveHoldDID for legacy manifests. + b.prewarmHoldCaches(ctx, collection, allRecords) + + // Phase 3: Process records in chunked transactions. + // All network I/O should be cached by now, so transactions stay fast. + const chunkSize = 20 + recordCount := 0 + + for i := 0; i < len(allRecords); i += chunkSize { + end := i + chunkSize + if end > len(allRecords) { + end = len(allRecords) + } + + tx, err := b.db.Begin() + if err != nil { + return recordCount, fmt.Errorf("failed to begin transaction: %w", err) + } + + txProcessor := NewProcessor(tx, false, b.processor.statsCache) + + for j := i; j < end; j++ { + if err := b.processRecordWith(ctx, txProcessor, did, collection, &allRecords[j]); err != nil { + slog.Warn("Backfill failed to process record", "uri", allRecords[j].URI, "error", err) continue } recordCount++ } - // Check if there are more pages - if cursor == "" { - break + if err := tx.Commit(); err != nil { + tx.Rollback() + return recordCount, fmt.Errorf("failed to commit transaction: %w", err) } - - recordCursor = cursor - } - - // Commit all per-record writes in one batch - if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("failed to commit transaction: %w", err) } // Reconciliation runs outside the transaction (involves network I/O and fewer writes) @@ -354,6 +384,53 @@ func (b *BackfillWorker) reconcileDeletions(did, collection string, foundManifes return nil } +// prewarmHoldCaches resolves hold DIDs and caches captain records before +// records are processed inside transactions. This ensures ProcessRecord's +// network-dependent code paths (ResolveHoldDID, queryCaptainRecord) hit +// cached data so transactions stay fast and don't timeout. +func (b *BackfillWorker) prewarmHoldCaches(ctx context.Context, collection string, records []atproto.Record) { + seen := make(map[string]bool) + + for _, record := range records { + var holdRef string + + switch collection { + case atproto.SailorProfileCollection: + var profileRecord atproto.SailorProfileRecord + if err := json.Unmarshal(record.Value, &profileRecord); err == nil { + holdRef = profileRecord.DefaultHold + } + case atproto.ManifestCollection: + var manifestRecord atproto.ManifestRecord + if err := json.Unmarshal(record.Value, &manifestRecord); err == nil { + // Only legacy manifests need network resolution (URL → DID) + if manifestRecord.HoldDID == "" && manifestRecord.HoldEndpoint != "" { + holdRef = manifestRecord.HoldEndpoint + } + } + default: + return // No hold references in other collections + } + + if holdRef == "" || seen[holdRef] { + continue + } + seen[holdRef] = true + + // Resolve hold identifier to DID (caches in resolver) + holdDID, err := atproto.ResolveHoldDID(ctx, holdRef) + if err != nil { + slog.Warn("Backfill failed to pre-resolve hold DID", "hold_ref", holdRef, "error", err) + continue + } + + // Pre-cache captain record (skips if cached within last hour) + if err := b.queryCaptainRecord(ctx, holdDID); err != nil { + slog.Warn("Backfill failed to pre-cache captain record", "hold_did", holdDID, "error", err) + } + } +} + // processRecordWith processes a single record using the given processor. // This allows backfillRepo to use a transactional processor while other callers use the default. func (b *BackfillWorker) processRecordWith(ctx context.Context, proc *Processor, did, collection string, record *atproto.Record) error { diff --git a/pkg/appview/jetstream/processor.go b/pkg/appview/jetstream/processor.go index 6f8049b..9ce49bf 100644 --- a/pkg/appview/jetstream/processor.go +++ b/pkg/appview/jetstream/processor.go @@ -925,7 +925,7 @@ func (p *Processor) ProcessAccount(ctx context.Context, did string, active bool, switch status { case "deleted": // Account permanently deleted - remove all cached data - if err := db.DeleteUserData(p.db, did); err != nil { + if _, err := db.DeleteUserData(p.db, did); err != nil { slog.Error("Failed to delete user data for deleted account", "component", "processor", "did", did, diff --git a/pkg/appview/public/icons.svg b/pkg/appview/public/icons.svg index 965a522..63020e1 100644 --- a/pkg/appview/public/icons.svg +++ b/pkg/appview/public/icons.svg @@ -6,7 +6,6 @@ - @@ -26,14 +25,15 @@ + - + diff --git a/pkg/appview/routes/routes.go b/pkg/appview/routes/routes.go index 29782b8..d257eb2 100644 --- a/pkg/appview/routes/routes.go +++ b/pkg/appview/routes/routes.go @@ -151,6 +151,14 @@ func RegisterUIRoutes(router chi.Router, deps UIDependencies) { &uihandlers.RepositoryPageHandler{BaseUIHandler: base}, ).ServeHTTP) + router.Get("/api/repo-tags/{handle}/*", middleware.OptionalAuth(deps.SessionStore, deps.Database)( + &uihandlers.RepositoryTagsHandler{BaseUIHandler: base}, + ).ServeHTTP) + + router.Get("/d/{handle}/*", middleware.OptionalAuth(deps.SessionStore, deps.Database)( + &uihandlers.DigestDetailHandler{BaseUIHandler: base}, + ).ServeHTTP) + // Authenticated routes router.Group(func(r chi.Router) { r.Use(middleware.RequireAuth(deps.SessionStore, deps.Database)) diff --git a/pkg/appview/templates/components/repo-card.html b/pkg/appview/templates/components/repo-card.html index a2971c1..5ab6340 100644 --- a/pkg/appview/templates/components/repo-card.html +++ b/pkg/appview/templates/components/repo-card.html @@ -68,7 +68,7 @@ {{ end }} {{ if not .LastUpdated.IsZero }} - {{ timeAgo .LastUpdated }} + {{ icon "history" "size-4" }}{{ timeAgoShort .LastUpdated }} {{ end }} diff --git a/pkg/appview/templates/pages/repository.html b/pkg/appview/templates/pages/repository.html index fb32ecc..21fec58 100644 --- a/pkg/appview/templates/pages/repository.html +++ b/pkg/appview/templates/pages/repository.html @@ -74,17 +74,15 @@
{{ if eq .ArtifactType "helm-chart" }}

Pull this chart

- {{ if .Tags }} - {{ $firstTag := index .Tags 0 }} - {{ template "docker-command" (print "helm pull oci://" $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name " --version " $firstTag.Tag.Tag) }} + {{ if .LatestTag }} + {{ template "docker-command" (print "helm pull oci://" $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name " --version " .LatestTag) }} {{ else }} {{ template "docker-command" (print "helm pull oci://" $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name) }} {{ end }} {{ else }}

Pull this image

- {{ if .Tags }} - {{ $firstTag := index .Tags 0 }} - {{ template "docker-command" (print "docker pull " $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name ":" $firstTag.Tag.Tag) }} + {{ if .LatestTag }} + {{ template "docker-command" (print "docker pull " $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name ":" .LatestTag) }} {{ else }} {{ template "docker-command" (print "docker pull " $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name ":latest") }} {{ end }} @@ -92,213 +90,49 @@
- - {{ if .ReadmeHTML }} -
- + +
+ +
+ + + +
+ {{ if .ReadmeHTML }}
-

Overview

{{ .ReadmeHTML }}
- - -
- {{ end }} - - -
-

Tags

- {{ if .Tags }} -
- {{ range .Tags }} -
-
-
- {{ .Tag.Tag }} - {{ if eq .ArtifactType "helm-chart" }} - {{ icon "helm" "size-3" }} Helm chart - {{ else if .IsMultiArch }} - Multi-arch - {{ end }} - {{ if .HasAttestations }} - - {{ end }} -
-
- - {{ if $.IsOwner }} - - {{ end }} -
-
-
-
- {{ .Tag.Digest }} - -
- {{ if .Platforms }} -
- {{ range .Platforms }} -
- {{ .OS }}/{{ .Architecture }}{{ if .Variant }}/{{ .Variant }}{{ end }} - {{ if .Digest }} - {{ .Digest }} - - {{ if .HoldEndpoint }} - - {{ end }} - {{ end }} -
- {{ end }} -
- {{ else if .HoldEndpoint }} - {{/* Single-arch: scan badge for the tag's own digest */}} -
- {{ end }} -
- {{ if eq .ArtifactType "helm-chart" }} - {{ template "docker-command" (print "helm pull oci://" $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name " --version " .Tag.Tag) }} - {{ else }} - {{ template "docker-command" (print "docker pull " $.RegistryURL "/" $.Owner.Handle "/" $.Repository.Name ":" .Tag.Tag) }} - {{ end }} -
- {{ end }} -
- {{ if $.ScanBatchParams }} - {{ range $.ScanBatchParams }} -
- {{ end }} - {{ end }} {{ else }} -

No tags available

+
+

No description available

+
{{ end }}
- -
-
-

Manifests

-
- {{ if $.IsOwner }} - - {{ end }} - + + - - {{ if .ReadmeHTML }} -
-
- {{ end }}
@@ -326,7 +160,7 @@

Delete Untagged Manifests

-

This will delete all untagged manifests in this repository.

+

This will delete all untagged manifests in this repository, including those not currently visible.

This action cannot be undone.

@@ -339,20 +173,6 @@
- - -
-

Vulnerability Scan Results

-
- -
-
-
-
-
-
- -
@@ -367,6 +187,76 @@
+ + {{ template "footer" . }} diff --git a/pkg/appview/templates/partials/vuln-badge.html b/pkg/appview/templates/partials/vuln-badge.html index 060ac71..e515aee 100644 --- a/pkg/appview/templates/partials/vuln-badge.html +++ b/pkg/appview/templates/partials/vuln-badge.html @@ -6,13 +6,11 @@ {{ else if eq .Total 0 }} {{ icon "shield-check" "size-3" }} Clean {{ else }} - + {{ end }} {{ end }} diff --git a/pkg/appview/templates/partials/vuln-details.html b/pkg/appview/templates/partials/vuln-details.html index 759a0f9..a72287c 100644 --- a/pkg/appview/templates/partials/vuln-details.html +++ b/pkg/appview/templates/partials/vuln-details.html @@ -9,17 +9,17 @@ {{ .Summary.Medium }} {{ .Summary.Low }} -

{{ .Error }}

- {{ if .ScannedAt }}

Scanned: {{ .ScannedAt }}

{{ end }} +

{{ .Error }}

+ {{ if .ScannedAt }}

Scanned: {{ .ScannedAt }}

{{ end }}
{{ else }} -

{{ .Error }}

+

{{ .Error }}

{{ end }} {{ else }}
- {{ .Summary.Total }} vulnerabilities found + {{ .Summary.Total }} vulnerabilities {{ .Summary.Critical }} {{ .Summary.High }} @@ -28,19 +28,19 @@
- {{ if .ScannedAt }}

Scanned: {{ .ScannedAt }}

{{ end }} + {{ if .ScannedAt }}

Scanned: {{ .ScannedAt }}

{{ end }} {{ if .Matches }} -
- +
+
- + - - + + @@ -55,27 +55,26 @@ - - - + diff --git a/pkg/appview/ui.go b/pkg/appview/ui.go index 8619b1f..d49e0a2 100644 --- a/pkg/appview/ui.go +++ b/pkg/appview/ui.go @@ -157,6 +157,22 @@ func Templates(overrides *BrandingOverrides) (*template.Template, error) { } }, + "timeAgoShort": func(t time.Time) string { + duration := time.Since(t) + + if duration < time.Minute { + return "now" + } else if duration < time.Hour { + return fmt.Sprintf("%dm", int(duration.Minutes())) + } else if duration < 24*time.Hour { + return fmt.Sprintf("%dh", int(duration.Hours())) + } else if duration < 365*24*time.Hour { + return fmt.Sprintf("%dd", int(duration.Hours()/24)) + } else { + return fmt.Sprintf("%dy", int(duration.Hours()/(24*365))) + } + }, + "humanizeBytes": func(bytes int64) string { const unit = 1024 if bytes < unit { diff --git a/pkg/atproto/cbor_gen.go b/pkg/atproto/cbor_gen.go index eca2f5e..6495ac8 100644 --- a/pkg/atproto/cbor_gen.go +++ b/pkg/atproto/cbor_gen.go @@ -2425,3 +2425,205 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *ImageConfigRecord) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Type (string) (string) + if len("$type") > 8192 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if len(t.Type) > 8192 { + return xerrors.Errorf("Value in field t.Type was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Type)); err != nil { + return err + } + + // t.Manifest (string) (string) + if len("manifest") > 8192 { + return xerrors.Errorf("Value in field \"manifest\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("manifest"))); err != nil { + return err + } + if _, err := cw.WriteString(string("manifest")); err != nil { + return err + } + + if len(t.Manifest) > 8192 { + return xerrors.Errorf("Value in field t.Manifest was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Manifest))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Manifest)); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 8192 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 8192 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.ConfigJSON (string) (string) + if len("configJson") > 8192 { + return xerrors.Errorf("Value in field \"configJson\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("configJson"))); err != nil { + return err + } + if _, err := cw.WriteString(string("configJson")); err != nil { + return err + } + + if len(t.ConfigJSON) > 8192 { + return xerrors.Errorf("Value in field t.ConfigJSON was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ConfigJSON))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.ConfigJSON)); err != nil { + return err + } + return nil +} + +func (t *ImageConfigRecord) UnmarshalCBOR(r io.Reader) (err error) { + *t = ImageConfigRecord{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ImageConfigRecord: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 10) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Type (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.Type = string(sval) + } + // t.Manifest (string) (string) + case "manifest": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.Manifest = string(sval) + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.ConfigJSON (string) (string) + case "configJson": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.ConfigJSON = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/pkg/atproto/endpoints.go b/pkg/atproto/endpoints.go index 6580ceb..107f6d0 100644 --- a/pkg/atproto/endpoints.go +++ b/pkg/atproto/endpoints.go @@ -46,6 +46,18 @@ const ( // Response: {"success": true, "layersCreated": 5, "postCreated": true, "postUri": "at://..."} HoldNotifyManifest = "/xrpc/io.atcr.hold.notifyManifest" + // HoldGetLayersForManifest returns layer records for a specific manifest. + // Method: GET + // Query: manifest={at-uri} + // Response: {"layers": [{...}]} + HoldGetLayersForManifest = "/xrpc/io.atcr.hold.getLayersForManifest" + + // HoldGetImageConfig returns the OCI image config record for a manifest. + // Method: GET + // Query: digest={manifest-digest} + // Response: ImageConfigRecord JSON + HoldGetImageConfig = "/xrpc/io.atcr.hold.image.getConfig" + // HoldGetQuota returns storage quota information for a user. // Method: GET // Query: userDid={did} diff --git a/pkg/atproto/generate.go b/pkg/atproto/generate.go index f8dda39..804b089 100644 --- a/pkg/atproto/generate.go +++ b/pkg/atproto/generate.go @@ -33,6 +33,7 @@ func main() { atproto.TangledProfileRecord{}, atproto.StatsRecord{}, atproto.ScanRecord{}, + atproto.ImageConfigRecord{}, ); err != nil { fmt.Printf("Failed to generate CBOR encoders: %v\n", err) os.Exit(1) diff --git a/pkg/atproto/lexicon.go b/pkg/atproto/lexicon.go index 9ff8059..952fa18 100644 --- a/pkg/atproto/lexicon.go +++ b/pkg/atproto/lexicon.go @@ -48,6 +48,10 @@ const ( // Stored in hold's embedded PDS to track scan results per manifest ScanCollection = "io.atcr.hold.scan" + // ImageConfigCollection is the collection name for OCI image configs + // Stored in hold's embedded PDS, one per manifest (keyed by manifest digest hex) + ImageConfigCollection = "io.atcr.hold.image.config" + // TangledProfileCollection is the collection name for tangled profiles // Stored in hold's embedded PDS (singleton record at rkey "self") TangledProfileCollection = "sh.tangled.actor.profile" @@ -820,6 +824,27 @@ func ScanRecordKey(manifestDigest string) string { return strings.TrimPrefix(manifestDigest, "sha256:") } +// ImageConfigRecord represents an OCI image config stored in the hold's embedded PDS +// Collection: io.atcr.hold.image.config +// One record per manifest, keyed by manifest digest hex (same pattern as ScanRecord) +// Stores the full OCI config JSON so the appview can display layer history including empty layers +type ImageConfigRecord struct { + Type string `json:"$type" cborgen:"$type"` + Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the manifest + ConfigJSON string `json:"configJson" cborgen:"configJson"` // Raw OCI image config JSON + CreatedAt string `json:"createdAt" cborgen:"createdAt"` // RFC3339 timestamp +} + +// NewImageConfigRecord creates a new image config record +func NewImageConfigRecord(manifestURI, configJSON string) *ImageConfigRecord { + return &ImageConfigRecord{ + Type: ImageConfigCollection, + Manifest: manifestURI, + ConfigJSON: configJSON, + CreatedAt: time.Now().Format(time.RFC3339), + } +} + // TangledProfileRecord represents a Tangled profile for the hold // Collection: sh.tangled.actor.profile (singleton record at rkey "self") // Stored in the hold's embedded PDS diff --git a/pkg/hold/admin/admin.go b/pkg/hold/admin/admin.go index 4d11295..f3fe7eb 100644 --- a/pkg/hold/admin/admin.go +++ b/pkg/hold/admin/admin.go @@ -417,6 +417,7 @@ func (ui *AdminUI) RegisterRoutes(r chi.Router) { r.Post("/admin/api/gc/reconcile", ui.handleGCReconcile) r.Post("/admin/api/gc/delete-records", ui.handleGCDeleteRecords) r.Post("/admin/api/gc/delete-blobs", ui.handleGCDeleteBlobs) + r.Post("/admin/api/gc/backfill-configs", ui.handleGCBackfillConfigs) r.Get("/admin/api/gc/status", ui.handleGCStatus) // API endpoints (for HTMX) diff --git a/pkg/hold/admin/handlers_gc.go b/pkg/hold/admin/handlers_gc.go index 769a01e..ea97a0e 100644 --- a/pkg/hold/admin/handlers_gc.go +++ b/pkg/hold/admin/handlers_gc.go @@ -136,6 +136,27 @@ func (ui *AdminUI) handleGCDeleteRecords(w http.ResponseWriter, r *http.Request) }) } +// handleGCBackfillConfigs starts image config backfill in the background +func (ui *AdminUI) handleGCBackfillConfigs(w http.ResponseWriter, r *http.Request) { + if ui.gc == nil { + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"}) + return + } + + session := getSessionFromContext(r.Context()) + + if ui.gc.StartBackfillConfigs() { + slog.Info("GC backfill configs started via admin panel", "by", session.DID) + } + + progress := ui.gc.GetProgress() + ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{ + Phase: progress.Phase, + Message: progress.Message, + OpType: progress.OperationType, + }) +} + // handleGCDeleteBlobs starts orphaned blob deletion in the background func (ui *AdminUI) handleGCDeleteBlobs(w http.ResponseWriter, r *http.Request) { if ui.gc == nil { diff --git a/pkg/hold/admin/public/icons.svg b/pkg/hold/admin/public/icons.svg index 965a522..63020e1 100644 --- a/pkg/hold/admin/public/icons.svg +++ b/pkg/hold/admin/public/icons.svg @@ -6,7 +6,6 @@ - @@ -26,14 +25,15 @@ + - + diff --git a/pkg/hold/admin/templates/partials/tab_storage.html b/pkg/hold/admin/templates/partials/tab_storage.html index 85c3f3a..7578910 100644 --- a/pkg/hold/admin/templates/partials/tab_storage.html +++ b/pkg/hold/admin/templates/partials/tab_storage.html @@ -41,6 +41,15 @@ {{ icon "search" "size-4" }} Scan for Orphans +
diff --git a/pkg/hold/gc/gc.go b/pkg/hold/gc/gc.go index dbdd72d..fabf931 100644 --- a/pkg/hold/gc/gc.go +++ b/pkg/hold/gc/gc.go @@ -815,6 +815,158 @@ func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing } } +// StartBackfillConfigs launches image config backfill in the background. +// Creates io.atcr.hold.image.config records for manifests that don't have one yet +// by fetching OCI config blobs from S3. +func (gc *GarbageCollector) StartBackfillConfigs() bool { + return gc.startBackground("backfill-configs", "records", "Scanning for manifests missing image config records...", func(ctx context.Context) error { + _, err := gc.doBackfillConfigs(ctx) + return err + }) +} + +// doBackfillConfigs creates image config records for manifests that are missing them. +func (gc *GarbageCollector) doBackfillConfigs(ctx context.Context) (*GCResult, error) { + recordsIndex := gc.pds.RecordsIndex() + if recordsIndex == nil { + return nil, fmt.Errorf("records index not available") + } + + // Step 1: Collect unique manifest URIs from layer records + manifestURIs := make(map[string]bool) + cursor := "" + totalScanned := 0 + + for { + records, nextCursor, err := recordsIndex.ListRecords(atproto.LayerCollection, 1000, cursor, true) + if err != nil { + return nil, fmt.Errorf("list layer records: %w", err) + } + + for _, rec := range records { + totalScanned++ + layer, err := gc.decodeLayerRecord(ctx, rec) + if err != nil { + continue + } + manifestURIs[layer.Manifest] = true + } + + if nextCursor == "" { + break + } + cursor = nextCursor + } + + gc.logger.Info("Found unique manifests from layer records", + "manifests", len(manifestURIs), + "layersScanned", totalScanned) + + // Step 2: For each manifest, check if config record exists, create if not + start := time.Now() + result := &GCResult{} + created := int64(0) + skipped := int64(0) + processed := 0 + httpClient := &http.Client{Timeout: 30 * time.Second} + + for manifestURI := range manifestURIs { + processed++ + gc.setProgress("records", + fmt.Sprintf("Backfilling configs (%d/%d manifests)...", processed, len(manifestURIs)), + "backfill-configs") + + aturi, err := syntax.ParseATURI(manifestURI) + if err != nil { + gc.logger.Warn("Invalid manifest URI", "uri", manifestURI, "error", err) + continue + } + + manifestDigest := "sha256:" + aturi.RecordKey().String() + + // Check if config record already exists + if _, _, err := gc.pds.GetImageConfigRecord(ctx, manifestDigest); err == nil { + skipped++ + continue + } + + userDID := aturi.Authority().String() + manifestRkey := aturi.RecordKey().String() + + pdsEndpoint, err := atproto.ResolveDIDToPDS(ctx, userDID) + if err != nil { + gc.logger.Warn("Failed to resolve PDS for backfill", "did", userDID, "error", err) + continue + } + + // Fetch manifest via getRecord to get config digest + reqURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", + pdsEndpoint, + url.QueryEscape(userDID), + url.QueryEscape(atproto.ManifestCollection), + url.QueryEscape(manifestRkey)) + + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + continue + } + resp, err := httpClient.Do(req) + if err != nil { + gc.logger.Warn("Failed to fetch manifest for backfill", "uri", manifestURI, "error", err) + continue + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + continue + } + + var envelope struct { + Value json.RawMessage `json:"value"` + } + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + resp.Body.Close() + continue + } + resp.Body.Close() + + var manifest atproto.ManifestRecord + if err := json.Unmarshal(envelope.Value, &manifest); err != nil { + continue + } + + if manifest.Config == nil || manifest.Config.Digest == "" { + continue + } + + // Fetch config blob from S3 + configBytes, err := gc.s3.GetBytes(ctx, s3.BlobPath(manifest.Config.Digest)) + if err != nil { + gc.logger.Warn("Failed to fetch config blob", "digest", manifest.Config.Digest, "error", err) + continue + } + + // Create image config record + configRecord := atproto.NewImageConfigRecord(manifestURI, string(configBytes)) + if _, _, err := gc.pds.CreateImageConfigRecord(ctx, configRecord, manifestDigest); err != nil { + gc.logger.Warn("Failed to create image config record", "manifest", manifestURI, "error", err) + continue + } + created++ + time.Sleep(200 * time.Millisecond) // throttle firehose events + } + + result.RecordsReconciled = created + result.Duration = time.Since(start) + + gc.mu.Lock() + gc.lastResult = result + gc.lastResultAt = time.Now() + gc.mu.Unlock() + + gc.logger.Info("Image config backfill complete", "created", created, "skipped", skipped) + return result, nil +} + // discoverUserDIDs returns all DIDs that may have manifests referencing this hold. // Union of: captain owner + crew members + distinct DIDs from layer records. func (gc *GarbageCollector) discoverUserDIDs(ctx context.Context) ([]string, error) { diff --git a/pkg/hold/oci/xrpc.go b/pkg/hold/oci/xrpc.go index 1a27c28..f500fa4 100644 --- a/pkg/hold/oci/xrpc.go +++ b/pkg/hold/oci/xrpc.go @@ -297,22 +297,43 @@ func (h *XRPCHandler) HandleNotifyManifest(w http.ResponseWriter, r *http.Reques // Build manifest AT-URI for layer records manifestURI := atproto.BuildManifestURI(req.UserDID, req.ManifestDigest) - // Create layer records for each blob - for _, layer := range req.Manifest.Layers { - record := atproto.NewLayerRecord( - layer.Digest, - layer.Size, - layer.MediaType, - req.UserDID, - manifestURI, - ) + // Skip layer record creation if records already exist for this manifest + existingLayers, _ := h.pds.ListLayerRecordsForManifest(ctx, manifestURI) + if len(existingLayers) > 0 { + layersCreated = len(existingLayers) + slog.Debug("Layer records already exist for manifest, skipping creation", + "manifestURI", manifestURI, "existing", len(existingLayers)) + } else { + // Create layer records for each blob + for _, layer := range req.Manifest.Layers { + record := atproto.NewLayerRecord( + layer.Digest, + layer.Size, + layer.MediaType, + req.UserDID, + manifestURI, + ) - _, _, err := h.pds.CreateLayerRecord(ctx, record) + _, _, err := h.pds.CreateLayerRecord(ctx, record) + if err != nil { + slog.Error("Failed to create layer record", "error", err) + // Continue creating other records + } else { + layersCreated++ + } + } + } + + // Store OCI image config as a separate record (best-effort) + if req.Manifest.Config.Digest != "" { + configBytes, err := h.s3Service.GetBytes(ctx, s3.BlobPath(req.Manifest.Config.Digest)) if err != nil { - slog.Error("Failed to create layer record", "error", err) - // Continue creating other records + slog.Warn("Failed to fetch config blob for image config record", "error", err, "configDigest", req.Manifest.Config.Digest) } else { - layersCreated++ + configRecord := atproto.NewImageConfigRecord(manifestURI, string(configBytes)) + if _, _, err := h.pds.CreateImageConfigRecord(ctx, configRecord, req.ManifestDigest); err != nil { + slog.Warn("Failed to create image config record", "error", err) + } } } diff --git a/pkg/hold/oci/xrpc_test.go b/pkg/hold/oci/xrpc_test.go index 55a1288..86a630f 100644 --- a/pkg/hold/oci/xrpc_test.go +++ b/pkg/hold/oci/xrpc_test.go @@ -678,7 +678,7 @@ func TestHandleCompleteUpload_MockS3_Success(t *testing.T) { // 6. Verify blob was moved to final location in mock S3 finalS3Key := "test-prefix/docker/registry/v2/blobs/sha256/ab/abc123def456/data" - if mockS3Client.GetObject(finalS3Key) == nil { + if mockS3Client.GetObjectBytes(finalS3Key) == nil { t.Errorf("Expected blob at final S3 key %s", finalS3Key) } } diff --git a/pkg/hold/pds/layer.go b/pkg/hold/pds/layer.go index a2f5841..882d8e3 100644 --- a/pkg/hold/pds/layer.go +++ b/pkg/hold/pds/layer.go @@ -90,6 +90,15 @@ func (p *HoldPDS) GetLayerRecord(ctx context.Context, rkey string) (*atproto.Lay return nil, fmt.Errorf("GetLayerRecord not yet implemented - use via XRPC listRecords instead") } +// UpdateLayerRecord updates an existing layer record by rkey. +func (p *HoldPDS) UpdateLayerRecord(ctx context.Context, rkey string, record *atproto.LayerRecord) error { + _, err := p.repomgr.UpdateRecord(ctx, p.uid, atproto.LayerCollection, rkey, record) + if err != nil { + return fmt.Errorf("failed to update layer record: %w", err) + } + return nil +} + // DeleteLayerRecord deletes a layer record by rkey // This deletes from both the repo (MST) and the records index func (p *HoldPDS) DeleteLayerRecord(ctx context.Context, rkey string) error { @@ -294,3 +303,78 @@ func (p *HoldPDS) ListLayerRecordsForUser(ctx context.Context, userDID string) ( return records, nil } + +// ListLayerRecordsForManifest returns all layer records for a specific manifest AT-URI. +func (p *HoldPDS) ListLayerRecordsForManifest(ctx context.Context, manifestURI string) ([]*atproto.LayerRecord, error) { + if p.recordsIndex == nil { + return nil, fmt.Errorf("records index not available") + } + + session, err := p.carstore.ReadOnlySession(p.uid) + if err != nil { + return nil, fmt.Errorf("failed to create session: %w", err) + } + + head, err := p.carstore.GetUserRepoHead(ctx, p.uid) + if err != nil { + return nil, fmt.Errorf("failed to get repo head: %w", err) + } + + if !head.Defined() { + return []*atproto.LayerRecord{}, nil + } + + repoHandle, err := repo.OpenRepo(ctx, session, head) + if err != nil { + return nil, fmt.Errorf("failed to open repo: %w", err) + } + + var records []*atproto.LayerRecord + seen := make(map[string]int) // digest → index in records slice + cursor := "" + batchSize := 1000 + + for { + indexRecords, nextCursor, err := p.recordsIndex.ListRecords(atproto.LayerCollection, batchSize, cursor, false) + if err != nil { + return nil, fmt.Errorf("failed to list layer records: %w", err) + } + + for _, rec := range indexRecords { + recordPath := rec.Collection + "/" + rec.Rkey + + _, recBytes, err := repoHandle.GetRecordBytes(ctx, recordPath) + if err != nil { + continue + } + + recordValue, err := lexutil.CborDecodeValue(*recBytes) + if err != nil { + continue + } + + layerRecord, ok := recordValue.(*atproto.LayerRecord) + if !ok { + continue + } + + if layerRecord.Manifest == manifestURI { + if _, exists := seen[layerRecord.Digest]; !exists { + seen[layerRecord.Digest] = len(records) + records = append(records, layerRecord) + } + } + } + + if nextCursor == "" { + break + } + cursor = nextCursor + } + + if records == nil { + records = []*atproto.LayerRecord{} + } + + return records, nil +} diff --git a/pkg/hold/pds/server.go b/pkg/hold/pds/server.go index 0afdf50..403cd1f 100644 --- a/pkg/hold/pds/server.go +++ b/pkg/hold/pds/server.go @@ -32,6 +32,7 @@ func init() { lexutil.RegisterType(atproto.TangledProfileCollection, &atproto.TangledProfileRecord{}) lexutil.RegisterType(atproto.StatsCollection, &atproto.StatsRecord{}) lexutil.RegisterType(atproto.ScanCollection, &atproto.ScanRecord{}) + lexutil.RegisterType(atproto.ImageConfigCollection, &atproto.ImageConfigRecord{}) } // HoldPDS is a minimal ATProto PDS implementation for a hold service diff --git a/pkg/hold/pds/xrpc.go b/pkg/hold/pds/xrpc.go index 6d32337..9e53438 100644 --- a/pkg/hold/pds/xrpc.go +++ b/pkg/hold/pds/xrpc.go @@ -168,6 +168,8 @@ func (h *XRPCHandler) RegisterHandlers(r chi.Router) { r.Get(atproto.RepoDescribeRepo, h.HandleDescribeRepo) r.Get(atproto.RepoGetRecord, h.HandleGetRecord) r.Get(atproto.RepoListRecords, h.HandleListRecords) + r.Get(atproto.HoldGetLayersForManifest, h.HandleGetLayersForManifest) + r.Get(atproto.HoldGetImageConfig, h.HandleGetImageConfig) // Sync endpoints r.Get(atproto.SyncListBlobs, h.HandleListBlobs) @@ -487,6 +489,49 @@ func (h *XRPCHandler) HandleGetRecord(w http.ResponseWriter, r *http.Request) { }) } +// HandleGetLayersForManifest returns layer records for a specific manifest AT-URI. +func (h *XRPCHandler) HandleGetLayersForManifest(w http.ResponseWriter, r *http.Request) { + manifestURI := r.URL.Query().Get("manifest") + if manifestURI == "" { + http.Error(w, `{"error":"InvalidRequest","message":"manifest parameter is required"}`, http.StatusBadRequest) + return + } + + records, err := h.pds.ListLayerRecordsForManifest(r.Context(), manifestURI) + if err != nil { + slog.Error("Failed to list layer records for manifest", "error", err, "manifest", manifestURI) + http.Error(w, `{"error":"InternalServerError","message":"failed to list layer records"}`, http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "layers": records, + }); err != nil { + slog.Error("Failed to encode layer records response", "error", err) + } +} + +// HandleGetImageConfig returns the OCI image config record for a manifest digest. +func (h *XRPCHandler) HandleGetImageConfig(w http.ResponseWriter, r *http.Request) { + digest := r.URL.Query().Get("digest") + if digest == "" { + http.Error(w, `{"error":"InvalidRequest","message":"digest parameter is required"}`, http.StatusBadRequest) + return + } + + _, record, err := h.pds.GetImageConfigRecord(r.Context(), digest) + if err != nil { + http.Error(w, `{"error":"NotFound","message":"image config not found"}`, http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(record); err != nil { + slog.Error("Failed to encode image config response", "error", err) + } +} + // HandleListRecords lists records in a collection // Spec: https://docs.bsky.app/docs/api/com-atproto-repo-list-records // Supports pagination via limit, cursor, and reverse parameters diff --git a/pkg/hold/pds/xrpc_test.go b/pkg/hold/pds/xrpc_test.go index 3453a6f..e131ccd 100644 --- a/pkg/hold/pds/xrpc_test.go +++ b/pkg/hold/pds/xrpc_test.go @@ -1062,6 +1062,111 @@ func TestHandleListRecords_Indexed_EmptyCollection(t *testing.T) { } } +// Tests for HandleGetLayersForManifest + +func TestHandleGetLayersForManifest(t *testing.T) { + handler, ctx := setupTestXRPCHandlerWithIndex(t) + + manifestURI := "at://did:plc:testuser/io.atcr.manifest/abc123" + otherManifestURI := "at://did:plc:testuser/io.atcr.manifest/def456" + + // Create layer records for the target manifest + for i := range 3 { + record := atproto.NewLayerRecord( + fmt.Sprintf("sha256:layer%d", i), + int64(1024*(i+1)), + "application/vnd.oci.image.layer.v1.tar+gzip", + "did:plc:testuser", + manifestURI, + ) + if _, _, err := handler.pds.CreateLayerRecord(ctx, record); err != nil { + t.Fatalf("Failed to create layer record %d: %v", i, err) + } + } + + // Create a layer record for a different manifest (should not be returned) + otherRecord := atproto.NewLayerRecord( + "sha256:otherlayer", + 2048, + "application/vnd.oci.image.layer.v1.tar+gzip", + "did:plc:testuser", + otherManifestURI, + ) + if _, _, err := handler.pds.CreateLayerRecord(ctx, otherRecord); err != nil { + t.Fatalf("Failed to create other layer record: %v", err) + } + + // Query layers for the target manifest + req := makeXRPCGetRequest(atproto.HoldGetLayersForManifest, map[string]string{ + "manifest": manifestURI, + }) + w := httptest.NewRecorder() + handler.HandleGetLayersForManifest(w, req) + + result := assertJSONResponse(t, w, http.StatusOK) + + layers, ok := result["layers"].([]any) + if !ok { + t.Fatal("Expected layers array in response") + } + + if len(layers) != 3 { + t.Fatalf("Expected 3 layers, got %d", len(layers)) + } + + // Verify all layers have digests and belong to the target manifest + digests := make(map[string]bool) + for i, l := range layers { + layer, ok := l.(map[string]any) + if !ok { + t.Fatalf("Layer %d: expected map, got %T", i, l) + } + digest, ok := layer["digest"].(string) + if !ok || digest == "" { + t.Errorf("Layer %d: expected non-empty digest", i) + } + digests[digest] = true + } + for i := range 3 { + expected := fmt.Sprintf("sha256:layer%d", i) + if !digests[expected] { + t.Errorf("Expected digest %q in results", expected) + } + } +} + +func TestHandleGetLayersForManifest_MissingParam(t *testing.T) { + handler, _ := setupTestXRPCHandlerWithIndex(t) + + req := makeXRPCGetRequest(atproto.HoldGetLayersForManifest, map[string]string{}) + w := httptest.NewRecorder() + handler.HandleGetLayersForManifest(w, req) + + if w.Code != http.StatusBadRequest { + t.Errorf("Expected status 400, got %d", w.Code) + } +} + +func TestHandleGetLayersForManifest_NoMatchingLayers(t *testing.T) { + handler, _ := setupTestXRPCHandlerWithIndex(t) + + req := makeXRPCGetRequest(atproto.HoldGetLayersForManifest, map[string]string{ + "manifest": "at://did:plc:nobody/io.atcr.manifest/nonexistent", + }) + w := httptest.NewRecorder() + handler.HandleGetLayersForManifest(w, req) + + result := assertJSONResponse(t, w, http.StatusOK) + + layers, ok := result["layers"].([]any) + if !ok { + t.Fatal("Expected layers array in response") + } + if len(layers) != 0 { + t.Errorf("Expected 0 layers, got %d", len(layers)) + } +} + // Tests for HandleDeleteRecord // TestHandleDeleteRecord tests com.atproto.repo.deleteRecord diff --git a/pkg/s3/mock.go b/pkg/s3/mock.go index e42ff56..deed775 100644 --- a/pkg/s3/mock.go +++ b/pkg/s3/mock.go @@ -400,8 +400,25 @@ func (m *MockS3Client) SetObject(key string, data []byte) { m.Objects[key] = append([]byte{}, data...) } -// GetObject is a test helper to read an object from the mock store (nil if not found). -func (m *MockS3Client) GetObject(key string) []byte { +// GetObject implements S3Client +func (m *MockS3Client) GetObject(ctx context.Context, input *awss3.GetObjectInput, opts ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + + key := aws.ToString(input.Key) + data, ok := m.Objects[key] + if !ok { + return nil, fmt.Errorf("NoSuchKey: %s", key) + } + + return &awss3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(bytes.Clone(data))), + ContentLength: aws.Int64(int64(len(data))), + }, nil +} + +// GetObjectBytes is a test helper to read an object from the mock store (nil if not found). +func (m *MockS3Client) GetObjectBytes(key string) []byte { m.mu.Lock() defer m.mu.Unlock() data, ok := m.Objects[key] diff --git a/pkg/s3/types.go b/pkg/s3/types.go index 19da9cd..8914ddd 100644 --- a/pkg/s3/types.go +++ b/pkg/s3/types.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "fmt" + "io" "log/slog" "net/url" "strings" @@ -27,6 +28,7 @@ type S3Client interface { AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error) // Direct object operations + GetObject(ctx context.Context, input *awss3.GetObjectInput, opts ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) @@ -70,6 +72,11 @@ func (r *RealS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab return r.client.AbortMultipartUpload(ctx, input, opts...) } +// GetObject implements S3Client +func (r *RealS3Client) GetObject(ctx context.Context, input *awss3.GetObjectInput, opts ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) { + return r.client.GetObject(ctx, input, opts...) +} + // HeadObject implements S3Client func (r *RealS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) { return r.client.HeadObject(ctx, input, opts...) @@ -149,6 +156,25 @@ type S3Service struct { PathPrefix string // S3 path prefix (if any) } +// GetBytes fetches an object from S3 and returns its contents as bytes. +func (s *S3Service) GetBytes(ctx context.Context, blobPath string) ([]byte, error) { + s3Key := strings.TrimPrefix(blobPath, "/") + if s.PathPrefix != "" { + s3Key = s.PathPrefix + "/" + s3Key + } + + result, err := s.Client.GetObject(ctx, &awss3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &s3Key, + }) + if err != nil { + return nil, fmt.Errorf("get object %s: %w", s3Key, err) + } + defer result.Body.Close() + + return io.ReadAll(result.Body) +} + // NewS3Service initializes the S3 client for presigned URL generation // S3 is required - this will return an error if not properly configured func NewS3Service(params map[string]any) (*S3Service, error) {
CVESeverity PackageInstalledFixed InVersionFix
{{ if eq .Severity "Critical" }} - Critical + C {{ else if eq .Severity "High" }} - High + H {{ else if eq .Severity "Medium" }} - Medium + M {{ else if eq .Severity "Low" }} - Low + L {{ else }} - {{ .Severity }} + ? {{ end }} - {{ .Package }} - {{ if .Type }}({{ .Type }}){{ end }} + + {{ .Package }}{{ if .Type }} ({{ .Type }}){{ end }} {{ .Version }} + {{ .Version }} {{ if .FixedIn }} {{ .FixedIn }} {{ else }} - No fix + - {{ end }}