admin panel fixes

This commit is contained in:
Evan Jarrett
2026-02-18 21:40:53 -06:00
parent 5615dd4132
commit 22b2d69cb3
9 changed files with 433 additions and 118 deletions
File diff suppressed because one or more lines are too long
+10 -1
View File
@@ -85,11 +85,20 @@ document.addEventListener('DOMContentLoaded', () => {
if (!wrapper || !input) return;
// Close on Escape key
// Close on Escape key, open on "/" key (GitHub-style)
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape' && wrapper.classList.contains('expanded')) {
closeSearch();
}
if (e.key === '/' && !wrapper.classList.contains('expanded')) {
const tag = e.target.tagName;
if (tag === 'INPUT' || tag === 'TEXTAREA' || e.target.isContentEditable) return;
e.preventDefault();
wrapper.classList.add('expanded');
input.focus();
}
});
// Close on click outside
+2 -1
View File
@@ -411,12 +411,13 @@ func (ui *AdminUI) RegisterRoutes(r chi.Router) {
r.Post("/admin/relays/crawl", ui.handleRelayCrawl)
r.Post("/admin/relays/crawl-all", ui.handleRelayCrawlAll)
// GC POSTs
// GC (background operations + polling status)
r.Post("/admin/api/gc/preview", ui.handleGCPreview)
r.Post("/admin/api/gc/run", ui.handleGCRun)
r.Post("/admin/api/gc/reconcile", ui.handleGCReconcile)
r.Post("/admin/api/gc/delete-records", ui.handleGCDeleteRecords)
r.Post("/admin/api/gc/delete-blobs", ui.handleGCDeleteBlobs)
r.Get("/admin/api/gc/status", ui.handleGCStatus)
// API endpoints (for HTMX)
r.Get("/admin/api/stats", ui.handleStatsAPI)
+122 -80
View File
@@ -21,6 +21,13 @@ type gcTabData struct {
LastResultAge string
}
// gcProgressData is the data passed to the gc_progress.html partial
type gcProgressData struct {
Phase string
Message string
OpType string
}
// handleGCTab returns the storage/GC tab content (HTMX partial)
func (ui *AdminUI) handleGCTab(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
@@ -44,136 +51,171 @@ func (ui *AdminUI) handleGCTab(w http.ResponseWriter, r *http.Request) {
ui.renderTemplate(w, "partials/tab_storage.html", data)
}
// handleGCPreview runs a GC preview (analysis only, no mutations)
// handleGCPreview starts a GC preview in the background
func (ui *AdminUI) handleGCPreview(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
preview, err := ui.gc.Preview(r.Context())
if err != nil {
slog.Error("GC preview failed", "error", err)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
return
session := getSessionFromContext(r.Context())
if ui.gc.StartPreview() {
slog.Info("GC preview started via admin panel", "by", session.DID)
}
session := getSessionFromContext(r.Context())
slog.Info("GC preview completed via admin panel",
"orphanedRecords", len(preview.OrphanedRecords),
"orphanedBlobs", len(preview.OrphanedBlobs),
"missingRecords", len(preview.MissingRecords),
"duration", preview.Duration,
"by", session.DID)
ui.renderTemplate(w, "partials/gc_preview.html", struct {
Preview *gc.GCPreview
}{Preview: preview})
// Whether we just started or it was already running, show progress
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
}
// handleGCRun executes an actual GC run
// handleGCRun starts a full GC run in the background
func (ui *AdminUI) handleGCRun(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
result, err := ui.gc.Run(r.Context())
if err != nil {
slog.Error("GC run failed", "error", err)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
return
session := getSessionFromContext(r.Context())
if ui.gc.StartRun() {
slog.Info("GC run started via admin panel", "by", session.DID)
}
session := getSessionFromContext(r.Context())
slog.Info("GC run completed via admin panel",
"blobsDeleted", result.BlobsDeleted,
"bytesReclaimed", result.BytesReclaimed,
"recordsDeleted", result.RecordsDeleted,
"recordsReconciled", result.RecordsReconciled,
"duration", result.Duration,
"by", session.DID)
ui.renderTemplate(w, "partials/gc_result.html", struct {
Result *gc.GCResult
}{Result: result})
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
}
// handleGCReconcile creates missing layer records without deleting anything
// handleGCReconcile starts record reconciliation in the background
func (ui *AdminUI) handleGCReconcile(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
result, err := ui.gc.Reconcile(r.Context())
if err != nil {
slog.Error("GC reconcile failed", "error", err)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
return
session := getSessionFromContext(r.Context())
if ui.gc.StartReconcile() {
slog.Info("GC reconcile started via admin panel", "by", session.DID)
}
session := getSessionFromContext(r.Context())
slog.Info("GC reconcile completed via admin panel",
"recordsReconciled", result.RecordsReconciled,
"duration", result.Duration,
"by", session.DID)
ui.renderTemplate(w, "partials/gc_result.html", struct {
Result *gc.GCResult
}{Result: result})
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
}
// handleGCDeleteRecords deletes orphaned layer records (no blob deletion)
// handleGCDeleteRecords starts orphaned record deletion in the background
func (ui *AdminUI) handleGCDeleteRecords(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
result, err := ui.gc.DeleteOrphanedRecords(r.Context())
if err != nil {
slog.Error("GC delete records failed", "error", err)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
return
session := getSessionFromContext(r.Context())
if ui.gc.StartDeleteRecords() {
slog.Info("GC delete records started via admin panel", "by", session.DID)
}
session := getSessionFromContext(r.Context())
slog.Info("GC delete orphaned records completed via admin panel",
"recordsDeleted", result.RecordsDeleted,
"orphanedRecords", result.OrphanedRecords,
"duration", result.Duration,
"by", session.DID)
ui.renderTemplate(w, "partials/gc_result.html", struct {
Result *gc.GCResult
}{Result: result})
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
}
// handleGCDeleteBlobs walks S3 and deletes unreferenced blobs
// handleGCDeleteBlobs starts orphaned blob deletion in the background
func (ui *AdminUI) handleGCDeleteBlobs(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
result, err := ui.gc.DeleteOrphanedBlobs(r.Context())
if err != nil {
slog.Error("GC delete blobs failed", "error", err)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
session := getSessionFromContext(r.Context())
if ui.gc.StartDeleteBlobs() {
slog.Info("GC delete blobs started via admin panel", "by", session.DID)
}
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
}
// handleGCStatus returns current progress or final results (polled by gc_progress.html)
func (ui *AdminUI) handleGCStatus(w http.ResponseWriter, r *http.Request) {
if ui.gc == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
return
}
session := getSessionFromContext(r.Context())
slog.Info("GC delete orphaned blobs completed via admin panel",
"blobsDeleted", result.BlobsDeleted,
"bytesReclaimed", result.BytesReclaimed,
"duration", result.Duration,
"by", session.DID)
progress := ui.gc.GetProgress()
ui.renderTemplate(w, "partials/gc_result.html", struct {
Result *gc.GCResult
}{Result: result})
// Still running — return progress partial (which will poll again)
if progress.Running {
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
Phase: progress.Phase,
Message: progress.Message,
OpType: progress.OperationType,
})
return
}
// Error — show error
if progress.Phase == "error" {
slog.Error("GC operation failed", "type", progress.OperationType, "error", progress.Error)
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{progress.Error})
return
}
// Complete — render final results based on operation type
switch progress.OperationType {
case "preview":
preview, _ := ui.gc.LastPreview()
if preview == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"No preview results available"})
return
}
slog.Info("GC preview completed via admin panel",
"orphanedRecords", len(preview.OrphanedRecords),
"orphanedBlobs", len(preview.OrphanedBlobs),
"missingRecords", len(preview.MissingRecords),
"duration", preview.Duration)
ui.renderTemplate(w, "partials/gc_preview.html", struct {
Preview *gc.GCPreview
}{Preview: preview})
default:
result, _ := ui.gc.LastResult()
if result == nil {
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"No results available"})
return
}
slog.Info("GC operation completed via admin panel",
"type", progress.OperationType,
"blobsDeleted", result.BlobsDeleted,
"recordsDeleted", result.RecordsDeleted,
"recordsReconciled", result.RecordsReconciled,
"duration", result.Duration)
ui.renderTemplate(w, "partials/gc_result.html", struct {
Result *gc.GCResult
}{Result: result})
}
}
// timeAgo returns a human-readable relative time string
@@ -142,8 +142,7 @@
<button class="btn btn-warning gap-2"
hx-post="/admin/api/gc/reconcile"
hx-target="#gc-results"
hx-swap="innerHTML"
hx-indicator="#gc-loading">
hx-swap="innerHTML">
{{ icon "file-plus" "size-4" }}
Reconcile {{len .Preview.MissingRecords}} Records
</button>
@@ -153,8 +152,7 @@
hx-post="/admin/api/gc/delete-records"
hx-target="#gc-results"
hx-swap="innerHTML"
hx-confirm="Delete {{len .Preview.OrphanedRecords}} orphaned layer records?"
hx-indicator="#gc-loading">
hx-confirm="Delete {{len .Preview.OrphanedRecords}} orphaned layer records?">
{{ icon "file-x" "size-4" }}
Delete {{len .Preview.OrphanedRecords}} Orphaned Records
</button>
@@ -164,8 +162,7 @@
hx-post="/admin/api/gc/delete-blobs"
hx-target="#gc-results"
hx-swap="innerHTML"
hx-confirm="Delete {{len .Preview.OrphanedBlobs}} orphaned blobs from S3? This cannot be undone."
hx-indicator="#gc-loading">
hx-confirm="Delete {{len .Preview.OrphanedBlobs}} orphaned blobs from S3? This cannot be undone.">
{{ icon "trash-2" "size-4" }}
Delete {{len .Preview.OrphanedBlobs}} Orphaned Blobs
</button>
@@ -0,0 +1,21 @@
{{define "partials/gc_progress.html"}}
<div hx-get="/admin/api/gc/status"
hx-trigger="load delay:2s"
hx-target="#gc-results"
hx-swap="innerHTML">
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
<span class="loading loading-spinner loading-md text-primary"></span>
<div>
<p class="font-medium">{{.Message}}</p>
<p class="text-sm text-base-content/50">
{{if eq .Phase "manifests"}}Fetching manifests from user PDS instances...
{{else if eq .Phase "records"}}Scanning layer records...
{{else if eq .Phase "blobs"}}Walking S3 storage...
{{else if eq .Phase "deleting"}}Processing changes...
{{else}}Working...
{{end}}
</p>
</div>
</div>
</div>
{{end}}
@@ -33,9 +33,9 @@
{{if .RepoStatus}}
{{if .RepoStatus.Active}}
{{if eq .RepoStatus.Rev $.CurrentRev}}
<span class="text-sm text-success">Known (rev: {{truncate .RepoStatus.Rev 12}})</span>
<span class="text-sm text-success">Known (rev: {{.RepoStatus.Rev}})</span>
{{else}}
<span class="text-sm text-warning">Behind (rev: {{truncate .RepoStatus.Rev 12}})</span>
<span class="text-sm text-warning">Behind (rev: {{.RepoStatus.Rev}})</span>
{{end}}
{{else}}
<span class="text-sm text-warning">Inactive</span>
@@ -37,24 +37,26 @@
hx-post="/admin/api/gc/preview"
hx-target="#gc-results"
hx-swap="innerHTML"
hx-indicator="#gc-loading"
{{if .Running}}disabled{{end}}>
{{ icon "search" "size-4" }}
Scan for Orphans
</button>
</div>
<div id="gc-loading" class="htmx-indicator mb-4">
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
<span class="loading loading-spinner loading-md text-primary"></span>
<div>
<p class="font-medium">Scanning storage...</p>
<p class="text-sm text-base-content/50">This may take a few minutes for large holds.</p>
<div id="gc-results">
{{if .Running}}
<div hx-get="/admin/api/gc/status"
hx-trigger="load delay:2s"
hx-target="#gc-results"
hx-swap="innerHTML">
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
<span class="loading loading-spinner loading-md text-primary"></span>
<div>
<p class="font-medium">Operation in progress...</p>
<p class="text-sm text-base-content/50">Checking status...</p>
</div>
</div>
</div>
</div>
<div id="gc-results">
<!-- Preview or run results will be swapped in here -->
{{end}}
</div>
{{end}}
+258 -15
View File
@@ -5,8 +5,10 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
@@ -82,6 +84,16 @@ type GarbageCollector struct {
lastPreviewAt time.Time
lastResult *GCResult
lastResultAt time.Time
// Progress tracking for background operations
phase string // "manifests", "records", "blobs", "deleting", "complete", "error"
progressMsg string // e.g. "Fetching manifests (35/69 users)"
operationType string // "preview", "run", "reconcile", "delete-records", "delete-blobs"
lastError error
// predecessorCache caches holdDID → "is this a predecessor of our hold?"
// A predecessor is a hold whose captain record has a successor label set.
predecessorCache map[string]bool
}
// GCResult contains statistics from a GC run
@@ -119,11 +131,12 @@ type analysisResult struct {
// NewGarbageCollector creates a new GC instance
func NewGarbageCollector(holdPDS *pds.HoldPDS, s3svc *s3.S3Service, cfg Config) *GarbageCollector {
return &GarbageCollector{
pds: holdPDS,
s3: s3svc,
cfg: cfg,
logger: slog.Default().With("component", "gc"),
stopCh: make(chan struct{}),
pds: holdPDS,
s3: s3svc,
cfg: cfg,
logger: slog.Default().With("component", "gc"),
stopCh: make(chan struct{}),
predecessorCache: make(map[string]bool),
}
}
@@ -145,6 +158,109 @@ func (gc *GarbageCollector) finish() {
gc.mu.Unlock()
}
// setProgress updates the progress fields (thread-safe).
func (gc *GarbageCollector) setProgress(phase, msg, opType string) {
gc.mu.Lock()
gc.phase = phase
gc.progressMsg = msg
gc.operationType = opType
if phase != "error" {
gc.lastError = nil
}
gc.mu.Unlock()
}
// GCProgress holds a snapshot of the current GC operation progress.
type GCProgress struct {
Phase string // "manifests", "records", "blobs", "deleting", "complete", "error"
Message string
OperationType string // "preview", "run", "reconcile", "delete-records", "delete-blobs"
Running bool
Error string
}
// GetProgress returns the current progress state.
func (gc *GarbageCollector) GetProgress() GCProgress {
gc.mu.Lock()
defer gc.mu.Unlock()
p := GCProgress{
Phase: gc.phase,
Message: gc.progressMsg,
OperationType: gc.operationType,
Running: gc.running,
}
if gc.lastError != nil {
p.Error = gc.lastError.Error()
}
return p
}
// startBackground is the common pattern for launching a GC operation in the background.
// Returns false if already running.
func (gc *GarbageCollector) startBackground(opType, initialPhase, initialMsg string, fn func(ctx context.Context) error) bool {
if !gc.tryStart() {
return false
}
gc.setProgress(initialPhase, initialMsg, opType)
go func() {
defer gc.finish()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
if err := fn(ctx); err != nil {
gc.mu.Lock()
gc.phase = "error"
gc.progressMsg = err.Error()
gc.lastError = err
gc.mu.Unlock()
gc.logger.Error("GC operation failed", "type", opType, "error", err)
return
}
gc.setProgress("complete", "", opType)
}()
return true
}
// StartPreview launches a GC preview in the background.
func (gc *GarbageCollector) StartPreview() bool {
return gc.startBackground("preview", "manifests", "Starting preview...", func(ctx context.Context) error {
_, err := gc.doPreview(ctx)
return err
})
}
// StartRun launches a full GC run in the background.
func (gc *GarbageCollector) StartRun() bool {
return gc.startBackground("run", "manifests", "Starting GC run...", func(ctx context.Context) error {
_, err := gc.doRun(ctx)
return err
})
}
// StartReconcile launches record reconciliation in the background.
func (gc *GarbageCollector) StartReconcile() bool {
return gc.startBackground("reconcile", "deleting", "Reconciling missing records...", func(ctx context.Context) error {
_, err := gc.doReconcile(ctx)
return err
})
}
// StartDeleteRecords launches orphaned record deletion in the background.
func (gc *GarbageCollector) StartDeleteRecords() bool {
return gc.startBackground("delete-records", "deleting", "Deleting orphaned records...", func(ctx context.Context) error {
_, err := gc.doDeleteOrphanedRecords(ctx)
return err
})
}
// StartDeleteBlobs launches orphaned blob deletion in the background.
func (gc *GarbageCollector) StartDeleteBlobs() bool {
return gc.startBackground("delete-blobs", "manifests", "Analyzing records before blob deletion...", func(ctx context.Context) error {
_, err := gc.doDeleteOrphanedBlobs(ctx)
return err
})
}
// Start begins the GC background process with a nightly ticker
func (gc *GarbageCollector) Start(ctx context.Context) {
if !gc.cfg.Enabled {
@@ -194,7 +310,11 @@ func (gc *GarbageCollector) Run(ctx context.Context) (*GCResult, error) {
return nil, fmt.Errorf("GC operation already in progress")
}
defer gc.finish()
return gc.doRun(ctx)
}
// doRun is the internal implementation of Run (caller must hold the running lock).
func (gc *GarbageCollector) doRun(ctx context.Context) (*GCResult, error) {
start := time.Now()
result := &GCResult{}
@@ -216,17 +336,20 @@ func (gc *GarbageCollector) Run(ctx context.Context) (*GCResult, error) {
"missingRecords", len(analysis.missingDetails))
// Reconcile: create missing layer records
gc.setProgress("deleting", "Reconciling missing records...", "run")
if len(analysis.missingDetails) > 0 {
gc.reconcileMissingRecords(ctx, analysis.missingDetails, result)
}
// Phase 2: Delete orphaned layer records
gc.setProgress("deleting", "Deleting orphaned records...", "run")
if err := gc.deleteOrphanedRecords(ctx, analysis.orphanedRkeys, result); err != nil {
gc.logger.Error("Phase 2 (delete orphaned records) failed", "error", err)
// Continue to phase 3 - we can still clean up blobs
}
// Phase 3: Walk storage and delete unreferenced blobs
gc.setProgress("blobs", "Deleting orphaned blobs...", "run")
if err := gc.deleteOrphanedBlobs(ctx, analysis.referenced, result); err != nil {
return nil, fmt.Errorf("phase 3 (delete orphaned blobs) failed: %w", err)
}
@@ -250,7 +373,11 @@ func (gc *GarbageCollector) Preview(ctx context.Context) (*GCPreview, error) {
return nil, fmt.Errorf("GC operation already in progress")
}
defer gc.finish()
return gc.doPreview(ctx)
}
// doPreview is the internal implementation of Preview (caller must hold the running lock).
func (gc *GarbageCollector) doPreview(ctx context.Context) (*GCPreview, error) {
start := time.Now()
gc.logger.Info("Starting GC preview")
@@ -262,6 +389,7 @@ func (gc *GarbageCollector) Preview(ctx context.Context) (*GCPreview, error) {
}
// Phase 2: Walk S3 to find orphaned blobs (read-only)
gc.setProgress("blobs", "Walking S3 storage...", "preview")
orphanedBlobs, totalBlobs, err := gc.scanOrphanedBlobDetails(ctx, analysis.referenced)
if err != nil {
return nil, fmt.Errorf("scan orphaned blobs: %w", err)
@@ -303,7 +431,11 @@ func (gc *GarbageCollector) Reconcile(ctx context.Context) (*GCResult, error) {
return nil, fmt.Errorf("GC operation already in progress")
}
defer gc.finish()
return gc.doReconcile(ctx)
}
// doReconcile is the internal implementation of Reconcile (caller must hold the running lock).
func (gc *GarbageCollector) doReconcile(ctx context.Context) (*GCResult, error) {
gc.mu.Lock()
preview := gc.lastPreview
gc.mu.Unlock()
@@ -337,7 +469,11 @@ func (gc *GarbageCollector) DeleteOrphanedRecords(ctx context.Context) (*GCResul
return nil, fmt.Errorf("GC operation already in progress")
}
defer gc.finish()
return gc.doDeleteOrphanedRecords(ctx)
}
// doDeleteOrphanedRecords is the internal implementation (caller must hold the running lock).
func (gc *GarbageCollector) doDeleteOrphanedRecords(ctx context.Context) (*GCResult, error) {
gc.mu.Lock()
preview := gc.lastPreview
gc.mu.Unlock()
@@ -381,7 +517,11 @@ func (gc *GarbageCollector) DeleteOrphanedBlobs(ctx context.Context) (*GCResult,
return nil, fmt.Errorf("GC operation already in progress")
}
defer gc.finish()
return gc.doDeleteOrphanedBlobs(ctx)
}
// doDeleteOrphanedBlobs is the internal implementation (caller must hold the running lock).
func (gc *GarbageCollector) doDeleteOrphanedBlobs(ctx context.Context) (*GCResult, error) {
start := time.Now()
result := &GCResult{}
@@ -395,6 +535,7 @@ func (gc *GarbageCollector) DeleteOrphanedBlobs(ctx context.Context) (*GCResult,
result.ReferencedBlobs = int64(len(analysis.referenced))
gc.setProgress("blobs", "Deleting orphaned blobs...", "delete-blobs")
if err := gc.deleteOrphanedBlobs(ctx, analysis.referenced, result); err != nil {
return nil, fmt.Errorf("delete orphaned blobs: %w", err)
}
@@ -432,8 +573,10 @@ func (gc *GarbageCollector) analyzeRecords(ctx context.Context) (*analysisResult
knownManifests := make(map[string]*manifestInfo)
fetchedUsers := make(map[string]bool)
totalUsers := len(userDIDs)
for _, did := range userDIDs {
result.usersChecked++
gc.setProgress("manifests", fmt.Sprintf("Fetching manifests (%d/%d users)", result.usersChecked, totalUsers), gc.operationType)
manifests, err := gc.fetchUserManifests(ctx, did)
if err != nil {
@@ -464,6 +607,7 @@ func (gc *GarbageCollector) analyzeRecords(ctx context.Context) (*analysisResult
"fetchedUsers", len(fetchedUsers))
// Step 3: Scan existing layer records to find orphans and build coveredPairs
gc.setProgress("records", "Scanning layer records...", gc.operationType)
coveredPairs := make(map[string]bool)
cursor := ""
@@ -599,7 +743,7 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
// reconcileMissingRecords creates layer records for manifest+layer pairs that are missing.
func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing []MissingRecordDetail, result *GCResult) {
for _, m := range missing {
for i, m := range missing {
record := atproto.NewLayerRecord(
m.Digest,
m.Size,
@@ -615,10 +759,16 @@ func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing
continue
}
result.RecordsReconciled++
gc.logger.Info("Created missing layer record",
"digest", m.Digest,
"manifest", m.ManifestURI,
"user", m.UserDID)
if result.RecordsReconciled%100 == 0 {
gc.logger.Info("Reconciliation progress",
"created", result.RecordsReconciled,
"total", len(missing))
}
// Throttle to avoid flooding the firehose (~100 records/sec)
if i < len(missing)-1 {
time.Sleep(10 * time.Millisecond)
}
}
if result.RecordsReconciled > 0 {
@@ -751,20 +901,113 @@ func (gc *GarbageCollector) fetchUserManifestsFromEndpoint(ctx context.Context,
return manifests, nil
}
// manifestBelongsToHold checks if a manifest references this hold via HoldDID or legacy HoldEndpoint.
// manifestBelongsToHold checks if a manifest references this hold via HoldDID,
// legacy HoldEndpoint, or a predecessor hold that has been migrated.
func (gc *GarbageCollector) manifestBelongsToHold(ctx context.Context, manifest *atproto.ManifestRecord, holdDID string) bool {
if manifest.HoldDID == holdDID {
manifestHoldDID := manifest.HoldDID
// Direct match
if manifestHoldDID == holdDID {
return true
}
// Legacy: check holdEndpoint converted to DID
if manifest.HoldEndpoint != "" {
// Legacy: resolve holdEndpoint to DID
if manifestHoldDID == "" && manifest.HoldEndpoint != "" {
resolved, err := atproto.ResolveHoldDID(ctx, manifest.HoldEndpoint)
if err != nil {
gc.logger.Debug("Failed to resolve hold DID from legacy endpoint", "holdEndpoint", manifest.HoldEndpoint, "error", err)
return false
}
return resolved == holdDID
manifestHoldDID = resolved
if manifestHoldDID == holdDID {
return true
}
}
if manifestHoldDID == "" {
return false
}
// Check if the manifest's hold is a predecessor (has a successor label set)
return gc.isPredecessorHold(ctx, manifestHoldDID)
}
// isPredecessorHold checks if the given holdDID is a predecessor of this hold
// by fetching its captain record and checking for a successor label.
// Results are cached to avoid repeated network calls.
func (gc *GarbageCollector) isPredecessorHold(ctx context.Context, holdDID string) bool {
if gc.predecessorCache == nil {
gc.predecessorCache = make(map[string]bool)
}
if isPredecessor, cached := gc.predecessorCache[holdDID]; cached {
return isPredecessor
}
isPredecessor := gc.checkPredecessor(ctx, holdDID)
gc.predecessorCache[holdDID] = isPredecessor
return isPredecessor
}
// checkPredecessor fetches a hold's captain record to check if it has a successor label
// (meaning the hold has been migrated/retired and its blobs are served by this hold).
func (gc *GarbageCollector) checkPredecessor(ctx context.Context, holdDID string) bool {
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
holdURL, err := atproto.ResolveHoldURL(fetchCtx, holdDID)
if err != nil {
gc.logger.Debug("GC: failed to resolve predecessor hold URL",
"holdDID", holdDID, "error", err)
return false
}
recordURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=self",
holdURL,
url.QueryEscape(holdDID),
url.QueryEscape(atproto.CaptainCollection),
)
req, err := http.NewRequestWithContext(fetchCtx, "GET", recordURL, nil)
if err != nil {
return false
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
gc.logger.Debug("GC: failed to fetch predecessor captain record",
"holdDID", holdDID, "error", err)
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return false
}
var envelope struct {
Value json.RawMessage `json:"value"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
return false
}
var captain atproto.CaptainRecord
if err := json.Unmarshal(envelope.Value, &captain); err != nil {
return false
}
if captain.Successor != "" {
gc.logger.Info("GC: discovered predecessor hold (has successor label)",
"holdDID", holdDID, "successor", captain.Successor)
return true
}
return false
}