mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-02 08:16:57 +00:00
admin panel fixes
This commit is contained in:
Vendored
+2
-2
File diff suppressed because one or more lines are too long
@@ -85,11 +85,20 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
|
||||
if (!wrapper || !input) return;
|
||||
|
||||
// Close on Escape key
|
||||
// Close on Escape key, open on "/" key (GitHub-style)
|
||||
document.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape' && wrapper.classList.contains('expanded')) {
|
||||
closeSearch();
|
||||
}
|
||||
|
||||
if (e.key === '/' && !wrapper.classList.contains('expanded')) {
|
||||
const tag = e.target.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || e.target.isContentEditable) return;
|
||||
|
||||
e.preventDefault();
|
||||
wrapper.classList.add('expanded');
|
||||
input.focus();
|
||||
}
|
||||
});
|
||||
|
||||
// Close on click outside
|
||||
|
||||
@@ -411,12 +411,13 @@ func (ui *AdminUI) RegisterRoutes(r chi.Router) {
|
||||
r.Post("/admin/relays/crawl", ui.handleRelayCrawl)
|
||||
r.Post("/admin/relays/crawl-all", ui.handleRelayCrawlAll)
|
||||
|
||||
// GC POSTs
|
||||
// GC (background operations + polling status)
|
||||
r.Post("/admin/api/gc/preview", ui.handleGCPreview)
|
||||
r.Post("/admin/api/gc/run", ui.handleGCRun)
|
||||
r.Post("/admin/api/gc/reconcile", ui.handleGCReconcile)
|
||||
r.Post("/admin/api/gc/delete-records", ui.handleGCDeleteRecords)
|
||||
r.Post("/admin/api/gc/delete-blobs", ui.handleGCDeleteBlobs)
|
||||
r.Get("/admin/api/gc/status", ui.handleGCStatus)
|
||||
|
||||
// API endpoints (for HTMX)
|
||||
r.Get("/admin/api/stats", ui.handleStatsAPI)
|
||||
|
||||
+122
-80
@@ -21,6 +21,13 @@ type gcTabData struct {
|
||||
LastResultAge string
|
||||
}
|
||||
|
||||
// gcProgressData is the data passed to the gc_progress.html partial
|
||||
type gcProgressData struct {
|
||||
Phase string
|
||||
Message string
|
||||
OpType string
|
||||
}
|
||||
|
||||
// handleGCTab returns the storage/GC tab content (HTMX partial)
|
||||
func (ui *AdminUI) handleGCTab(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
@@ -44,136 +51,171 @@ func (ui *AdminUI) handleGCTab(w http.ResponseWriter, r *http.Request) {
|
||||
ui.renderTemplate(w, "partials/tab_storage.html", data)
|
||||
}
|
||||
|
||||
// handleGCPreview runs a GC preview (analysis only, no mutations)
|
||||
// handleGCPreview starts a GC preview in the background
|
||||
func (ui *AdminUI) handleGCPreview(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
preview, err := ui.gc.Preview(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("GC preview failed", "error", err)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
|
||||
return
|
||||
session := getSessionFromContext(r.Context())
|
||||
|
||||
if ui.gc.StartPreview() {
|
||||
slog.Info("GC preview started via admin panel", "by", session.DID)
|
||||
}
|
||||
|
||||
session := getSessionFromContext(r.Context())
|
||||
slog.Info("GC preview completed via admin panel",
|
||||
"orphanedRecords", len(preview.OrphanedRecords),
|
||||
"orphanedBlobs", len(preview.OrphanedBlobs),
|
||||
"missingRecords", len(preview.MissingRecords),
|
||||
"duration", preview.Duration,
|
||||
"by", session.DID)
|
||||
|
||||
ui.renderTemplate(w, "partials/gc_preview.html", struct {
|
||||
Preview *gc.GCPreview
|
||||
}{Preview: preview})
|
||||
// Whether we just started or it was already running, show progress
|
||||
progress := ui.gc.GetProgress()
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGCRun executes an actual GC run
|
||||
// handleGCRun starts a full GC run in the background
|
||||
func (ui *AdminUI) handleGCRun(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := ui.gc.Run(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("GC run failed", "error", err)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
|
||||
return
|
||||
session := getSessionFromContext(r.Context())
|
||||
|
||||
if ui.gc.StartRun() {
|
||||
slog.Info("GC run started via admin panel", "by", session.DID)
|
||||
}
|
||||
|
||||
session := getSessionFromContext(r.Context())
|
||||
slog.Info("GC run completed via admin panel",
|
||||
"blobsDeleted", result.BlobsDeleted,
|
||||
"bytesReclaimed", result.BytesReclaimed,
|
||||
"recordsDeleted", result.RecordsDeleted,
|
||||
"recordsReconciled", result.RecordsReconciled,
|
||||
"duration", result.Duration,
|
||||
"by", session.DID)
|
||||
|
||||
ui.renderTemplate(w, "partials/gc_result.html", struct {
|
||||
Result *gc.GCResult
|
||||
}{Result: result})
|
||||
progress := ui.gc.GetProgress()
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGCReconcile creates missing layer records without deleting anything
|
||||
// handleGCReconcile starts record reconciliation in the background
|
||||
func (ui *AdminUI) handleGCReconcile(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := ui.gc.Reconcile(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("GC reconcile failed", "error", err)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
|
||||
return
|
||||
session := getSessionFromContext(r.Context())
|
||||
|
||||
if ui.gc.StartReconcile() {
|
||||
slog.Info("GC reconcile started via admin panel", "by", session.DID)
|
||||
}
|
||||
|
||||
session := getSessionFromContext(r.Context())
|
||||
slog.Info("GC reconcile completed via admin panel",
|
||||
"recordsReconciled", result.RecordsReconciled,
|
||||
"duration", result.Duration,
|
||||
"by", session.DID)
|
||||
|
||||
ui.renderTemplate(w, "partials/gc_result.html", struct {
|
||||
Result *gc.GCResult
|
||||
}{Result: result})
|
||||
progress := ui.gc.GetProgress()
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGCDeleteRecords deletes orphaned layer records (no blob deletion)
|
||||
// handleGCDeleteRecords starts orphaned record deletion in the background
|
||||
func (ui *AdminUI) handleGCDeleteRecords(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := ui.gc.DeleteOrphanedRecords(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("GC delete records failed", "error", err)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
|
||||
return
|
||||
session := getSessionFromContext(r.Context())
|
||||
|
||||
if ui.gc.StartDeleteRecords() {
|
||||
slog.Info("GC delete records started via admin panel", "by", session.DID)
|
||||
}
|
||||
|
||||
session := getSessionFromContext(r.Context())
|
||||
slog.Info("GC delete orphaned records completed via admin panel",
|
||||
"recordsDeleted", result.RecordsDeleted,
|
||||
"orphanedRecords", result.OrphanedRecords,
|
||||
"duration", result.Duration,
|
||||
"by", session.DID)
|
||||
|
||||
ui.renderTemplate(w, "partials/gc_result.html", struct {
|
||||
Result *gc.GCResult
|
||||
}{Result: result})
|
||||
progress := ui.gc.GetProgress()
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGCDeleteBlobs walks S3 and deletes unreferenced blobs
|
||||
// handleGCDeleteBlobs starts orphaned blob deletion in the background
|
||||
func (ui *AdminUI) handleGCDeleteBlobs(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := ui.gc.DeleteOrphanedBlobs(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("GC delete blobs failed", "error", err)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()})
|
||||
session := getSessionFromContext(r.Context())
|
||||
|
||||
if ui.gc.StartDeleteBlobs() {
|
||||
slog.Info("GC delete blobs started via admin panel", "by", session.DID)
|
||||
}
|
||||
|
||||
progress := ui.gc.GetProgress()
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGCStatus returns current progress or final results (polled by gc_progress.html)
|
||||
func (ui *AdminUI) handleGCStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if ui.gc == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"})
|
||||
return
|
||||
}
|
||||
|
||||
session := getSessionFromContext(r.Context())
|
||||
slog.Info("GC delete orphaned blobs completed via admin panel",
|
||||
"blobsDeleted", result.BlobsDeleted,
|
||||
"bytesReclaimed", result.BytesReclaimed,
|
||||
"duration", result.Duration,
|
||||
"by", session.DID)
|
||||
progress := ui.gc.GetProgress()
|
||||
|
||||
ui.renderTemplate(w, "partials/gc_result.html", struct {
|
||||
Result *gc.GCResult
|
||||
}{Result: result})
|
||||
// Still running — return progress partial (which will poll again)
|
||||
if progress.Running {
|
||||
ui.renderTemplate(w, "partials/gc_progress.html", gcProgressData{
|
||||
Phase: progress.Phase,
|
||||
Message: progress.Message,
|
||||
OpType: progress.OperationType,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Error — show error
|
||||
if progress.Phase == "error" {
|
||||
slog.Error("GC operation failed", "type", progress.OperationType, "error", progress.Error)
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{progress.Error})
|
||||
return
|
||||
}
|
||||
|
||||
// Complete — render final results based on operation type
|
||||
switch progress.OperationType {
|
||||
case "preview":
|
||||
preview, _ := ui.gc.LastPreview()
|
||||
if preview == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"No preview results available"})
|
||||
return
|
||||
}
|
||||
slog.Info("GC preview completed via admin panel",
|
||||
"orphanedRecords", len(preview.OrphanedRecords),
|
||||
"orphanedBlobs", len(preview.OrphanedBlobs),
|
||||
"missingRecords", len(preview.MissingRecords),
|
||||
"duration", preview.Duration)
|
||||
ui.renderTemplate(w, "partials/gc_preview.html", struct {
|
||||
Preview *gc.GCPreview
|
||||
}{Preview: preview})
|
||||
|
||||
default:
|
||||
result, _ := ui.gc.LastResult()
|
||||
if result == nil {
|
||||
ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"No results available"})
|
||||
return
|
||||
}
|
||||
slog.Info("GC operation completed via admin panel",
|
||||
"type", progress.OperationType,
|
||||
"blobsDeleted", result.BlobsDeleted,
|
||||
"recordsDeleted", result.RecordsDeleted,
|
||||
"recordsReconciled", result.RecordsReconciled,
|
||||
"duration", result.Duration)
|
||||
ui.renderTemplate(w, "partials/gc_result.html", struct {
|
||||
Result *gc.GCResult
|
||||
}{Result: result})
|
||||
}
|
||||
}
|
||||
|
||||
// timeAgo returns a human-readable relative time string
|
||||
|
||||
@@ -142,8 +142,7 @@
|
||||
<button class="btn btn-warning gap-2"
|
||||
hx-post="/admin/api/gc/reconcile"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#gc-loading">
|
||||
hx-swap="innerHTML">
|
||||
{{ icon "file-plus" "size-4" }}
|
||||
Reconcile {{len .Preview.MissingRecords}} Records
|
||||
</button>
|
||||
@@ -153,8 +152,7 @@
|
||||
hx-post="/admin/api/gc/delete-records"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML"
|
||||
hx-confirm="Delete {{len .Preview.OrphanedRecords}} orphaned layer records?"
|
||||
hx-indicator="#gc-loading">
|
||||
hx-confirm="Delete {{len .Preview.OrphanedRecords}} orphaned layer records?">
|
||||
{{ icon "file-x" "size-4" }}
|
||||
Delete {{len .Preview.OrphanedRecords}} Orphaned Records
|
||||
</button>
|
||||
@@ -164,8 +162,7 @@
|
||||
hx-post="/admin/api/gc/delete-blobs"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML"
|
||||
hx-confirm="Delete {{len .Preview.OrphanedBlobs}} orphaned blobs from S3? This cannot be undone."
|
||||
hx-indicator="#gc-loading">
|
||||
hx-confirm="Delete {{len .Preview.OrphanedBlobs}} orphaned blobs from S3? This cannot be undone.">
|
||||
{{ icon "trash-2" "size-4" }}
|
||||
Delete {{len .Preview.OrphanedBlobs}} Orphaned Blobs
|
||||
</button>
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{{define "partials/gc_progress.html"}}
|
||||
<div hx-get="/admin/api/gc/status"
|
||||
hx-trigger="load delay:2s"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML">
|
||||
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
|
||||
<span class="loading loading-spinner loading-md text-primary"></span>
|
||||
<div>
|
||||
<p class="font-medium">{{.Message}}</p>
|
||||
<p class="text-sm text-base-content/50">
|
||||
{{if eq .Phase "manifests"}}Fetching manifests from user PDS instances...
|
||||
{{else if eq .Phase "records"}}Scanning layer records...
|
||||
{{else if eq .Phase "blobs"}}Walking S3 storage...
|
||||
{{else if eq .Phase "deleting"}}Processing changes...
|
||||
{{else}}Working...
|
||||
{{end}}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -33,9 +33,9 @@
|
||||
{{if .RepoStatus}}
|
||||
{{if .RepoStatus.Active}}
|
||||
{{if eq .RepoStatus.Rev $.CurrentRev}}
|
||||
<span class="text-sm text-success">Known (rev: {{truncate .RepoStatus.Rev 12}})</span>
|
||||
<span class="text-sm text-success">Known (rev: {{.RepoStatus.Rev}})</span>
|
||||
{{else}}
|
||||
<span class="text-sm text-warning">Behind (rev: {{truncate .RepoStatus.Rev 12}})</span>
|
||||
<span class="text-sm text-warning">Behind (rev: {{.RepoStatus.Rev}})</span>
|
||||
{{end}}
|
||||
{{else}}
|
||||
<span class="text-sm text-warning">Inactive</span>
|
||||
|
||||
@@ -37,24 +37,26 @@
|
||||
hx-post="/admin/api/gc/preview"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#gc-loading"
|
||||
{{if .Running}}disabled{{end}}>
|
||||
{{ icon "search" "size-4" }}
|
||||
Scan for Orphans
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="gc-loading" class="htmx-indicator mb-4">
|
||||
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
|
||||
<span class="loading loading-spinner loading-md text-primary"></span>
|
||||
<div>
|
||||
<p class="font-medium">Scanning storage...</p>
|
||||
<p class="text-sm text-base-content/50">This may take a few minutes for large holds.</p>
|
||||
<div id="gc-results">
|
||||
{{if .Running}}
|
||||
<div hx-get="/admin/api/gc/status"
|
||||
hx-trigger="load delay:2s"
|
||||
hx-target="#gc-results"
|
||||
hx-swap="innerHTML">
|
||||
<div class="flex items-center gap-3 p-4 bg-base-100 rounded-lg shadow-sm">
|
||||
<span class="loading loading-spinner loading-md text-primary"></span>
|
||||
<div>
|
||||
<p class="font-medium">Operation in progress...</p>
|
||||
<p class="text-sm text-base-content/50">Checking status...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="gc-results">
|
||||
<!-- Preview or run results will be swapped in here -->
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
+258
-15
@@ -5,8 +5,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -82,6 +84,16 @@ type GarbageCollector struct {
|
||||
lastPreviewAt time.Time
|
||||
lastResult *GCResult
|
||||
lastResultAt time.Time
|
||||
|
||||
// Progress tracking for background operations
|
||||
phase string // "manifests", "records", "blobs", "deleting", "complete", "error"
|
||||
progressMsg string // e.g. "Fetching manifests (35/69 users)"
|
||||
operationType string // "preview", "run", "reconcile", "delete-records", "delete-blobs"
|
||||
lastError error
|
||||
|
||||
// predecessorCache caches holdDID → "is this a predecessor of our hold?"
|
||||
// A predecessor is a hold whose captain record has a successor label set.
|
||||
predecessorCache map[string]bool
|
||||
}
|
||||
|
||||
// GCResult contains statistics from a GC run
|
||||
@@ -119,11 +131,12 @@ type analysisResult struct {
|
||||
// NewGarbageCollector creates a new GC instance
|
||||
func NewGarbageCollector(holdPDS *pds.HoldPDS, s3svc *s3.S3Service, cfg Config) *GarbageCollector {
|
||||
return &GarbageCollector{
|
||||
pds: holdPDS,
|
||||
s3: s3svc,
|
||||
cfg: cfg,
|
||||
logger: slog.Default().With("component", "gc"),
|
||||
stopCh: make(chan struct{}),
|
||||
pds: holdPDS,
|
||||
s3: s3svc,
|
||||
cfg: cfg,
|
||||
logger: slog.Default().With("component", "gc"),
|
||||
stopCh: make(chan struct{}),
|
||||
predecessorCache: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,6 +158,109 @@ func (gc *GarbageCollector) finish() {
|
||||
gc.mu.Unlock()
|
||||
}
|
||||
|
||||
// setProgress updates the progress fields (thread-safe).
|
||||
func (gc *GarbageCollector) setProgress(phase, msg, opType string) {
|
||||
gc.mu.Lock()
|
||||
gc.phase = phase
|
||||
gc.progressMsg = msg
|
||||
gc.operationType = opType
|
||||
if phase != "error" {
|
||||
gc.lastError = nil
|
||||
}
|
||||
gc.mu.Unlock()
|
||||
}
|
||||
|
||||
// GCProgress holds a snapshot of the current GC operation progress.
|
||||
type GCProgress struct {
|
||||
Phase string // "manifests", "records", "blobs", "deleting", "complete", "error"
|
||||
Message string
|
||||
OperationType string // "preview", "run", "reconcile", "delete-records", "delete-blobs"
|
||||
Running bool
|
||||
Error string
|
||||
}
|
||||
|
||||
// GetProgress returns the current progress state.
|
||||
func (gc *GarbageCollector) GetProgress() GCProgress {
|
||||
gc.mu.Lock()
|
||||
defer gc.mu.Unlock()
|
||||
p := GCProgress{
|
||||
Phase: gc.phase,
|
||||
Message: gc.progressMsg,
|
||||
OperationType: gc.operationType,
|
||||
Running: gc.running,
|
||||
}
|
||||
if gc.lastError != nil {
|
||||
p.Error = gc.lastError.Error()
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// startBackground is the common pattern for launching a GC operation in the background.
|
||||
// Returns false if already running.
|
||||
func (gc *GarbageCollector) startBackground(opType, initialPhase, initialMsg string, fn func(ctx context.Context) error) bool {
|
||||
if !gc.tryStart() {
|
||||
return false
|
||||
}
|
||||
gc.setProgress(initialPhase, initialMsg, opType)
|
||||
go func() {
|
||||
defer gc.finish()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if err := fn(ctx); err != nil {
|
||||
gc.mu.Lock()
|
||||
gc.phase = "error"
|
||||
gc.progressMsg = err.Error()
|
||||
gc.lastError = err
|
||||
gc.mu.Unlock()
|
||||
gc.logger.Error("GC operation failed", "type", opType, "error", err)
|
||||
return
|
||||
}
|
||||
gc.setProgress("complete", "", opType)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// StartPreview launches a GC preview in the background.
|
||||
func (gc *GarbageCollector) StartPreview() bool {
|
||||
return gc.startBackground("preview", "manifests", "Starting preview...", func(ctx context.Context) error {
|
||||
_, err := gc.doPreview(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// StartRun launches a full GC run in the background.
|
||||
func (gc *GarbageCollector) StartRun() bool {
|
||||
return gc.startBackground("run", "manifests", "Starting GC run...", func(ctx context.Context) error {
|
||||
_, err := gc.doRun(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// StartReconcile launches record reconciliation in the background.
|
||||
func (gc *GarbageCollector) StartReconcile() bool {
|
||||
return gc.startBackground("reconcile", "deleting", "Reconciling missing records...", func(ctx context.Context) error {
|
||||
_, err := gc.doReconcile(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// StartDeleteRecords launches orphaned record deletion in the background.
|
||||
func (gc *GarbageCollector) StartDeleteRecords() bool {
|
||||
return gc.startBackground("delete-records", "deleting", "Deleting orphaned records...", func(ctx context.Context) error {
|
||||
_, err := gc.doDeleteOrphanedRecords(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// StartDeleteBlobs launches orphaned blob deletion in the background.
|
||||
func (gc *GarbageCollector) StartDeleteBlobs() bool {
|
||||
return gc.startBackground("delete-blobs", "manifests", "Analyzing records before blob deletion...", func(ctx context.Context) error {
|
||||
_, err := gc.doDeleteOrphanedBlobs(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// Start begins the GC background process with a nightly ticker
|
||||
func (gc *GarbageCollector) Start(ctx context.Context) {
|
||||
if !gc.cfg.Enabled {
|
||||
@@ -194,7 +310,11 @@ func (gc *GarbageCollector) Run(ctx context.Context) (*GCResult, error) {
|
||||
return nil, fmt.Errorf("GC operation already in progress")
|
||||
}
|
||||
defer gc.finish()
|
||||
return gc.doRun(ctx)
|
||||
}
|
||||
|
||||
// doRun is the internal implementation of Run (caller must hold the running lock).
|
||||
func (gc *GarbageCollector) doRun(ctx context.Context) (*GCResult, error) {
|
||||
start := time.Now()
|
||||
result := &GCResult{}
|
||||
|
||||
@@ -216,17 +336,20 @@ func (gc *GarbageCollector) Run(ctx context.Context) (*GCResult, error) {
|
||||
"missingRecords", len(analysis.missingDetails))
|
||||
|
||||
// Reconcile: create missing layer records
|
||||
gc.setProgress("deleting", "Reconciling missing records...", "run")
|
||||
if len(analysis.missingDetails) > 0 {
|
||||
gc.reconcileMissingRecords(ctx, analysis.missingDetails, result)
|
||||
}
|
||||
|
||||
// Phase 2: Delete orphaned layer records
|
||||
gc.setProgress("deleting", "Deleting orphaned records...", "run")
|
||||
if err := gc.deleteOrphanedRecords(ctx, analysis.orphanedRkeys, result); err != nil {
|
||||
gc.logger.Error("Phase 2 (delete orphaned records) failed", "error", err)
|
||||
// Continue to phase 3 - we can still clean up blobs
|
||||
}
|
||||
|
||||
// Phase 3: Walk storage and delete unreferenced blobs
|
||||
gc.setProgress("blobs", "Deleting orphaned blobs...", "run")
|
||||
if err := gc.deleteOrphanedBlobs(ctx, analysis.referenced, result); err != nil {
|
||||
return nil, fmt.Errorf("phase 3 (delete orphaned blobs) failed: %w", err)
|
||||
}
|
||||
@@ -250,7 +373,11 @@ func (gc *GarbageCollector) Preview(ctx context.Context) (*GCPreview, error) {
|
||||
return nil, fmt.Errorf("GC operation already in progress")
|
||||
}
|
||||
defer gc.finish()
|
||||
return gc.doPreview(ctx)
|
||||
}
|
||||
|
||||
// doPreview is the internal implementation of Preview (caller must hold the running lock).
|
||||
func (gc *GarbageCollector) doPreview(ctx context.Context) (*GCPreview, error) {
|
||||
start := time.Now()
|
||||
|
||||
gc.logger.Info("Starting GC preview")
|
||||
@@ -262,6 +389,7 @@ func (gc *GarbageCollector) Preview(ctx context.Context) (*GCPreview, error) {
|
||||
}
|
||||
|
||||
// Phase 2: Walk S3 to find orphaned blobs (read-only)
|
||||
gc.setProgress("blobs", "Walking S3 storage...", "preview")
|
||||
orphanedBlobs, totalBlobs, err := gc.scanOrphanedBlobDetails(ctx, analysis.referenced)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan orphaned blobs: %w", err)
|
||||
@@ -303,7 +431,11 @@ func (gc *GarbageCollector) Reconcile(ctx context.Context) (*GCResult, error) {
|
||||
return nil, fmt.Errorf("GC operation already in progress")
|
||||
}
|
||||
defer gc.finish()
|
||||
return gc.doReconcile(ctx)
|
||||
}
|
||||
|
||||
// doReconcile is the internal implementation of Reconcile (caller must hold the running lock).
|
||||
func (gc *GarbageCollector) doReconcile(ctx context.Context) (*GCResult, error) {
|
||||
gc.mu.Lock()
|
||||
preview := gc.lastPreview
|
||||
gc.mu.Unlock()
|
||||
@@ -337,7 +469,11 @@ func (gc *GarbageCollector) DeleteOrphanedRecords(ctx context.Context) (*GCResul
|
||||
return nil, fmt.Errorf("GC operation already in progress")
|
||||
}
|
||||
defer gc.finish()
|
||||
return gc.doDeleteOrphanedRecords(ctx)
|
||||
}
|
||||
|
||||
// doDeleteOrphanedRecords is the internal implementation (caller must hold the running lock).
|
||||
func (gc *GarbageCollector) doDeleteOrphanedRecords(ctx context.Context) (*GCResult, error) {
|
||||
gc.mu.Lock()
|
||||
preview := gc.lastPreview
|
||||
gc.mu.Unlock()
|
||||
@@ -381,7 +517,11 @@ func (gc *GarbageCollector) DeleteOrphanedBlobs(ctx context.Context) (*GCResult,
|
||||
return nil, fmt.Errorf("GC operation already in progress")
|
||||
}
|
||||
defer gc.finish()
|
||||
return gc.doDeleteOrphanedBlobs(ctx)
|
||||
}
|
||||
|
||||
// doDeleteOrphanedBlobs is the internal implementation (caller must hold the running lock).
|
||||
func (gc *GarbageCollector) doDeleteOrphanedBlobs(ctx context.Context) (*GCResult, error) {
|
||||
start := time.Now()
|
||||
result := &GCResult{}
|
||||
|
||||
@@ -395,6 +535,7 @@ func (gc *GarbageCollector) DeleteOrphanedBlobs(ctx context.Context) (*GCResult,
|
||||
|
||||
result.ReferencedBlobs = int64(len(analysis.referenced))
|
||||
|
||||
gc.setProgress("blobs", "Deleting orphaned blobs...", "delete-blobs")
|
||||
if err := gc.deleteOrphanedBlobs(ctx, analysis.referenced, result); err != nil {
|
||||
return nil, fmt.Errorf("delete orphaned blobs: %w", err)
|
||||
}
|
||||
@@ -432,8 +573,10 @@ func (gc *GarbageCollector) analyzeRecords(ctx context.Context) (*analysisResult
|
||||
knownManifests := make(map[string]*manifestInfo)
|
||||
fetchedUsers := make(map[string]bool)
|
||||
|
||||
totalUsers := len(userDIDs)
|
||||
for _, did := range userDIDs {
|
||||
result.usersChecked++
|
||||
gc.setProgress("manifests", fmt.Sprintf("Fetching manifests (%d/%d users)", result.usersChecked, totalUsers), gc.operationType)
|
||||
|
||||
manifests, err := gc.fetchUserManifests(ctx, did)
|
||||
if err != nil {
|
||||
@@ -464,6 +607,7 @@ func (gc *GarbageCollector) analyzeRecords(ctx context.Context) (*analysisResult
|
||||
"fetchedUsers", len(fetchedUsers))
|
||||
|
||||
// Step 3: Scan existing layer records to find orphans and build coveredPairs
|
||||
gc.setProgress("records", "Scanning layer records...", gc.operationType)
|
||||
coveredPairs := make(map[string]bool)
|
||||
|
||||
cursor := ""
|
||||
@@ -599,7 +743,7 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
|
||||
|
||||
// reconcileMissingRecords creates layer records for manifest+layer pairs that are missing.
|
||||
func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing []MissingRecordDetail, result *GCResult) {
|
||||
for _, m := range missing {
|
||||
for i, m := range missing {
|
||||
record := atproto.NewLayerRecord(
|
||||
m.Digest,
|
||||
m.Size,
|
||||
@@ -615,10 +759,16 @@ func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing
|
||||
continue
|
||||
}
|
||||
result.RecordsReconciled++
|
||||
gc.logger.Info("Created missing layer record",
|
||||
"digest", m.Digest,
|
||||
"manifest", m.ManifestURI,
|
||||
"user", m.UserDID)
|
||||
if result.RecordsReconciled%100 == 0 {
|
||||
gc.logger.Info("Reconciliation progress",
|
||||
"created", result.RecordsReconciled,
|
||||
"total", len(missing))
|
||||
}
|
||||
|
||||
// Throttle to avoid flooding the firehose (~100 records/sec)
|
||||
if i < len(missing)-1 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
if result.RecordsReconciled > 0 {
|
||||
@@ -751,20 +901,113 @@ func (gc *GarbageCollector) fetchUserManifestsFromEndpoint(ctx context.Context,
|
||||
return manifests, nil
|
||||
}
|
||||
|
||||
// manifestBelongsToHold checks if a manifest references this hold via HoldDID or legacy HoldEndpoint.
|
||||
// manifestBelongsToHold checks if a manifest references this hold via HoldDID,
|
||||
// legacy HoldEndpoint, or a predecessor hold that has been migrated.
|
||||
func (gc *GarbageCollector) manifestBelongsToHold(ctx context.Context, manifest *atproto.ManifestRecord, holdDID string) bool {
|
||||
if manifest.HoldDID == holdDID {
|
||||
manifestHoldDID := manifest.HoldDID
|
||||
|
||||
// Direct match
|
||||
if manifestHoldDID == holdDID {
|
||||
return true
|
||||
}
|
||||
// Legacy: check holdEndpoint converted to DID
|
||||
if manifest.HoldEndpoint != "" {
|
||||
|
||||
// Legacy: resolve holdEndpoint to DID
|
||||
if manifestHoldDID == "" && manifest.HoldEndpoint != "" {
|
||||
resolved, err := atproto.ResolveHoldDID(ctx, manifest.HoldEndpoint)
|
||||
if err != nil {
|
||||
gc.logger.Debug("Failed to resolve hold DID from legacy endpoint", "holdEndpoint", manifest.HoldEndpoint, "error", err)
|
||||
return false
|
||||
}
|
||||
return resolved == holdDID
|
||||
manifestHoldDID = resolved
|
||||
if manifestHoldDID == holdDID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if manifestHoldDID == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if the manifest's hold is a predecessor (has a successor label set)
|
||||
return gc.isPredecessorHold(ctx, manifestHoldDID)
|
||||
}
|
||||
|
||||
// isPredecessorHold checks if the given holdDID is a predecessor of this hold
|
||||
// by fetching its captain record and checking for a successor label.
|
||||
// Results are cached to avoid repeated network calls.
|
||||
func (gc *GarbageCollector) isPredecessorHold(ctx context.Context, holdDID string) bool {
|
||||
if gc.predecessorCache == nil {
|
||||
gc.predecessorCache = make(map[string]bool)
|
||||
}
|
||||
|
||||
if isPredecessor, cached := gc.predecessorCache[holdDID]; cached {
|
||||
return isPredecessor
|
||||
}
|
||||
|
||||
isPredecessor := gc.checkPredecessor(ctx, holdDID)
|
||||
gc.predecessorCache[holdDID] = isPredecessor
|
||||
return isPredecessor
|
||||
}
|
||||
|
||||
// checkPredecessor fetches a hold's captain record to check if it has a successor label
|
||||
// (meaning the hold has been migrated/retired and its blobs are served by this hold).
|
||||
func (gc *GarbageCollector) checkPredecessor(ctx context.Context, holdDID string) bool {
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
holdURL, err := atproto.ResolveHoldURL(fetchCtx, holdDID)
|
||||
if err != nil {
|
||||
gc.logger.Debug("GC: failed to resolve predecessor hold URL",
|
||||
"holdDID", holdDID, "error", err)
|
||||
return false
|
||||
}
|
||||
|
||||
recordURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=self",
|
||||
holdURL,
|
||||
url.QueryEscape(holdDID),
|
||||
url.QueryEscape(atproto.CaptainCollection),
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(fetchCtx, "GET", recordURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
gc.logger.Debug("GC: failed to fetch predecessor captain record",
|
||||
"holdDID", holdDID, "error", err)
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var envelope struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var captain atproto.CaptainRecord
|
||||
if err := json.Unmarshal(envelope.Value, &captain); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if captain.Successor != "" {
|
||||
gc.logger.Info("GC: discovered predecessor hold (has successor label)",
|
||||
"holdDID", holdDID, "successor", captain.Successor)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user