remove distribution from hold, add vulnerability scanning in appview.

1. Removing distribution/distribution from the Hold Service (biggest change)
  The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service:
  - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go
  - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which
  broke SigV4 signatures)
  - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver
  - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method
  - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file)
2. Vulnerability Scan UI in AppView (new feature)
  Displays scan results from the hold's PDS on the repository page:
  - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports
  - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table)
  - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links)
  - Repository page: Lazy-loads scan badges per manifest via HTMX
  - Tests: ~590 lines of test coverage for both handlers
3. S3 Diagnostic Tool
  New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output.
4. Deployment Tooling
  - New syncServiceUnit() for comparing/updating systemd units on servers
  - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload
5. DB Migration
  0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration.
6. Documentation
  - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory
  - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md
  - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side
7. go.mod
  aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).
This commit is contained in:
Evan Jarrett
2026-02-13 15:26:24 -06:00
parent 434a5f1eee
commit de02e1f046
38 changed files with 3134 additions and 962 deletions
@@ -0,0 +1,25 @@
description: Rebuild hold_captain_records to match schema.sql (provider→successor rename was missed when migration 0010 was recorded but not executed on a fresh DB)
query: |
-- Recreate table to match schema.sql exactly
CREATE TABLE IF NOT EXISTS hold_captain_records_new (
hold_did TEXT PRIMARY KEY,
owner_did TEXT NOT NULL,
public BOOLEAN NOT NULL,
allow_all_crew BOOLEAN NOT NULL,
deployed_at TEXT,
region TEXT,
successor TEXT,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Copy data (only guaranteed-common columns; successor will be NULL)
INSERT OR IGNORE INTO hold_captain_records_new (hold_did, owner_did, public, allow_all_crew, deployed_at, region, updated_at)
SELECT hold_did, owner_did, public, allow_all_crew, deployed_at, region, updated_at
FROM hold_captain_records;
-- Swap tables
DROP TABLE hold_captain_records;
ALTER TABLE hold_captain_records_new RENAME TO hold_captain_records;
-- Recreate index
CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at);
+125
View File
@@ -0,0 +1,125 @@
package handlers
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/url"
"strings"
"time"
"atcr.io/pkg/atproto"
)
// ScanResultHandler handles HTMX requests for vulnerability scan badges.
// Returns an HTML fragment (vuln-badge partial) that replaces the placeholder span.
type ScanResultHandler struct {
BaseUIHandler
}
// vulnBadgeData is the template data for the vuln-badge partial.
type vulnBadgeData struct {
Critical int64
High int64
Medium int64
Low int64
Total int64
ScannedAt string
Found bool // true if scan record exists
Error bool // true if hold unreachable or error
Digest string // for the detail modal link
HoldEndpoint string // for the detail modal link
}
func (h *ScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
digest := r.URL.Query().Get("digest")
holdEndpoint := r.URL.Query().Get("holdEndpoint")
if digest == "" || holdEndpoint == "" {
// Missing params — render nothing
w.Header().Set("Content-Type", "text/html")
return
}
// Derive hold DID from endpoint URL
holdDID := atproto.ResolveHoldDIDFromURL(holdEndpoint)
if holdDID == "" {
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
// Compute rkey from digest (strip sha256: prefix)
rkey := strings.TrimPrefix(digest, "sha256:")
// Fetch scan record from hold's PDS
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
holdEndpoint,
url.QueryEscape(holdDID),
url.QueryEscape(atproto.ScanCollection),
url.QueryEscape(rkey),
)
req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil)
if err != nil {
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
// Hold unreachable or timeout — render nothing
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
// No scan record — scanning disabled or not yet scanned. Render nothing.
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
if resp.StatusCode != http.StatusOK {
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
// Parse the getRecord response envelope
var envelope struct {
Value json.RawMessage `json:"value"`
}
if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil {
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
var scanRecord atproto.ScanRecord
if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil {
h.renderBadge(w, vulnBadgeData{Error: true})
return
}
h.renderBadge(w, vulnBadgeData{
Critical: scanRecord.Critical,
High: scanRecord.High,
Medium: scanRecord.Medium,
Low: scanRecord.Low,
Total: scanRecord.Total,
ScannedAt: scanRecord.ScannedAt,
Found: true,
Digest: digest,
HoldEndpoint: holdEndpoint,
})
}
func (h *ScanResultHandler) renderBadge(w http.ResponseWriter, data vulnBadgeData) {
w.Header().Set("Content-Type", "text/html")
if err := h.Templates.ExecuteTemplate(w, "vuln-badge", data); err != nil {
slog.Warn("Failed to render vuln badge", "error", err)
}
}
+255
View File
@@ -0,0 +1,255 @@
package handlers_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"atcr.io/pkg/appview"
"atcr.io/pkg/appview/handlers"
)
// mockScanRecord returns a getRecord JSON envelope wrapping a scan record
func mockScanRecord(critical, high, medium, low, total int64) string {
record := map[string]any{
"$type": "io.atcr.hold.scan",
"manifest": "at://did:plc:test/io.atcr.manifest/abc123",
"repository": "myapp",
"userDid": "did:plc:test",
"critical": critical,
"high": high,
"medium": medium,
"low": low,
"total": total,
"scannerVersion": "atcr-scanner-v1.0.0",
"scannedAt": "2025-01-15T10:30:00Z",
}
envelope := map[string]any{
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
"cid": "bafyreiabc123",
"value": record,
}
b, _ := json.Marshal(envelope)
return string(b)
}
func setupScanResultHandler(t *testing.T, holdURL string) *handlers.ScanResultHandler {
t.Helper()
templates, err := appview.Templates(nil)
if err != nil {
t.Fatalf("Failed to load templates: %v", err)
}
return &handlers.ScanResultHandler{
BaseUIHandler: handlers.BaseUIHandler{
Templates: templates,
},
}
}
func TestScanResult_WithVulnerabilities(t *testing.T) {
// Mock hold that returns a scan record with vulnerabilities
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecord(2, 5, 10, 3, 20)))
}))
defer hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Errorf("Expected status %d, got %d", http.StatusOK, rr.Code)
}
body := rr.Body.String()
// Should contain severity badges
if !strings.Contains(body, "badge-error") {
t.Error("Expected body to contain badge-error for critical vulnerabilities")
}
if !strings.Contains(body, "C:2") {
t.Error("Expected body to contain 'C:2' for critical count")
}
if !strings.Contains(body, "badge-warning") {
t.Error("Expected body to contain badge-warning for high vulnerabilities")
}
if !strings.Contains(body, "H:5") {
t.Error("Expected body to contain 'H:5' for high count")
}
if !strings.Contains(body, "M:10") {
t.Error("Expected body to contain 'M:10' for medium count")
}
if !strings.Contains(body, "L:3") {
t.Error("Expected body to contain 'L:3' for low count")
}
// Should be clickable (has openVulnDetails)
if !strings.Contains(body, "openVulnDetails") {
t.Error("Expected body to contain openVulnDetails click handler")
}
}
func TestScanResult_Clean(t *testing.T) {
// Mock hold that returns a scan record with zero vulnerabilities
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecord(0, 0, 0, 0, 0)))
}))
defer hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, "Clean") {
t.Error("Expected body to contain 'Clean' for zero-vulnerability scan")
}
if !strings.Contains(body, "badge-success") {
t.Error("Expected body to contain badge-success for clean scan")
}
// Should NOT be clickable
if strings.Contains(body, "openVulnDetails") {
t.Error("Clean badge should not have openVulnDetails click handler")
}
}
func TestScanResult_NotFound(t *testing.T) {
// Mock hold that returns 404 (no scan record — scanning disabled or not yet scanned)
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "record not found", http.StatusNotFound)
}))
defer hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
// 404 = no scan record. Should render NOTHING — not "Scan pending".
if body != "" {
t.Errorf("Expected empty body for 404, got: %q", body)
}
}
func TestScanResult_HoldError(t *testing.T) {
// Mock hold that returns 500
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "internal error", http.StatusInternalServerError)
}))
defer hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for hold error, got: %q", body)
}
}
func TestScanResult_HoldUnreachable(t *testing.T) {
// Use a server that's already closed (unreachable)
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for unreachable hold, got: %q", body)
}
}
func TestScanResult_MissingParams(t *testing.T) {
handler := setupScanResultHandler(t, "")
// No params at all
req := httptest.NewRequest("GET", "/api/scan-result", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for missing params, got: %q", body)
}
}
func TestScanResult_MissingDigest(t *testing.T) {
handler := setupScanResultHandler(t, "")
req := httptest.NewRequest("GET", "/api/scan-result?holdEndpoint=https://hold.example.com", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for missing digest, got: %q", body)
}
}
func TestScanResult_MissingHoldEndpoint(t *testing.T) {
handler := setupScanResultHandler(t, "")
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for missing holdEndpoint, got: %q", body)
}
}
func TestScanResult_OnlyCriticalShown(t *testing.T) {
// Only critical vulns, no high/medium/low
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecord(3, 0, 0, 0, 3)))
}))
defer hold.Close()
handler := setupScanResultHandler(t, hold.URL)
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, "C:3") {
t.Error("Expected body to contain 'C:3'")
}
// Zero-count badges should NOT appear
if strings.Contains(body, "H:0") {
t.Error("Should not contain 'H:0' for zero high count")
}
if strings.Contains(body, "M:0") {
t.Error("Should not contain 'M:0' for zero medium count")
}
if strings.Contains(body, "L:0") {
t.Error("Should not contain 'L:0' for zero low count")
}
}
+244
View File
@@ -0,0 +1,244 @@
package handlers
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/url"
"sort"
"strings"
"time"
"atcr.io/pkg/atproto"
)
// VulnDetailsHandler handles requests for the vulnerability detail modal content.
// Returns an HTML fragment (vuln-details partial) for insertion into the modal body.
type VulnDetailsHandler struct {
BaseUIHandler
}
// grypeReport is the minimal Grype JSON structure we need.
type grypeReport struct {
Matches []grypeMatch `json:"matches"`
}
type grypeMatch struct {
Vulnerability grypeVuln `json:"vulnerability"`
Artifact grypeArtifact `json:"artifact"`
}
type grypeVuln struct {
ID string `json:"id"`
Severity string `json:"severity"`
Fix grypeFix `json:"fix"`
}
type grypeFix struct {
Versions []string `json:"versions"`
State string `json:"state"`
}
type grypeArtifact struct {
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
}
// vulnDetailsData is the template data for the vuln-details partial.
type vulnDetailsData struct {
Matches []vulnMatch
Summary vulnSummary
Error string // non-empty if something went wrong
ScannedAt string
}
type vulnMatch struct {
CVEURL string
CVEID string
Severity string // Critical, High, Medium, Low, Negligible, Unknown
Package string
Version string
FixedIn string
Type string // deb, npm, gem, etc.
}
type vulnSummary struct {
Critical int64
High int64
Medium int64
Low int64
Total int64
}
// severityOrder maps severity strings to sort order (lower = more severe).
var severityOrder = map[string]int{
"Critical": 0,
"High": 1,
"Medium": 2,
"Low": 3,
"Negligible": 4,
"Unknown": 5,
}
func (h *VulnDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
digest := r.URL.Query().Get("digest")
holdEndpoint := r.URL.Query().Get("holdEndpoint")
if digest == "" || holdEndpoint == "" {
h.renderDetails(w, vulnDetailsData{Error: "Missing required parameters"})
return
}
holdDID := atproto.ResolveHoldDIDFromURL(holdEndpoint)
if holdDID == "" {
h.renderDetails(w, vulnDetailsData{Error: "Could not resolve hold identity"})
return
}
rkey := strings.TrimPrefix(digest, "sha256:")
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
// Step 1: Fetch the scan record to get the VulnReportBlob CID
scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
holdEndpoint,
url.QueryEscape(holdDID),
url.QueryEscape(atproto.ScanCollection),
url.QueryEscape(rkey),
)
req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil)
if err != nil {
h.renderDetails(w, vulnDetailsData{Error: "Failed to build request"})
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
h.renderDetails(w, vulnDetailsData{Error: "Hold service unreachable"})
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
h.renderDetails(w, vulnDetailsData{Error: "No scan record found"})
return
}
var envelope struct {
Value json.RawMessage `json:"value"`
}
if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil {
h.renderDetails(w, vulnDetailsData{Error: "Failed to parse scan record"})
return
}
var scanRecord atproto.ScanRecord
if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil {
h.renderDetails(w, vulnDetailsData{Error: "Failed to parse scan record"})
return
}
summary := vulnSummary{
Critical: scanRecord.Critical,
High: scanRecord.High,
Medium: scanRecord.Medium,
Low: scanRecord.Low,
Total: scanRecord.Total,
}
// Step 2: Fetch the vulnerability report blob
if scanRecord.VulnReportBlob == nil || scanRecord.VulnReportBlob.Ref.String() == "" {
h.renderDetails(w, vulnDetailsData{
Summary: summary,
ScannedAt: scanRecord.ScannedAt,
Error: "No detailed vulnerability report available. Only summary counts were recorded.",
})
return
}
blobCID := scanRecord.VulnReportBlob.Ref.String()
blobURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s",
holdEndpoint,
url.QueryEscape(holdDID),
url.QueryEscape(blobCID),
)
blobReq, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil)
if err != nil {
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to build blob request"})
return
}
blobResp, err := http.DefaultClient.Do(blobReq)
if err != nil {
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to fetch vulnerability report"})
return
}
defer blobResp.Body.Close()
if blobResp.StatusCode != http.StatusOK {
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Vulnerability report not accessible"})
return
}
// Step 3: Parse the Grype JSON
var report grypeReport
if err := json.NewDecoder(blobResp.Body).Decode(&report); err != nil {
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to parse vulnerability report"})
return
}
// Convert to template data
matches := make([]vulnMatch, 0, len(report.Matches))
for _, m := range report.Matches {
fixedIn := ""
if len(m.Vulnerability.Fix.Versions) > 0 {
fixedIn = strings.Join(m.Vulnerability.Fix.Versions, ", ")
}
cveURL := ""
if strings.HasPrefix(m.Vulnerability.ID, "CVE-") {
cveURL = "https://nvd.nist.gov/vuln/detail/" + m.Vulnerability.ID
} else if strings.HasPrefix(m.Vulnerability.ID, "GHSA-") {
cveURL = "https://github.com/advisories/" + m.Vulnerability.ID
}
matches = append(matches, vulnMatch{
CVEID: m.Vulnerability.ID,
CVEURL: cveURL,
Severity: m.Vulnerability.Severity,
Package: m.Artifact.Name,
Version: m.Artifact.Version,
FixedIn: fixedIn,
Type: m.Artifact.Type,
})
}
// Sort by severity (critical first)
sort.Slice(matches, func(i, j int) bool {
oi := severityOrder[matches[i].Severity]
oj := severityOrder[matches[j].Severity]
if oi != oj {
return oi < oj
}
return matches[i].CVEID < matches[j].CVEID
})
h.renderDetails(w, vulnDetailsData{
Matches: matches,
Summary: summary,
ScannedAt: scanRecord.ScannedAt,
})
}
func (h *VulnDetailsHandler) renderDetails(w http.ResponseWriter, data vulnDetailsData) {
w.Header().Set("Content-Type", "text/html")
if err := h.Templates.ExecuteTemplate(w, "vuln-details", data); err != nil {
slog.Warn("Failed to render vuln details", "error", err)
}
}
+336
View File
@@ -0,0 +1,336 @@
package handlers_test
import (
"crypto/sha256"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"atcr.io/pkg/appview"
"atcr.io/pkg/appview/handlers"
lexutil "github.com/bluesky-social/indigo/lex/util"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
// mockGrypeReport returns a minimal Grype JSON report
func mockGrypeReport() string {
report := map[string]any{
"matches": []map[string]any{
{
"vulnerability": map[string]any{
"id": "CVE-2024-1234",
"severity": "Critical",
"fix": map[string]any{"versions": []string{"1.2.4"}, "state": "fixed"},
},
"artifact": map[string]any{
"name": "libssl",
"version": "1.1.1",
"type": "deb",
},
},
{
"vulnerability": map[string]any{
"id": "CVE-2024-5678",
"severity": "Low",
"fix": map[string]any{"versions": []string{}, "state": "not-fixed"},
},
"artifact": map[string]any{
"name": "zlib",
"version": "1.2.11",
"type": "deb",
},
},
{
"vulnerability": map[string]any{
"id": "GHSA-abcd-efgh-ijkl",
"severity": "High",
"fix": map[string]any{"versions": []string{"2.0.0"}, "state": "fixed"},
},
"artifact": map[string]any{
"name": "express",
"version": "4.17.1",
"type": "npm",
},
},
},
}
b, _ := json.Marshal(report)
return string(b)
}
// testCID creates a valid CIDv1 from arbitrary data (for test fixtures)
func testCID(data string) cid.Cid {
hash := sha256.Sum256([]byte(data))
mh, _ := multihash.Encode(hash[:], multihash.SHA2_256)
return cid.NewCidV1(0x55, mh) // raw codec
}
// mockScanRecordWithBlob returns a getRecord envelope with a VulnReportBlob reference.
// Uses a real CID so LexBlob JSON unmarshaling works correctly.
func mockScanRecordWithBlob(critical, high, medium, low, total int64) string {
blobCID := testCID("test-vuln-report")
// Build the record using the actual LexBlob type for correct JSON format
blob := &lexutil.LexBlob{
Ref: lexutil.LexLink(blobCID),
MimeType: "application/vnd.atcr.vulnerabilities+json",
Size: 12345,
}
// Marshal blob separately to get the canonical JSON format
blobJSON, _ := json.Marshal(blob)
// Build the full record as a map, inserting the pre-marshaled blob
record := map[string]json.RawMessage{
"$type": jsonStr("io.atcr.hold.scan"),
"manifest": jsonStr("at://did:plc:test/io.atcr.manifest/abc123"),
"repository": jsonStr("myapp"),
"userDid": jsonStr("did:plc:test"),
"vulnReportBlob": blobJSON,
"critical": jsonInt(critical),
"high": jsonInt(high),
"medium": jsonInt(medium),
"low": jsonInt(low),
"total": jsonInt(total),
"scannerVersion": jsonStr("atcr-scanner-v1.0.0"),
"scannedAt": jsonStr("2025-01-15T10:30:00Z"),
}
recordJSON, _ := json.Marshal(record)
envelope := map[string]any{
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
"cid": "bafyreiabc123",
"value": json.RawMessage(recordJSON),
}
b, _ := json.Marshal(envelope)
return string(b)
}
func jsonStr(s string) json.RawMessage {
b, _ := json.Marshal(s)
return b
}
func jsonInt(n int64) json.RawMessage {
b, _ := json.Marshal(n)
return b
}
// mockScanRecordWithoutBlob returns a getRecord envelope without VulnReportBlob
func mockScanRecordWithoutBlob(critical, high, medium, low, total int64) string {
record := map[string]any{
"$type": "io.atcr.hold.scan",
"manifest": "at://did:plc:test/io.atcr.manifest/abc123",
"repository": "myapp",
"userDid": "did:plc:test",
"critical": critical,
"high": high,
"medium": medium,
"low": low,
"total": total,
"scannerVersion": "atcr-scanner-v1.0.0",
"scannedAt": "2025-01-15T10:30:00Z",
}
envelope := map[string]any{
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
"cid": "bafyreiabc123",
"value": record,
}
b, _ := json.Marshal(envelope)
return string(b)
}
func setupVulnDetailsHandler(t *testing.T) *handlers.VulnDetailsHandler {
t.Helper()
templates, err := appview.Templates(nil)
if err != nil {
t.Fatalf("Failed to load templates: %v", err)
}
return &handlers.VulnDetailsHandler{
BaseUIHandler: handlers.BaseUIHandler{
Templates: templates,
},
}
}
func TestVulnDetails_FullReport(t *testing.T) {
grypeJSON := mockGrypeReport()
// Mock hold that serves both getRecord and getBlob
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
if strings.Contains(path, "getRecord") {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecordWithBlob(1, 1, 0, 1, 3)))
} else if strings.Contains(path, "getBlob") {
// Serve the Grype JSON directly (no redirect in tests)
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(grypeJSON))
} else {
http.Error(w, "not found", http.StatusNotFound)
}
}))
defer hold.Close()
handler := setupVulnDetailsHandler(t)
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Errorf("Expected status %d, got %d", http.StatusOK, rr.Code)
}
body := rr.Body.String()
// Should contain CVE IDs
if !strings.Contains(body, "CVE-2024-1234") {
t.Error("Expected body to contain CVE-2024-1234")
}
if !strings.Contains(body, "GHSA-abcd-efgh-ijkl") {
t.Error("Expected body to contain GHSA-abcd-efgh-ijkl")
}
// Should contain package names
if !strings.Contains(body, "libssl") {
t.Error("Expected body to contain package name 'libssl'")
}
if !strings.Contains(body, "express") {
t.Error("Expected body to contain package name 'express'")
}
// Should contain NVD link for CVE
if !strings.Contains(body, "nvd.nist.gov") {
t.Error("Expected body to contain NVD link")
}
// Should contain GitHub advisory link for GHSA
if !strings.Contains(body, "github.com/advisories") {
t.Error("Expected body to contain GitHub advisory link")
}
// Should contain fix version
if !strings.Contains(body, "1.2.4") {
t.Error("Expected body to contain fix version '1.2.4'")
}
// Should contain "No fix" for unfixed vuln
if !strings.Contains(body, "No fix") {
t.Error("Expected body to contain 'No fix' for unfixed vulnerability")
}
// Should contain a table
if !strings.Contains(body, "<table") {
t.Error("Expected body to contain a table element")
}
}
func TestVulnDetails_NoVulnReportBlob(t *testing.T) {
// Mock hold returns scan record WITHOUT VulnReportBlob
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecordWithoutBlob(2, 5, 10, 3, 20)))
}))
defer hold.Close()
handler := setupVulnDetailsHandler(t)
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
// Should show summary counts
if !strings.Contains(body, "2 Critical") {
t.Error("Expected body to contain '2 Critical' summary")
}
// Should indicate no detailed report
if !strings.Contains(body, "No detailed vulnerability report") {
t.Error("Expected body to indicate no detailed report available")
}
// Should NOT contain a table
if strings.Contains(body, "<table") {
t.Error("Should not render a table when no VulnReportBlob")
}
}
func TestVulnDetails_NotFound(t *testing.T) {
// Mock hold returns 404
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "not found", http.StatusNotFound)
}))
defer hold.Close()
handler := setupVulnDetailsHandler(t)
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
// Should show an error message (modal still needs content)
if !strings.Contains(body, "No scan record found") {
t.Error("Expected body to contain error message for 404")
}
}
func TestVulnDetails_SortsBySeverity(t *testing.T) {
grypeJSON := mockGrypeReport() // Has Critical, Low, High in that order
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
if strings.Contains(path, "getRecord") {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecordWithBlob(1, 1, 0, 1, 3)))
} else if strings.Contains(path, "getBlob") {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(grypeJSON))
}
}))
defer hold.Close()
handler := setupVulnDetailsHandler(t)
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
// Critical should appear before High, which should appear before Low
critIdx := strings.Index(body, "CVE-2024-1234") // Critical
highIdx := strings.Index(body, "GHSA-abcd-efgh") // High
lowIdx := strings.Index(body, "CVE-2024-5678") // Low
if critIdx == -1 || highIdx == -1 || lowIdx == -1 {
t.Fatal("Expected all three CVEs to be present in body")
}
if critIdx > highIdx {
t.Error("Critical CVE should appear before High CVE")
}
if highIdx > lowIdx {
t.Error("High CVE should appear before Low CVE")
}
}
func TestVulnDetails_MissingParams(t *testing.T) {
handler := setupVulnDetailsHandler(t)
req := httptest.NewRequest("GET", "/api/vuln-details", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, "Missing required parameters") {
t.Error("Expected error message for missing parameters")
}
}
File diff suppressed because one or more lines are too long
+4
View File
@@ -122,6 +122,10 @@ func RegisterUIRoutes(router chi.Router, deps UIDependencies) {
// Manifest health check API endpoint (HTMX polling)
router.Get("/api/manifest-health", (&uihandlers.ManifestHealthHandler{BaseUIHandler: base}).ServeHTTP)
// Vulnerability scan result API endpoints (HTMX lazy loading + modal content)
router.Get("/api/scan-result", (&uihandlers.ScanResultHandler{BaseUIHandler: base}).ServeHTTP)
router.Get("/api/vuln-details", (&uihandlers.VulnDetailsHandler{BaseUIHandler: base}).ServeHTTP)
router.Get("/u/{handle}", middleware.OptionalAuth(deps.SessionStore, deps.Database)(
&uihandlers.UserPageHandler{BaseUIHandler: base},
).ServeHTTP)
+19
View File
@@ -363,6 +363,24 @@ document.addEventListener('DOMContentLoaded', () => {
}
});
// Vulnerability details modal
async function openVulnDetails(digest, holdEndpoint) {
const modal = document.getElementById('vuln-detail-modal');
const body = document.getElementById('vuln-modal-body');
if (!modal || !body) return;
// Show modal with loading spinner
body.innerHTML = '<div class="flex justify-center py-8"><span class="loading loading-spinner loading-lg"></span></div>';
modal.showModal();
try {
const resp = await fetch(`/api/vuln-details?digest=${encodeURIComponent(digest)}&holdEndpoint=${encodeURIComponent(holdEndpoint)}`);
body.innerHTML = await resp.text();
} catch {
body.innerHTML = '<p class="text-error">Failed to load vulnerability details</p>';
}
}
// Login page recent accounts helper (works alongside actor-typeahead web component)
class RecentAccountsHelper {
constructor(inputElement) {
@@ -692,3 +710,4 @@ window.copyToClipboard = copyToClipboard;
window.toggleOfflineManifests = toggleOfflineManifests;
window.deleteManifest = deleteManifest;
window.closeManifestDeleteModal = closeManifestDeleteModal;
window.openVulnDetails = openVulnDetails;
@@ -208,6 +208,13 @@
{{ else if not .Reachable }}
<span class="badge badge-sm badge-warning">{{ icon "alert-triangle" "size-3" }} Offline</span>
{{ end }}
{{/* Vulnerability scan badge (lazy-loaded from hold) */}}
{{ if and (not .IsManifestList) .Manifest.HoldEndpoint }}
<span hx-get="/api/scan-result?digest={{ .Manifest.Digest | urlquery }}&holdEndpoint={{ .Manifest.HoldEndpoint | urlquery }}"
hx-trigger="load delay:1s"
hx-swap="outerHTML">
</span>
{{ end }}
</div>
<div class="flex items-center gap-2">
<code class="font-mono text-xs text-base-content/60 truncate max-w-40" title="{{ .Manifest.Digest }}">{{ .Manifest.Digest }}</code>
@@ -283,6 +290,20 @@
</form>
</dialog>
<!-- Vulnerability Details Modal -->
<dialog id="vuln-detail-modal" class="modal">
<div class="modal-box max-w-4xl">
<h3 class="text-lg font-bold">Vulnerability Scan Results</h3>
<div id="vuln-modal-body" class="py-4">
<span class="loading loading-spinner loading-md"></span>
</div>
<div class="modal-action">
<form method="dialog"><button class="btn">Close</button></form>
</div>
</div>
<form method="dialog" class="modal-backdrop"><button>close</button></form>
</dialog>
{{ template "footer" . }}
</body>
</html>
@@ -0,0 +1,24 @@
{{ define "vuln-badge" }}
{{ if .Error }}
{{/* Silently hide on error / no scan record — scan badges are non-critical */}}
{{ else if eq .Total 0 }}
<span class="badge badge-sm badge-success" title="No vulnerabilities found (scanned {{ .ScannedAt }})">{{ icon "shield-check" "size-3" }} Clean</span>
{{ else }}
<button class="flex items-center gap-1 cursor-pointer hover:opacity-80 transition-opacity"
onclick="openVulnDetails('{{ .Digest }}', '{{ .HoldEndpoint }}')"
title="Click for vulnerability details (scanned {{ .ScannedAt }})">
{{ if gt .Critical 0 }}
<span class="badge badge-sm badge-error">C:{{ .Critical }}</span>
{{ end }}
{{ if gt .High 0 }}
<span class="badge badge-sm badge-warning">H:{{ .High }}</span>
{{ end }}
{{ if gt .Medium 0 }}
<span class="badge badge-sm badge-soft badge-warning">M:{{ .Medium }}</span>
{{ end }}
{{ if gt .Low 0 }}
<span class="badge badge-sm badge-info">L:{{ .Low }}</span>
{{ end }}
</button>
{{ end }}
{{ end }}
@@ -0,0 +1,87 @@
{{ define "vuln-details" }}
{{ if .Error }}
{{ if .Summary.Total }}
<!-- Summary available but no detailed report -->
<div class="space-y-4">
<div class="flex flex-wrap gap-2">
{{ if gt .Summary.Critical 0 }}<span class="badge badge-error">{{ .Summary.Critical }} Critical</span>{{ end }}
{{ if gt .Summary.High 0 }}<span class="badge badge-warning">{{ .Summary.High }} High</span>{{ end }}
{{ if gt .Summary.Medium 0 }}<span class="badge badge-soft badge-warning">{{ .Summary.Medium }} Medium</span>{{ end }}
{{ if gt .Summary.Low 0 }}<span class="badge badge-info">{{ .Summary.Low }} Low</span>{{ end }}
</div>
<p class="text-base-content/60 text-sm">{{ .Error }}</p>
{{ if .ScannedAt }}<p class="text-base-content/40 text-xs">Scanned: {{ .ScannedAt }}</p>{{ end }}
</div>
{{ else }}
<p class="text-base-content/60">{{ .Error }}</p>
{{ end }}
{{ else }}
<div class="space-y-4">
<!-- Summary badges -->
<div class="flex flex-wrap items-center gap-2">
<span class="font-semibold text-sm">{{ .Summary.Total }} vulnerabilities found</span>
{{ if gt .Summary.Critical 0 }}<span class="badge badge-error">{{ .Summary.Critical }} Critical</span>{{ end }}
{{ if gt .Summary.High 0 }}<span class="badge badge-warning">{{ .Summary.High }} High</span>{{ end }}
{{ if gt .Summary.Medium 0 }}<span class="badge badge-soft badge-warning">{{ .Summary.Medium }} Medium</span>{{ end }}
{{ if gt .Summary.Low 0 }}<span class="badge badge-info">{{ .Summary.Low }} Low</span>{{ end }}
</div>
{{ if .ScannedAt }}<p class="text-base-content/40 text-xs">Scanned: {{ .ScannedAt }}</p>{{ end }}
{{ if .Matches }}
<!-- CVE table -->
<div class="overflow-x-auto max-h-96">
<table class="table table-sm table-pin-rows">
<thead>
<tr>
<th>CVE</th>
<th>Severity</th>
<th>Package</th>
<th>Installed</th>
<th>Fixed In</th>
</tr>
</thead>
<tbody>
{{ range .Matches }}
<tr>
<td class="font-mono text-xs">
{{ if .CVEURL }}
<a href="{{ .CVEURL }}" target="_blank" rel="noopener noreferrer" class="link link-primary">{{ .CVEID }}</a>
{{ else }}
{{ .CVEID }}
{{ end }}
</td>
<td>
{{ if eq .Severity "Critical" }}
<span class="badge badge-sm badge-error">Critical</span>
{{ else if eq .Severity "High" }}
<span class="badge badge-sm badge-warning">High</span>
{{ else if eq .Severity "Medium" }}
<span class="badge badge-sm badge-soft badge-warning">Medium</span>
{{ else if eq .Severity "Low" }}
<span class="badge badge-sm badge-info">Low</span>
{{ else }}
<span class="badge badge-sm badge-ghost">{{ .Severity }}</span>
{{ end }}
</td>
<td>
<span class="font-mono text-xs">{{ .Package }}</span>
{{ if .Type }}<span class="text-base-content/40 text-xs">({{ .Type }})</span>{{ end }}
</td>
<td class="font-mono text-xs">{{ .Version }}</td>
<td class="font-mono text-xs">
{{ if .FixedIn }}
<span class="text-success">{{ .FixedIn }}</span>
{{ else }}
<span class="text-base-content/40">No fix</span>
{{ end }}
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
{{ end }}
</div>
{{ end }}
{{ end }}
+37 -1
View File
@@ -1851,7 +1851,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error {
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{172}); err != nil {
if _, err := cw.Write([]byte{173}); err != nil {
return err
}
@@ -2118,6 +2118,22 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error {
if _, err := cw.WriteString(string(t.ScannerVersion)); err != nil {
return err
}
// t.VulnReportBlob (util.LexBlob) (struct)
if len("vulnReportBlob") > 8192 {
return xerrors.Errorf("Value in field \"vulnReportBlob\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vulnReportBlob"))); err != nil {
return err
}
if _, err := cw.WriteString(string("vulnReportBlob")); err != nil {
return err
}
if err := t.VulnReportBlob.MarshalCBOR(cw); err != nil {
return err
}
return nil
}
@@ -2378,6 +2394,26 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) {
t.ScannerVersion = string(sval)
}
// t.VulnReportBlob (util.LexBlob) (struct)
case "vulnReportBlob":
{
b, err := cr.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := cr.UnreadByte(); err != nil {
return err
}
t.VulnReportBlob = new(util.LexBlob)
if err := t.VulnReportBlob.UnmarshalCBOR(cr); err != nil {
return xerrors.Errorf("unmarshaling t.VulnReportBlob pointer: %w", err)
}
}
}
default:
// Field doesn't exist on this type, so ignore it
+15 -12
View File
@@ -801,30 +801,33 @@ func CrewRecordKey(memberDID string) string {
// RKey is deterministic: based on manifest digest (one scan per manifest)
type ScanRecord struct {
Type string `json:"$type" cborgen:"$type"`
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the scanned manifest (e.g., "at://did:plc:xyz/io.atcr.manifest/abc123...")
Repository string `json:"repository" cborgen:"repository"` // Repository name (e.g., "myapp")
UserDID string `json:"userDid" cborgen:"userDid"` // DID of the image owner
SbomBlob *lexutil.LexBlob `json:"sbomBlob,omitempty" cborgen:"sbomBlob"` // SBOM blob uploaded to hold's PDS blob storage
Critical int64 `json:"critical" cborgen:"critical"` // Count of critical vulnerabilities
High int64 `json:"high" cborgen:"high"` // Count of high vulnerabilities
Medium int64 `json:"medium" cborgen:"medium"` // Count of medium vulnerabilities
Low int64 `json:"low" cborgen:"low"` // Count of low vulnerabilities
Total int64 `json:"total" cborgen:"total"` // Total vulnerability count
ScannerVersion string `json:"scannerVersion" cborgen:"scannerVersion"` // Scanner version (e.g., "atcr-scanner-v1.0.0")
ScannedAt string `json:"scannedAt" cborgen:"scannedAt"` // RFC3339 timestamp of scan completion
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the scanned manifest (e.g., "at://did:plc:xyz/io.atcr.manifest/abc123...")
Repository string `json:"repository" cborgen:"repository"` // Repository name (e.g., "myapp")
UserDID string `json:"userDid" cborgen:"userDid"` // DID of the image owner
SbomBlob *lexutil.LexBlob `json:"sbomBlob,omitempty" cborgen:"sbomBlob"` // SBOM blob uploaded to hold's PDS blob storage
VulnReportBlob *lexutil.LexBlob `json:"vulnReportBlob,omitempty" cborgen:"vulnReportBlob"` // Grype vulnerability report blob (full CVE details)
Critical int64 `json:"critical" cborgen:"critical"` // Count of critical vulnerabilities
High int64 `json:"high" cborgen:"high"` // Count of high vulnerabilities
Medium int64 `json:"medium" cborgen:"medium"` // Count of medium vulnerabilities
Low int64 `json:"low" cborgen:"low"` // Count of low vulnerabilities
Total int64 `json:"total" cborgen:"total"` // Total vulnerability count
ScannerVersion string `json:"scannerVersion" cborgen:"scannerVersion"` // Scanner version (e.g., "atcr-scanner-v1.0.0")
ScannedAt string `json:"scannedAt" cborgen:"scannedAt"` // RFC3339 timestamp of scan completion
}
// NewScanRecord creates a new scan record
// manifestDigest: the manifest digest (e.g., "sha256:abc123...")
// userDID: the DID of the image owner (used to build the manifest AT-URI)
// sbomBlob: blob reference from uploading SBOM to PDS blob storage (nil if no SBOM)
func NewScanRecord(manifestDigest, repository, userDID string, sbomBlob *lexutil.LexBlob, critical, high, medium, low, total int, scannerVersion string) *ScanRecord {
// vulnReportBlob: blob reference from uploading Grype vulnerability report (nil if no report)
func NewScanRecord(manifestDigest, repository, userDID string, sbomBlob, vulnReportBlob *lexutil.LexBlob, critical, high, medium, low, total int, scannerVersion string) *ScanRecord {
return &ScanRecord{
Type: ScanCollection,
Manifest: BuildManifestURI(userDID, manifestDigest),
Repository: repository,
UserDID: userDID,
SbomBlob: sbomBlob,
VulnReportBlob: vulnReportBlob,
Critical: int64(critical),
High: int64(high),
Medium: int64(medium),
+16 -42
View File
@@ -12,7 +12,6 @@ import (
"path/filepath"
"time"
"github.com/distribution/distribution/v3/configuration"
"github.com/spf13/viper"
"atcr.io/pkg/config"
@@ -64,7 +63,7 @@ type RegistrationConfig struct {
Region string `yaml:"region" comment:"Deployment region, auto-detected from cloud metadata or S3 config."`
}
// StorageConfig holds S3 storage credentials and the internal distribution config.
// StorageConfig holds S3 storage credentials.
type StorageConfig struct {
// S3-compatible access key.
AccessKey string `yaml:"access_key" comment:"S3-compatible access key (AWS, Storj, Minio, UpCloud)."`
@@ -83,24 +82,24 @@ type StorageConfig struct {
// CDN pull zone URL for presigned download URLs.
PullZone string `yaml:"pull_zone" comment:"CDN pull zone URL for downloads. When set, presigned GET/HEAD URLs use this host instead of the S3 endpoint. Uploads and API calls still use the S3 endpoint."`
// Internal distribution storage config, built from the above fields.
distStorage configuration.Storage `yaml:"-"`
}
// Type returns the storage driver type name (always "s3").
func (s StorageConfig) Type() string {
return "s3"
}
// Parameters returns the distribution driver parameters.
func (s StorageConfig) Parameters() configuration.Parameters {
if s.distStorage != nil {
if params, ok := s.distStorage["s3"]; ok {
return params
}
// S3Params returns a params map suitable for s3.NewS3Service.
func (s StorageConfig) S3Params() map[string]any {
params := map[string]any{
"accesskey": s.AccessKey,
"secretkey": s.SecretKey,
"region": s.Region,
"bucket": s.Bucket,
}
return nil
if s.Endpoint != "" {
params["regionendpoint"] = s.Endpoint
params["forcepathstyle"] = true
}
if s.PullZone != "" {
params["pullzone"] = s.PullZone
}
return params
}
// ServerConfig defines server settings
@@ -276,9 +275,6 @@ func LoadConfig(yamlPath string) (*Config, error) {
// Store config path for subsystem config loading (e.g. billing)
cfg.configPath = yamlPath
// Build distribution storage config from struct fields
cfg.Storage.distStorage = buildStorageConfigFromFields(cfg.Storage)
// Detect region from cloud metadata or S3 config
if meta, err := DetectCloudMetadata(context.Background()); err == nil && meta != nil {
cfg.Registration.Region = meta.Region
@@ -290,25 +286,3 @@ func LoadConfig(yamlPath string) (*Config, error) {
return cfg, nil
}
// buildStorageConfigFromFields creates S3 storage configuration from StorageConfig fields.
func buildStorageConfigFromFields(sc StorageConfig) configuration.Storage {
params := make(map[string]any)
params["accesskey"] = sc.AccessKey
params["secretkey"] = sc.SecretKey
params["region"] = sc.Region
params["bucket"] = sc.Bucket
if sc.Endpoint != "" {
params["regionendpoint"] = sc.Endpoint
params["forcepathstyle"] = true
}
if sc.PullZone != "" {
params["pullzone"] = sc.PullZone
}
storageCfg := configuration.Storage{}
storageCfg["s3"] = configuration.Parameters(params)
return storageCfg
}
+4 -18
View File
@@ -187,7 +187,7 @@ func TestLoadConfig_KeyPathDefault(t *testing.T) {
}
}
func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
func TestS3Params_Complete(t *testing.T) {
sc := StorageConfig{
AccessKey: "test-access-key",
SecretKey: "test-secret-key",
@@ -196,14 +196,7 @@ func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
Endpoint: "https://s3.example.com",
}
cfg := buildStorageConfigFromFields(sc)
s3Params, ok := cfg["s3"]
if !ok {
t.Fatal("Expected s3 storage config")
}
params := map[string]any(s3Params)
params := sc.S3Params()
if params["accesskey"] != "test-access-key" {
t.Errorf("Expected accesskey=test-access-key, got %v", params["accesskey"])
@@ -222,7 +215,7 @@ func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
}
}
func TestBuildStorageConfigFromFields_S3_NoEndpoint(t *testing.T) {
func TestS3Params_NoEndpoint(t *testing.T) {
sc := StorageConfig{
AccessKey: "test-key",
SecretKey: "test-secret",
@@ -231,14 +224,7 @@ func TestBuildStorageConfigFromFields_S3_NoEndpoint(t *testing.T) {
Endpoint: "", // No custom endpoint
}
cfg := buildStorageConfigFromFields(sc)
s3Params, ok := cfg["s3"]
if !ok {
t.Fatal("Expected s3 storage config")
}
params := map[string]any(s3Params)
params := sc.S3Params()
// Should have default region
if params["region"] != "us-east-1" {
+15 -22
View File
@@ -14,8 +14,8 @@ import (
"atcr.io/pkg/atproto"
"atcr.io/pkg/hold/pds"
"atcr.io/pkg/s3"
"github.com/bluesky-social/indigo/atproto/syntax"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
)
// maxPreviewItems caps per-category detail slices to prevent memory/HTML bloat
@@ -64,7 +64,7 @@ type GCPreview struct {
// GarbageCollector handles cleanup of orphaned blobs from storage
type GarbageCollector struct {
pds *pds.HoldPDS
driver storagedriver.StorageDriver
s3 *s3.S3Service
cfg Config
logger *slog.Logger
@@ -117,10 +117,10 @@ type analysisResult struct {
}
// NewGarbageCollector creates a new GC instance
func NewGarbageCollector(holdPDS *pds.HoldPDS, driver storagedriver.StorageDriver, cfg Config) *GarbageCollector {
func NewGarbageCollector(holdPDS *pds.HoldPDS, s3svc *s3.S3Service, cfg Config) *GarbageCollector {
return &GarbageCollector{
pds: holdPDS,
driver: driver,
s3: s3svc,
cfg: cfg,
logger: slog.Default().With("component", "gc"),
stopCh: make(chan struct{}),
@@ -454,15 +454,12 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
totalBlobs := 0
blobsPath := "/docker/registry/v2/blobs"
err := gc.driver.Walk(ctx, blobsPath, func(fi storagedriver.FileInfo) error {
if fi.IsDir() {
return nil
}
if !strings.HasSuffix(fi.Path(), "/data") {
err := gc.s3.WalkBlobs(ctx, blobsPath, func(key string, size int64) error {
if !strings.HasSuffix(key, "/data") {
return nil
}
digest := extractDigestFromPath(fi.Path())
digest := extractDigestFromPath(key)
if digest == "" {
return nil
}
@@ -473,7 +470,7 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
if len(orphaned) < maxPreviewItems {
orphaned = append(orphaned, OrphanedBlobDetail{
Digest: digest,
Size: fi.Size(),
Size: size,
})
}
}
@@ -677,18 +674,14 @@ func (gc *GarbageCollector) deleteOrphanedRecords(ctx context.Context, orphanedR
func (gc *GarbageCollector) deleteOrphanedBlobs(ctx context.Context, referenced map[string]bool, result *GCResult) error {
blobsPath := "/docker/registry/v2/blobs"
err := gc.driver.Walk(ctx, blobsPath, func(fi storagedriver.FileInfo) error {
if fi.IsDir() {
return nil
}
err := gc.s3.WalkBlobs(ctx, blobsPath, func(key string, size int64) error {
// Only process data files
if !strings.HasSuffix(fi.Path(), "/data") {
if !strings.HasSuffix(key, "/data") {
return nil
}
// Extract digest from path
digest := extractDigestFromPath(fi.Path())
digest := extractDigestFromPath(key)
if digest == "" {
return nil
}
@@ -700,15 +693,15 @@ func (gc *GarbageCollector) deleteOrphanedBlobs(ctx context.Context, referenced
result.OrphanedBlobs++
if err := gc.driver.Delete(ctx, fi.Path()); err != nil {
gc.logger.Error("Failed to delete blob", "path", fi.Path(), "error", err)
if err := gc.s3.Delete(ctx, key); err != nil {
gc.logger.Error("Failed to delete blob", "path", key, "error", err)
return nil // Continue with other blobs
}
result.BlobsDeleted++
result.BytesReclaimed += fi.Size()
result.BytesReclaimed += size
gc.logger.Debug("Deleted orphaned blob",
"digest", digest,
"size", fi.Size())
"size", size)
return nil
})
+3 -4
View File
@@ -256,7 +256,7 @@ func (h *XRPCHandler) CompleteMultipartUploadWithManager(ctx context.Context, up
"source", sourcePath,
"dest", destPath)
if _, err := h.driver.Stat(ctx, sourcePath); err != nil {
if _, err := h.s3Service.Stat(ctx, sourcePath); err != nil {
slog.Error("Source blob not found after multipart complete",
"path", sourcePath,
"error", err)
@@ -264,9 +264,8 @@ func (h *XRPCHandler) CompleteMultipartUploadWithManager(ctx context.Context, up
}
slog.Debug("Source blob verified", "path", sourcePath)
// Move from temp to final digest location using driver
// Driver handles path management correctly (including S3 prefix)
if err := h.driver.Move(ctx, sourcePath, destPath); err != nil {
// Move from temp to final digest location (S3 copy + delete)
if err := h.s3Service.Move(ctx, sourcePath, destPath); err != nil {
slog.Error("Failed to move blob",
"source", sourcePath,
"dest", destPath,
+2 -5
View File
@@ -12,14 +12,12 @@ import (
"atcr.io/pkg/hold/pds"
"atcr.io/pkg/hold/quota"
"atcr.io/pkg/s3"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
)
// XRPCHandler handles OCI-specific XRPC endpoints for multipart uploads
type XRPCHandler struct {
driver storagedriver.StorageDriver
s3Service s3.S3Service
MultipartMgr *MultipartManager // Exported for access in route handlers
pds *pds.HoldPDS
@@ -30,9 +28,8 @@ type XRPCHandler struct {
}
// NewXRPCHandler creates a new OCI XRPC handler
func NewXRPCHandler(holdPDS *pds.HoldPDS, s3Service s3.S3Service, driver storagedriver.StorageDriver, enableBlueskyPosts bool, httpClient pds.HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
func NewXRPCHandler(holdPDS *pds.HoldPDS, s3Service s3.S3Service, enableBlueskyPosts bool, httpClient pds.HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
return &XRPCHandler{
driver: driver,
MultipartMgr: NewMultipartManager(),
s3Service: s3Service,
pds: holdPDS,
@@ -366,7 +363,7 @@ func (h *XRPCHandler) HandleNotifyManifest(w http.ResponseWriter, r *http.Reques
postURI, err = h.pds.CreateManifestPost(
ctx,
h.driver,
&h.s3Service,
req.Repository,
req.Tag,
userHandle,
+35 -214
View File
@@ -2,7 +2,6 @@ package oci
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
@@ -10,17 +9,12 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
"atcr.io/pkg/atproto"
"atcr.io/pkg/auth/oauth"
"atcr.io/pkg/hold/pds"
"atcr.io/pkg/s3"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
_ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws"
)
// Shared test resources for OCI package
@@ -68,148 +62,10 @@ func (m *mockPDSClient) Do(req *http.Request) (*http.Response, error) {
}, nil
}
// mockStorageDriver implements storagedriver.StorageDriver for testing
type mockStorageDriver struct {
mu sync.RWMutex
blobs map[string][]byte
// Error injection for testing error handling
StatError error
MoveError error
}
func newMockStorageDriver() *mockStorageDriver {
return &mockStorageDriver{
blobs: make(map[string][]byte),
}
}
func (m *mockStorageDriver) Name() string { return "mock" }
func (m *mockStorageDriver) GetContent(ctx context.Context, path string) ([]byte, error) {
m.mu.RLock()
defer m.mu.RUnlock()
if data, ok := m.blobs[path]; ok {
return data, nil
}
return nil, storagedriver.PathNotFoundError{Path: path}
}
func (m *mockStorageDriver) PutContent(ctx context.Context, path string, content []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
m.blobs[path] = content
return nil
}
func (m *mockStorageDriver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
data, err := m.GetContent(ctx, path)
if err != nil {
return nil, err
}
return io.NopCloser(bytes.NewReader(data[offset:])), nil
}
func (m *mockStorageDriver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
return &mockFileWriter{driver: m, path: path}, nil
}
func (m *mockStorageDriver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
m.mu.RLock()
defer m.mu.RUnlock()
// Check for injected error
if m.StatError != nil {
return nil, m.StatError
}
if data, ok := m.blobs[path]; ok {
return &mockFileInfo{path: path, size: int64(len(data))}, nil
}
return nil, storagedriver.PathNotFoundError{Path: path}
}
func (m *mockStorageDriver) List(ctx context.Context, path string) ([]string, error) {
return nil, nil
}
func (m *mockStorageDriver) Move(ctx context.Context, sourcePath string, destPath string) error {
m.mu.Lock()
defer m.mu.Unlock()
// Check for injected error
if m.MoveError != nil {
return m.MoveError
}
if data, ok := m.blobs[sourcePath]; ok {
m.blobs[destPath] = data
delete(m.blobs, sourcePath)
return nil
}
return storagedriver.PathNotFoundError{Path: sourcePath}
}
func (m *mockStorageDriver) Delete(ctx context.Context, path string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.blobs, path)
return nil
}
func (m *mockStorageDriver) RedirectURL(r *http.Request, path string) (string, error) {
return "", storagedriver.ErrUnsupportedMethod{}
}
func (m *mockStorageDriver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error {
return nil
}
// mockFileWriter implements storagedriver.FileWriter
type mockFileWriter struct {
driver *mockStorageDriver
path string
buf bytes.Buffer
}
func (w *mockFileWriter) Write(p []byte) (int, error) {
return w.buf.Write(p)
}
func (w *mockFileWriter) Size() int64 {
return int64(w.buf.Len())
}
func (w *mockFileWriter) Close() error {
return nil
}
func (w *mockFileWriter) Cancel(ctx context.Context) error {
return nil
}
func (w *mockFileWriter) Commit(ctx context.Context) error {
w.driver.mu.Lock()
defer w.driver.mu.Unlock()
w.driver.blobs[w.path] = w.buf.Bytes()
return nil
}
// mockFileInfo implements storagedriver.FileInfo
type mockFileInfo struct {
path string
size int64
}
func (f *mockFileInfo) Path() string { return f.path }
func (f *mockFileInfo) Size() int64 { return f.size }
func (f *mockFileInfo) ModTime() time.Time { return time.Time{} }
func (f *mockFileInfo) IsDir() bool { return false }
// setupTestOCIHandlerWithMockS3 creates a test OCI XRPC handler with mock S3
// This does NOT require real S3 credentials - uses MockS3Client
// Returns the handler, mock S3 client, and mock storage driver for test manipulation
func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client, *mockStorageDriver) {
// Returns the handler and mock S3 client for test manipulation
func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client) {
t.Helper()
// Create temp directory for PDS database
@@ -226,9 +82,6 @@ func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client
PathPrefix: "test-prefix",
}
// Create mock storage driver
mockDriver := newMockStorageDriver()
// Create minimal PDS for DID/auth
dbPath := ":memory:"
keyPath := filepath.Join(tmpDir, "signing-key")
@@ -268,9 +121,9 @@ func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client
mockClient := &mockPDSClient{}
// Create OCI handler with mock S3
handler := NewXRPCHandler(holdPDS, s3Service, mockDriver, false, mockClient, nil)
handler := NewXRPCHandler(holdPDS, s3Service, false, mockClient, nil)
return handler, mockS3Client, mockDriver
return handler, mockS3Client
}
// setupTestOCIHandlerWithS3 creates a test OCI XRPC handler with S3 driver
@@ -307,12 +160,6 @@ func setupTestOCIHandlerWithS3(t *testing.T) (*XRPCHandler, bool) {
}
s3Params["rootdirectory"] = storageDir
driver, err := factory.Create(ctx, "s3", s3Params)
if err != nil {
t.Logf("Failed to create S3 storage driver: %v", err)
return nil, false
}
// Create S3 service
s3Service, err := s3.NewS3Service(s3Params)
if err != nil {
@@ -359,7 +206,7 @@ func setupTestOCIHandlerWithS3(t *testing.T) (*XRPCHandler, bool) {
mockClient := &mockPDSClient{}
// Create OCI handler with S3
handler := NewXRPCHandler(holdPDS, *s3Service, driver, false, mockClient, nil)
handler := NewXRPCHandler(holdPDS, *s3Service, false, mockClient, nil)
return handler, true
}
@@ -392,7 +239,7 @@ func decodeJSONResponse(t *testing.T, w *httptest.ResponseRecorder, v any) {
// Tests for HandleInitiateUpload - Mock S3 (no credentials required)
func TestHandleInitiateUpload_MockS3_Success(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
"digest": "sha256:abc123",
@@ -421,7 +268,7 @@ func TestHandleInitiateUpload_MockS3_Success(t *testing.T) {
}
func TestHandleInitiateUpload_MockS3_MissingDigest(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{})
addMockAuth(req)
@@ -437,7 +284,7 @@ func TestHandleInitiateUpload_MockS3_MissingDigest(t *testing.T) {
// Tests for full Mock S3 upload flow (no credentials required)
func TestFullMockS3UploadFlow(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// 1. Initiate upload
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
@@ -506,7 +353,7 @@ func TestFullMockS3UploadFlow(t *testing.T) {
}
func TestHandleGetPartUploadUrl_MockS3_InvalidSession(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldGetPartUploadURL, map[string]any{
"uploadId": "invalid-upload-id",
@@ -523,7 +370,7 @@ func TestHandleGetPartUploadUrl_MockS3_InvalidSession(t *testing.T) {
}
func TestHandleAbortUpload_MockS3_InvalidSession(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldAbortUpload, map[string]string{
"uploadId": "invalid-upload-id",
@@ -764,7 +611,7 @@ func TestFullS3UploadFlow(t *testing.T) {
// Tests for HandleCompleteUpload with Mock S3
func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// 1. Initiate upload with temp path
tempDigest := "uploads/temp-test-complete"
@@ -796,12 +643,8 @@ func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
t.Fatalf("Expected status 200 for part URL, got %d: %s", partW.Code, partW.Body.String())
}
// 3. Pre-populate mock storage driver with temp blob (simulates S3 upload completing)
// Path format: /docker/registry/v2/uploads/temp-{id}/data
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test blob content"))
// 4. Complete upload with parts
// 3. Complete upload with parts
// (Mock S3 CompleteMultipartUpload auto-populates object for Stat/Move)
finalDigest := "sha256:abc123def456"
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
"uploadId": uploadID,
@@ -833,17 +676,15 @@ func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
t.Errorf("Expected 1 Complete call, got %d", len(mockS3Client.CompleteCalls))
}
// 6. Verify blob was moved to final location
// Final path format: /docker/registry/v2/blobs/sha256/ab/abc123def456/data
finalBlobPath := "/docker/registry/v2/blobs/sha256/ab/abc123def456/data"
_, err := mockDriver.Stat(t.Context(), finalBlobPath)
if err != nil {
t.Errorf("Expected blob at final location %s, got error: %v", finalBlobPath, err)
// 6. Verify blob was moved to final location in mock S3
finalS3Key := "test-prefix/docker/registry/v2/blobs/sha256/ab/abc123def456/data"
if mockS3Client.GetObject(finalS3Key) == nil {
t.Errorf("Expected blob at final S3 key %s", finalS3Key)
}
}
func TestHandleCompleteUpload_MockS3_InvalidSession(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
"uploadId": "non-existent-upload-id",
@@ -863,7 +704,7 @@ func TestHandleCompleteUpload_MockS3_InvalidSession(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_MissingParams(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
tests := []struct {
name string
@@ -891,7 +732,7 @@ func TestHandleCompleteUpload_MockS3_MissingParams(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Setup upload session
tempDigest := "uploads/temp-etag-test"
@@ -906,10 +747,6 @@ func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
decodeJSONResponse(t, initW, &initResp)
uploadID := initResp["uploadId"].(string)
// Pre-populate mock driver
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
// Complete with unquoted ETags
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
"uploadId": uploadID,
@@ -938,7 +775,7 @@ func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Setup upload session
tempDigest := "uploads/temp-s3-error"
@@ -953,10 +790,6 @@ func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
decodeJSONResponse(t, initW, &initResp)
uploadID := initResp["uploadId"].(string)
// Pre-populate mock driver
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
// Inject S3 error
mockS3Client.CompleteError = fmt.Errorf("simulated S3 CompleteMultipartUpload failure")
@@ -983,7 +816,7 @@ func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
handler, _, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Setup upload session
tempDigest := "uploads/temp-stat-error"
@@ -998,12 +831,8 @@ func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
decodeJSONResponse(t, initW, &initResp)
uploadID := initResp["uploadId"].(string)
// Pre-populate mock driver (so S3 complete succeeds)
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
// Inject Stat error (simulates blob not found after S3 complete)
mockDriver.StatError = fmt.Errorf("simulated stat failure")
// Inject HeadObject error (simulates blob not found after S3 complete)
mockS3Client.HeadObjectError = fmt.Errorf("simulated stat failure")
// Complete upload should fail
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
@@ -1023,7 +852,7 @@ func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
handler, _, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Setup upload session
tempDigest := "uploads/temp-move-error"
@@ -1038,12 +867,8 @@ func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
decodeJSONResponse(t, initW, &initResp)
uploadID := initResp["uploadId"].(string)
// Pre-populate mock driver
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
// Inject Move error
mockDriver.MoveError = fmt.Errorf("simulated move failure")
// Inject CopyObject error (Move = Copy + Delete, so Copy error simulates move failure)
mockS3Client.CopyObjectError = fmt.Errorf("simulated move failure")
// Complete upload should fail
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
@@ -1068,7 +893,7 @@ func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
}
func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Setup upload session
tempDigest := "uploads/temp-unsorted"
@@ -1083,10 +908,6 @@ func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
decodeJSONResponse(t, initW, &initResp)
uploadID := initResp["uploadId"].(string)
// Pre-populate mock driver
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
// Complete with unsorted parts (3, 1, 2) - handler should sort them
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
"uploadId": uploadID,
@@ -1117,7 +938,7 @@ func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
// Tests for HandleInitiateUpload edge cases
func TestHandleInitiateUpload_MockS3_S3Error(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// Inject S3 error
mockS3Client.CreateMultipartError = fmt.Errorf("simulated S3 CreateMultipartUpload failure")
@@ -1141,7 +962,7 @@ func TestHandleInitiateUpload_MockS3_S3Error(t *testing.T) {
}
func TestHandleInitiateUpload_MockS3_WhitespaceDigest(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
"digest": " ", // Whitespace only
@@ -1162,7 +983,7 @@ func TestHandleInitiateUpload_MockS3_WhitespaceDigest(t *testing.T) {
// Tests for HandleGetPartUploadURL edge cases
func TestHandleGetPartUploadUrl_MockS3_MissingParams(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
tests := []struct {
name string
@@ -1189,7 +1010,7 @@ func TestHandleGetPartUploadUrl_MockS3_MissingParams(t *testing.T) {
}
func TestHandleGetPartUploadUrl_MockS3_ValidSession(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// First initiate an upload
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
@@ -1237,7 +1058,7 @@ func TestHandleGetPartUploadUrl_MockS3_ValidSession(t *testing.T) {
// Tests for HandleAbortUpload edge cases
func TestHandleAbortUpload_MockS3_MissingUploadId(t *testing.T) {
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
handler, _ := setupTestOCIHandlerWithMockS3(t)
req := makeJSONRequest("POST", atproto.HoldAbortUpload, map[string]string{})
addMockAuth(req)
@@ -1251,7 +1072,7 @@ func TestHandleAbortUpload_MockS3_MissingUploadId(t *testing.T) {
}
func TestHandleAbortUpload_MockS3_S3Error(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// First initiate an upload
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
@@ -1288,7 +1109,7 @@ func TestHandleAbortUpload_MockS3_S3Error(t *testing.T) {
}
func TestHandleAbortUpload_MockS3_ValidSession(t *testing.T) {
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
// First initiate an upload
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
+3 -3
View File
@@ -10,8 +10,8 @@ import (
"time"
"atcr.io/pkg/atproto"
"atcr.io/pkg/s3"
bsky "github.com/bluesky-social/indigo/api/bsky"
"github.com/distribution/distribution/v3/registry/storage/driver"
)
// CreateManifestPost creates a Bluesky post announcing a manifest upload
@@ -19,7 +19,7 @@ import (
// artifactType is "container-image", "helm-chart", or "unknown"
func (p *HoldPDS) CreateManifestPost(
ctx context.Context,
storageDriver driver.StorageDriver,
s3svc *s3.S3Service,
repository, tag, userHandle, userDID, digest string,
totalSize int64,
platforms []string,
@@ -50,7 +50,7 @@ func (p *HoldPDS) CreateManifestPost(
slog.Warn("Failed to fetch OG image, posting without embed", "error", err)
} else {
// Upload OG image as blob
thumbBlob, err := uploadBlobToStorage(ctx, storageDriver, p.did, ogImageData, "image/png")
thumbBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, ogImageData, "image/png")
if err != nil {
slog.Warn("Failed to upload OG image blob", "error", err)
} else {
+9 -29
View File
@@ -1,7 +1,6 @@
package pds
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
@@ -11,9 +10,9 @@ import (
"time"
"atcr.io/pkg/atproto"
"atcr.io/pkg/s3"
bsky "github.com/bluesky-social/indigo/api/bsky"
lexutil "github.com/bluesky-social/indigo/lex/util"
"github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
@@ -68,9 +67,9 @@ func downloadImage(ctx context.Context, url string) ([]byte, string, error) {
return data, contentType, nil
}
// uploadBlobToStorage uploads a blob to the hold's storage and returns a blob reference
// This stores the blob at the ATProto path for the hold's DID
func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
// uploadBlobToStorage uploads a blob to the hold's S3 storage and returns a blob reference.
// This stores the blob at the ATProto path for the hold's DID.
func uploadBlobToStorage(ctx context.Context, s3svc *s3.S3Service, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty blob data")
}
@@ -90,33 +89,14 @@ func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver
// ATProto uses CIDv1 with raw codec for blobs
blobCID := cid.NewCidV1(0x55, mh)
// Store blob via distribution driver at ATProto path
// Store blob via S3 at ATProto path
path := atprotoBlobPath(did, blobCID.String())
// Write blob to storage using distribution driver
writer, err := storageDriver.Writer(ctx, path, false)
if err != nil {
return nil, fmt.Errorf("failed to create writer: %w", err)
}
// Write data
n, err := io.Copy(writer, bytes.NewReader(data))
if err != nil {
writer.Cancel(ctx)
return nil, fmt.Errorf("failed to write blob: %w", err)
}
// Commit the write
if err := writer.Commit(ctx); err != nil {
return nil, fmt.Errorf("failed to commit blob: %w", err)
}
if n != size {
return nil, fmt.Errorf("size mismatch: wrote %d bytes, expected %d", n, size)
if err := s3svc.PutBytes(ctx, path, data, mimeType); err != nil {
return nil, fmt.Errorf("failed to put blob: %w", err)
}
// Create blob reference in the format expected by bsky.ActorProfile
// LexLink is a type alias for cid.Cid
lexLink := lexutil.LexLink(blobCID)
blob := &lexutil.LexBlob{
Ref: lexLink,
@@ -129,7 +109,7 @@ func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver
// CreateProfileRecord creates the app.bsky.actor.profile record for the hold
// This will FAIL if the profile record already exists.
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, storageDriver driver.StorageDriver, displayName, description, avatarURL string) (cid.Cid, error) {
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, s3svc *s3.S3Service, displayName, description, avatarURL string) (cid.Cid, error) {
// Create profile struct
profile := &bsky.ActorProfile{
DisplayName: &displayName,
@@ -147,7 +127,7 @@ func (p *HoldPDS) CreateProfileRecord(ctx context.Context, storageDriver driver.
slog.Debug("Uploading avatar blob",
"size", len(imageData),
"mimeType", mimeType)
avatarBlob, err := uploadBlobToStorage(ctx, storageDriver, p.did, imageData, mimeType)
avatarBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, imageData, mimeType)
if err != nil {
return cid.Undef, fmt.Errorf("failed to upload avatar blob: %w", err)
}
+21 -8
View File
@@ -13,8 +13,8 @@ import (
"time"
"atcr.io/pkg/atproto"
"atcr.io/pkg/s3"
lexutil "github.com/bluesky-social/indigo/lex/util"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/gorilla/websocket"
)
@@ -28,7 +28,7 @@ type ScanBroadcaster struct {
db *sql.DB
holdDID string
holdEndpoint string
driver storagedriver.StorageDriver
s3 *s3.S3Service
pds *HoldPDS
ackTimeout time.Duration
secret string // Shared secret for scanner authentication
@@ -80,7 +80,7 @@ type VulnerabilitySummary struct {
// NewScanBroadcaster creates a new scan job broadcaster
// dbPath should point to a SQLite database file (e.g., "/path/to/pds/db.sqlite3")
func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver storagedriver.StorageDriver, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, s3svc *s3.S3Service, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
dsn := dbPath
if dbPath != ":memory:" && !strings.HasPrefix(dbPath, "file:") {
dsn = "file:" + dbPath
@@ -99,7 +99,7 @@ func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver sto
db: db,
holdDID: holdDID,
holdEndpoint: holdEndpoint,
driver: driver,
s3: s3svc,
pds: holdPDS,
ackTimeout: 5 * time.Minute,
secret: secret,
@@ -119,13 +119,13 @@ func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver sto
// NewScanBroadcasterWithDB creates a scan job broadcaster using an existing *sql.DB connection.
// The caller is responsible for the DB lifecycle.
func NewScanBroadcasterWithDB(holdDID, holdEndpoint, secret string, db *sql.DB, driver storagedriver.StorageDriver, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
func NewScanBroadcasterWithDB(holdDID, holdEndpoint, secret string, db *sql.DB, s3svc *s3.S3Service, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
sb := &ScanBroadcaster{
subscribers: make([]*ScanSubscriber, 0),
db: db,
holdDID: holdDID,
holdEndpoint: holdEndpoint,
driver: driver,
s3: s3svc,
pds: holdPDS,
ackTimeout: 5 * time.Minute,
secret: secret,
@@ -424,7 +424,7 @@ func (sb *ScanBroadcaster) handleResult(sub *ScanSubscriber, msg ScannerMessage)
// Upload SBOM as a blob to the hold's PDS blob storage (like manifest blobs)
var sbomBlob *lexutil.LexBlob
if msg.SBOM != "" {
blob, err := uploadBlobToStorage(ctx, sb.driver, sb.holdDID, []byte(msg.SBOM), "application/spdx+json")
blob, err := uploadBlobToStorage(ctx, sb.s3, sb.holdDID, []byte(msg.SBOM), "application/spdx+json")
if err != nil {
slog.Error("Failed to upload SBOM blob to PDS storage",
"seq", msg.Seq,
@@ -434,11 +434,24 @@ func (sb *ScanBroadcaster) handleResult(sub *ScanSubscriber, msg ScannerMessage)
}
}
// Upload vulnerability report as a blob (full Grype JSON with CVE details)
var vulnReportBlob *lexutil.LexBlob
if msg.VulnReport != "" {
blob, err := uploadBlobToStorage(ctx, sb.s3, sb.holdDID, []byte(msg.VulnReport), "application/vnd.atcr.vulnerabilities+json")
if err != nil {
slog.Error("Failed to upload VulnReport blob to PDS storage",
"seq", msg.Seq,
"error", err)
} else {
vulnReportBlob = blob
}
}
// Store scan result as a record in the hold's embedded PDS
if msg.Summary != nil {
scanRecord := atproto.NewScanRecord(
manifestDigest, repository, userDID,
sbomBlob,
sbomBlob, vulnReportBlob,
msg.Summary.Critical, msg.Summary.High, msg.Summary.Medium, msg.Summary.Low, msg.Summary.Total,
"atcr-scanner-v1.0.0",
)
+5 -5
View File
@@ -13,11 +13,11 @@ import (
"atcr.io/pkg/atproto"
"atcr.io/pkg/auth/oauth"
holddb "atcr.io/pkg/hold/db"
"atcr.io/pkg/s3"
"github.com/bluesky-social/indigo/atproto/atcrypto"
lexutil "github.com/bluesky-social/indigo/lex/util"
"github.com/bluesky-social/indigo/models"
"github.com/bluesky-social/indigo/repo"
"github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/ipfs/go-cid"
)
@@ -231,7 +231,7 @@ func (p *HoldPDS) GetRecordBytes(ctx context.Context, recordPath string) (cid.Ci
}
// Bootstrap initializes the hold with the captain record, owner as first crew member, and profile
func (p *HoldPDS) Bootstrap(ctx context.Context, storageDriver driver.StorageDriver, ownerDID string, public bool, allowAllCrew bool, avatarURL, region string) error {
func (p *HoldPDS) Bootstrap(ctx context.Context, s3svc *s3.S3Service, ownerDID string, public bool, allowAllCrew bool, avatarURL, region string) error {
if ownerDID == "" {
return nil
}
@@ -317,15 +317,15 @@ func (p *HoldPDS) Bootstrap(ctx context.Context, storageDriver driver.StorageDri
// Create Bluesky profile record (idempotent - check if exists first)
// This runs even if captain exists (for existing holds being upgraded)
// Skip if no storage driver (e.g., in tests)
if storageDriver != nil {
// Skip if no S3 service (e.g., in tests)
if s3svc != nil {
_, _, err = p.GetProfileRecord(ctx)
if err != nil {
// Bluesky profile doesn't exist, create it
displayName := "Cargo Hold"
description := "ahoy from the cargo hold"
_, err = p.CreateProfileRecord(ctx, storageDriver, displayName, description, avatarURL)
_, err = p.CreateProfileRecord(ctx, s3svc, displayName, description, avatarURL)
if err != nil {
return fmt.Errorf("failed to create bluesky profile record: %w", err)
}
+2 -2
View File
@@ -55,7 +55,7 @@ func TestStatusPost(t *testing.T) {
}
// Create handler for XRPC endpoints
handler := NewXRPCHandler(holdPDS, s3.S3Service{}, nil, nil, &mockPDSClient{}, nil)
handler := NewXRPCHandler(holdPDS, s3.S3Service{}, nil, &mockPDSClient{}, nil)
// Helper function to list posts via XRPC
listPosts := func() ([]map[string]any, error) {
@@ -283,7 +283,7 @@ func TestMain(m *testing.M) {
}
// Create shared handler
sharedHandler = NewXRPCHandler(sharedPDS, s3.S3Service{}, nil, nil, &mockPDSClient{}, nil)
sharedHandler = NewXRPCHandler(sharedPDS, s3.S3Service{}, nil, &mockPDSClient{}, nil)
// Run tests
code := m.Run()
+10 -34
View File
@@ -12,7 +12,6 @@ import (
"github.com/bluesky-social/indigo/api/bsky"
lexutil "github.com/bluesky-social/indigo/lex/util"
"github.com/bluesky-social/indigo/repo"
"github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
"github.com/gorilla/websocket"
@@ -46,7 +45,6 @@ const (
type XRPCHandler struct {
pds *HoldPDS
s3Service s3.S3Service
storageDriver driver.StorageDriver
broadcaster *EventBroadcaster
scanBroadcaster *ScanBroadcaster // Scan job dispatcher for connected scanners
httpClient HTTPClient // For testing - allows injecting mock HTTP client
@@ -68,14 +66,13 @@ type PartUploadInfo struct {
}
// NewXRPCHandler creates a new XRPC handler
func NewXRPCHandler(pds *HoldPDS, s3Service s3.S3Service, storageDriver driver.StorageDriver, broadcaster *EventBroadcaster, httpClient HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
func NewXRPCHandler(pds *HoldPDS, s3Service s3.S3Service, broadcaster *EventBroadcaster, httpClient HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
return &XRPCHandler{
pds: pds,
s3Service: s3Service,
storageDriver: storageDriver,
broadcaster: broadcaster,
httpClient: httpClient,
quotaMgr: quotaMgr,
pds: pds,
s3Service: s3Service,
broadcaster: broadcaster,
httpClient: httpClient,
quotaMgr: quotaMgr,
}
}
@@ -1052,32 +1049,11 @@ func (h *XRPCHandler) HandleUploadBlob(w http.ResponseWriter, r *http.Request) {
// ATProto uses CIDv1 with raw codec for blobs
blobCID := cid.NewCidV1(0x55, mh)
// Store blob via distribution driver at ATProto path
// Store blob via S3 at ATProto path
path := atprotoBlobPath(did, blobCID.String())
// Write blob to storage using distribution driver
writer, err := h.storageDriver.Writer(r.Context(), path, false)
if err != nil {
http.Error(w, fmt.Sprintf("failed to create writer: %v", err), http.StatusInternalServerError)
return
}
// Write data
n, err := io.Copy(writer, bytes.NewReader(blobData))
if err != nil {
writer.Cancel(r.Context())
http.Error(w, fmt.Sprintf("failed to write blob: %v", err), http.StatusInternalServerError)
return
}
// Commit the write
if err := writer.Commit(r.Context()); err != nil {
http.Error(w, fmt.Sprintf("failed to commit blob: %v", err), http.StatusInternalServerError)
return
}
if n != size {
http.Error(w, fmt.Sprintf("size mismatch: wrote %d bytes, expected %d", n, size), http.StatusInternalServerError)
if err := h.s3Service.PutBytes(r.Context(), path, blobData, "application/octet-stream"); err != nil {
http.Error(w, fmt.Sprintf("failed to put blob: %v", err), http.StatusInternalServerError)
return
}
@@ -1259,7 +1235,7 @@ func (h *XRPCHandler) HandleListBlobs(w http.ResponseWriter, r *http.Request) {
safeDID := strings.ReplaceAll(did, ":", "-")
blobsPath := fmt.Sprintf("/repos/%s/blobs", safeDID)
entries, err := h.storageDriver.List(r.Context(), blobsPath)
entries, err := h.s3Service.ListPrefix(r.Context(), blobsPath)
if err != nil {
// Path doesn't exist = no blobs, return empty list
render.JSON(w, r, map[string]any{"cids": []string{}})
+24 -62
View File
@@ -18,8 +18,6 @@ import (
"atcr.io/pkg/s3"
indigoAtproto "github.com/bluesky-social/indigo/api/atproto"
"github.com/bluesky-social/indigo/events"
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
"github.com/go-chi/chi/v5"
"github.com/gorilla/websocket"
"github.com/ipfs/go-cid"
@@ -76,7 +74,7 @@ func setupTestXRPCHandler(t *testing.T) (*XRPCHandler, context.Context) {
mockS3 := s3.S3Service{}
// Create XRPC handler with mock HTTP client
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
return handler, ctx
}
@@ -143,7 +141,7 @@ func setupTestXRPCHandlerWithIndex(t *testing.T) (*XRPCHandler, context.Context)
mockS3 := s3.S3Service{}
// Create XRPC handler with mock HTTP client
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
return handler, ctx
}
@@ -753,7 +751,7 @@ func TestHandleListRecords_EmptyCollection(t *testing.T) {
pds, ctx := setupTestPDS(t) // Don't bootstrap - no records created yet
mockClient := &mockPDSClient{}
mockS3 := s3.S3Service{}
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
// Initialize repo manually (setupTestPDS doesn't call Bootstrap, so no crew members)
err := pds.repomgr.InitNewActor(ctx, pds.uid, "", pds.did, "", "", "")
@@ -1231,7 +1229,7 @@ func TestHandleListRepos_EmptyRepo(t *testing.T) {
pds, ctx := setupTestPDS(t) // Don't bootstrap
mockClient := &mockPDSClient{}
mockS3 := s3.S3Service{}
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
// setupTestPDS creates the PDS/database but doesn't initialize the repo
// Check if implementation returns repos before initialization
@@ -1317,7 +1315,7 @@ func TestHandleGetRepoStatus_EmptyRepo(t *testing.T) {
pds, ctx := setupTestPDS(t) // Don't bootstrap
mockClient := &mockPDSClient{}
mockS3 := s3.S3Service{}
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
holdDID := "did:web:hold.example.com"
// Initialize repo but don't add any records
@@ -1960,27 +1958,6 @@ func TestHandleAtprotoDID(t *testing.T) {
// Mock S3 Service for testing blob endpoints
// mockS3Service is a simple mock that tracks calls and returns test URLs
type mockS3Service struct {
// Track calls
downloadCalls []string // Track digests requested for download
}
func newMockS3Service() *mockS3Service {
return &mockS3Service{
downloadCalls: []string{},
}
}
// toS3Service converts the mock to an s3.S3Service
// Returns empty s3.S3Service since we're not testing S3 presigned URLs in these tests
func (m *mockS3Service) toS3Service() s3.S3Service {
return s3.S3Service{
Client: nil, // Not testing presigned URLs
Bucket: "",
PathPrefix: "",
}
}
// setupTestXRPCHandlerWithMockS3 creates handler with MockS3Client for testing presigned URLs
func setupTestXRPCHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client, context.Context) {
t.Helper()
@@ -2029,27 +2006,17 @@ func setupTestXRPCHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Clien
PathPrefix: "test-prefix",
}
// Create filesystem storage driver for tests
storageDir := filepath.Join(tmpDir, "storage")
params := map[string]any{
"rootdirectory": storageDir,
}
driver, err := factory.Create(ctx, "filesystem", params)
if err != nil {
t.Fatalf("Failed to create storage driver: %v", err)
}
// Create mock PDS client for DPoP validation
mockClient := &mockPDSClient{}
// Create XRPC handler with mock S3 client and real filesystem driver
handler := NewXRPCHandler(pds, s3Service, driver, nil, mockClient, nil)
// Create XRPC handler with mock S3 client
handler := NewXRPCHandler(pds, s3Service, nil, mockClient, nil)
return handler, mockS3Client, ctx
}
// setupTestXRPCHandlerWithBlobs creates handler with mock s3 service and real filesystem driver
func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *mockS3Service, context.Context) {
// setupTestXRPCHandlerWithBlobs creates handler with MockS3Client for upload/list testing
func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *s3.MockS3Client, context.Context) {
t.Helper()
ctx := context.Background()
@@ -2088,26 +2055,21 @@ func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *mockS3Service,
t.Fatalf("Failed to bootstrap PDS: %v", err)
}
// Create mock s3 service that returns test URLs
mockS3Svc := newMockS3Service()
// Create filesystem storage driver for tests
storageDir := filepath.Join(tmpDir, "storage")
params := map[string]any{
"rootdirectory": storageDir,
}
driver, err := factory.Create(ctx, "filesystem", params)
if err != nil {
t.Fatalf("Failed to create storage driver: %v", err)
// Create MockS3Client for blob upload/list
mockS3Client := s3.NewMockS3Client("https://mock-s3.example.com")
s3Service := s3.S3Service{
Client: mockS3Client,
Bucket: "test-bucket",
PathPrefix: "",
}
// Create mock PDS client for DPoP validation
mockClient := &mockPDSClient{}
// Create XRPC handler with mock s3 service and real filesystem driver
handler := NewXRPCHandler(pds, mockS3Svc.toS3Service(), driver, nil, mockClient, nil)
// Create XRPC handler
handler := NewXRPCHandler(pds, s3Service, nil, mockClient, nil)
return handler, mockS3Svc, ctx
return handler, mockS3Client, ctx
}
// Tests for HandleUploadBlob
@@ -2391,9 +2353,9 @@ func TestHandleGetBlob(t *testing.T) {
t.Error("Expected Location header in 307 redirect")
}
// Should be XRPC proxy URL since we don't have S3 client
if !strings.Contains(location, "/xrpc/com.atproto.sync.getBlob") {
t.Errorf("Expected XRPC proxy URL, got: %s", location)
// Should be a presigned URL from the mock S3 client
if !strings.Contains(location, "mock-s3.example.com") {
t.Errorf("Expected presigned S3 URL, got: %s", location)
}
}
@@ -2457,9 +2419,9 @@ func TestHandleGetBlob_HeadMethod(t *testing.T) {
t.Error("Expected Location header in 307 redirect")
}
// Should be XRPC proxy URL since we don't have S3 client
if !strings.Contains(location, "/xrpc/com.atproto.sync.getBlob") {
t.Errorf("Expected XRPC proxy URL, got: %s", location)
// Should be a presigned URL from the mock S3 client
if !strings.Contains(location, "mock-s3.example.com") {
t.Errorf("Expected presigned S3 URL, got: %s", location)
}
}
+11 -24
View File
@@ -21,9 +21,6 @@ import (
"atcr.io/pkg/logging"
"atcr.io/pkg/s3"
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
_ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
@@ -72,6 +69,7 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
// Initialize embedded PDS if database path is configured
var xrpcHandler *pds.XRPCHandler
var s3Service *s3.S3Service
if cfg.Database.Path != "" {
holdDID := pds.GenerateDIDFromURL(cfg.Server.PublicURL)
slog.Info("Initializing embedded PDS", "did", holdDID)
@@ -109,14 +107,14 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
s.broadcaster = pds.NewEventBroadcaster(holdDID, 100, ":memory:")
}
// Create storage driver from config (needed for bootstrap profile avatar)
driver, err := factory.Create(ctx, cfg.Storage.Type(), cfg.Storage.Parameters())
// Create S3 service (used for bootstrap, handlers, GC, etc.)
s3Service, err = s3.NewS3Service(cfg.Storage.S3Params())
if err != nil {
return nil, fmt.Errorf("failed to create storage driver: %w", err)
return nil, fmt.Errorf("failed to create S3 service: %w", err)
}
// Bootstrap PDS with captain record, hold owner as first crew member, and profile
if err := s.PDS.Bootstrap(ctx, driver, cfg.Registration.OwnerDID, cfg.Server.Public, cfg.Registration.AllowAllCrew, cfg.Registration.ProfileAvatarURL, cfg.Registration.Region); err != nil {
if err := s.PDS.Bootstrap(ctx, s3Service, cfg.Registration.OwnerDID, cfg.Server.Public, cfg.Registration.AllowAllCrew, cfg.Registration.ProfileAvatarURL, cfg.Registration.Region); err != nil {
return nil, fmt.Errorf("failed to bootstrap PDS: %w", err)
}
@@ -163,32 +161,21 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
slog.Info("Quota enforcement disabled (no quota tiers configured)")
}
// Create blob store adapter and XRPC handlers
// Create XRPC handlers
var ociHandler *oci.XRPCHandler
if s.PDS != nil {
ctx := context.Background()
driver, err := factory.Create(ctx, cfg.Storage.Type(), cfg.Storage.Parameters())
if err != nil {
return nil, fmt.Errorf("failed to create storage driver: %w", err)
}
s3Service, err := s3.NewS3Service(cfg.Storage.Parameters())
if err != nil {
return nil, fmt.Errorf("failed to create S3 service: %w", err)
}
xrpcHandler = pds.NewXRPCHandler(s.PDS, *s3Service, driver, s.broadcaster, nil, s.QuotaManager)
ociHandler = oci.NewXRPCHandler(s.PDS, *s3Service, driver, cfg.Registration.EnableBlueskyPosts, nil, s.QuotaManager)
xrpcHandler = pds.NewXRPCHandler(s.PDS, *s3Service, s.broadcaster, nil, s.QuotaManager)
ociHandler = oci.NewXRPCHandler(s.PDS, *s3Service, cfg.Registration.EnableBlueskyPosts, nil, s.QuotaManager)
// Initialize scan broadcaster if scanner secret is configured
if cfg.Scanner.Secret != "" {
holdDID := pds.GenerateDIDFromURL(cfg.Server.PublicURL)
var sb *pds.ScanBroadcaster
if s.holdDB != nil {
sb, err = pds.NewScanBroadcasterWithDB(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, s.holdDB.DB, driver, s.PDS)
sb, err = pds.NewScanBroadcasterWithDB(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, s.holdDB.DB, s3Service, s.PDS)
} else {
scanDBPath := cfg.Database.Path + "/db.sqlite3"
sb, err = pds.NewScanBroadcaster(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, scanDBPath, driver, s.PDS)
sb, err = pds.NewScanBroadcaster(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, scanDBPath, s3Service, s.PDS)
}
if err != nil {
return nil, fmt.Errorf("failed to initialize scan broadcaster: %w", err)
@@ -200,7 +187,7 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
}
// Initialize garbage collector
s.garbageCollector = gc.NewGarbageCollector(s.PDS, driver, cfg.GC)
s.garbageCollector = gc.NewGarbageCollector(s.PDS, s3Service, cfg.GC)
slog.Info("Garbage collector initialized",
"enabled", cfg.GC.Enabled)
}
+178
View File
@@ -1,13 +1,17 @@
package s3
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
)
@@ -22,6 +26,9 @@ type MockS3Client struct {
// If empty, a UUID is generated.
UploadID string
// Objects stores in-memory blobs for PutObject/HeadObject/DeleteObject/CopyObject/ListObjectsV2.
Objects map[string][]byte
// Track calls for verification in tests
mu sync.Mutex
CreateMultipartCalls []CreateMultipartCall
@@ -36,6 +43,8 @@ type MockS3Client struct {
CreateMultipartError error
CompleteError error
AbortError error
HeadObjectError error
CopyObjectError error
}
// CreateMultipartCall records a CreateMultipartUpload call
@@ -89,6 +98,7 @@ type PutObjectCall struct {
func NewMockS3Client(testServerURL string) *MockS3Client {
return &MockS3Client{
TestServerURL: testServerURL,
Objects: make(map[string][]byte),
CreateMultipartCalls: []CreateMultipartCall{},
CompleteCalls: []CompleteCall{},
AbortCalls: []AbortCall{},
@@ -144,6 +154,14 @@ func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3
return nil, m.CompleteError
}
// Store a placeholder object at the key so Stat/HeadObject works after complete
key := aws.ToString(input.Key)
if m.Objects != nil {
if _, exists := m.Objects[key]; !exists {
m.Objects[key] = []byte("completed-multipart")
}
}
// Return a mock ETag
etag := "\"mock-etag-" + uuid.New().String() + "\""
return &awss3.CompleteMultipartUploadOutput{
@@ -169,6 +187,141 @@ func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
return &awss3.AbortMultipartUploadOutput{}, nil
}
// HeadObject implements S3Client
func (m *MockS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.HeadObjectError != nil {
return nil, m.HeadObjectError
}
key := aws.ToString(input.Key)
data, ok := m.Objects[key]
if !ok {
return nil, fmt.Errorf("NoSuchKey: object %s not found", key)
}
size := int64(len(data))
return &awss3.HeadObjectOutput{
ContentLength: &size,
}, nil
}
// PutObject implements S3Client
func (m *MockS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
key := aws.ToString(input.Key)
m.PutObjectCalls = append(m.PutObjectCalls, PutObjectCall{
Bucket: aws.ToString(input.Bucket),
Key: key,
})
if input.Body != nil {
data, err := io.ReadAll(input.Body)
if err != nil {
return nil, err
}
m.Objects[key] = data
} else {
m.Objects[key] = []byte{}
}
return &awss3.PutObjectOutput{}, nil
}
// CopyObject implements S3Client
func (m *MockS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.CopyObjectError != nil {
return nil, m.CopyObjectError
}
// CopySource is "bucket/key"
copySource := aws.ToString(input.CopySource)
// Strip bucket prefix to get key
parts := strings.SplitN(copySource, "/", 2)
srcKey := copySource
if len(parts) == 2 {
srcKey = parts[1]
}
data, ok := m.Objects[srcKey]
if !ok {
return nil, fmt.Errorf("NoSuchKey: source object %s not found", srcKey)
}
dstKey := aws.ToString(input.Key)
m.Objects[dstKey] = append([]byte{}, data...)
return &awss3.CopyObjectOutput{}, nil
}
// DeleteObject implements S3Client
func (m *MockS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
key := aws.ToString(input.Key)
delete(m.Objects, key)
return &awss3.DeleteObjectOutput{}, nil
}
// ListObjectsV2 implements S3Client
func (m *MockS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
m.mu.Lock()
defer m.mu.Unlock()
prefix := aws.ToString(input.Prefix)
delimiter := aws.ToString(input.Delimiter)
var contents []s3types.Object
commonPrefixes := map[string]bool{}
for key, data := range m.Objects {
if !strings.HasPrefix(key, prefix) {
continue
}
if delimiter != "" {
// Check if there's a delimiter after the prefix
rest := strings.TrimPrefix(key, prefix)
idx := strings.Index(rest, delimiter)
if idx >= 0 {
// Has delimiter — this is a common prefix, not a content object
cp := prefix + rest[:idx+len(delimiter)]
commonPrefixes[cp] = true
continue
}
}
size := int64(len(data))
k := key
contents = append(contents, s3types.Object{
Key: &k,
Size: &size,
})
}
var cps []s3types.CommonPrefix
for cp := range commonPrefixes {
p := cp
cps = append(cps, s3types.CommonPrefix{Prefix: &p})
}
falseVal := false
return &awss3.ListObjectsV2Output{
Contents: contents,
CommonPrefixes: cps,
IsTruncated: &falseVal,
}, nil
}
// PresignUploadPart implements S3Client
// Returns a mock presigned URL for test server
func (m *MockS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) {
@@ -229,6 +382,31 @@ func (m *MockS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObj
Key: aws.ToString(input.Key),
})
// Also store the body if provided (for PresignPutObject used in tests that also check objects)
if input.Body != nil {
key := aws.ToString(input.Key)
data, _ := io.ReadAll(input.Body)
m.Objects[key] = data
}
url := fmt.Sprintf("%s/put/%s", m.TestServerURL, aws.ToString(input.Key))
return url, nil
}
// SetObject is a test helper to pre-populate an object in the mock store.
func (m *MockS3Client) SetObject(key string, data []byte) {
m.mu.Lock()
defer m.mu.Unlock()
m.Objects[key] = append([]byte{}, data...)
}
// GetObject is a test helper to read an object from the mock store (nil if not found).
func (m *MockS3Client) GetObject(key string) []byte {
m.mu.Lock()
defer m.mu.Unlock()
data, ok := m.Objects[key]
if !ok {
return nil
}
return bytes.Clone(data)
}
+222 -11
View File
@@ -3,9 +3,11 @@
package s3
import (
"bytes"
"context"
"fmt"
"log/slog"
"net/url"
"strings"
"time"
@@ -24,6 +26,13 @@ type S3Client interface {
CompleteMultipartUpload(ctx context.Context, input *awss3.CompleteMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error)
// Direct object operations
HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error)
DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
// Presigned URL operations - return URL string directly
PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error)
PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error)
@@ -61,31 +70,49 @@ func (r *RealS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
return r.client.AbortMultipartUpload(ctx, input, opts...)
}
// HeadObject implements S3Client
func (r *RealS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
return r.client.HeadObject(ctx, input, opts...)
}
// PutObject implements S3Client
func (r *RealS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
return r.client.PutObject(ctx, input, opts...)
}
// CopyObject implements S3Client
func (r *RealS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
return r.client.CopyObject(ctx, input, opts...)
}
// DeleteObject implements S3Client
func (r *RealS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
return r.client.DeleteObject(ctx, input, opts...)
}
// ListObjectsV2 implements S3Client
func (r *RealS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
return r.client.ListObjectsV2(ctx, input, opts...)
}
// PresignGetObject implements S3Client
func (r *RealS3Client) PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error) {
result, err := r.presign.PresignGetObject(ctx, input, func(opts *awss3.PresignOptions) {
opts.Expires = expires
if r.pullZone != "" {
opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
o.BaseEndpoint = aws.String(r.pullZone)
})
}
})
if err != nil {
return "", err
}
return result.URL, nil
return r.applyPullZone(result.URL), nil
}
// PresignHeadObject implements S3Client
// Note: pull zone is intentionally NOT applied to HEAD requests. CDNs like
// Bunny may convert HEAD to GET when proxying, which breaks the SigV4
// signature. HEAD responses have no body, so CDN caching provides no benefit.
func (r *RealS3Client) PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error) {
result, err := r.presign.PresignHeadObject(ctx, input, func(opts *awss3.PresignOptions) {
opts.Expires = expires
if r.pullZone != "" {
opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
o.BaseEndpoint = aws.String(r.pullZone)
})
}
})
if err != nil {
return "", err
@@ -193,6 +220,190 @@ func NewS3Service(params map[string]any) (*S3Service, error) {
}, nil
}
// s3Key converts a blob path (with leading /) to an S3 key with prefix.
func (s *S3Service) s3Key(blobPath string) string {
key := strings.TrimPrefix(blobPath, "/")
if s.PathPrefix != "" {
key = s.PathPrefix + "/" + key
}
return key
}
// Stat returns the size of an object at blobPath, or an error if it doesn't exist.
func (s *S3Service) Stat(ctx context.Context, blobPath string) (int64, error) {
key := s.s3Key(blobPath)
out, err := s.Client.HeadObject(ctx, &awss3.HeadObjectInput{
Bucket: &s.Bucket,
Key: &key,
})
if err != nil {
return 0, err
}
if out.ContentLength != nil {
return *out.ContentLength, nil
}
return 0, nil
}
// PutBytes uploads data to blobPath with the given content type.
func (s *S3Service) PutBytes(ctx context.Context, blobPath string, data []byte, contentType string) error {
key := s.s3Key(blobPath)
_, err := s.Client.PutObject(ctx, &awss3.PutObjectInput{
Bucket: &s.Bucket,
Key: &key,
Body: bytes.NewReader(data),
ContentType: &contentType,
})
return err
}
// Move copies srcPath to dstPath then deletes srcPath.
func (s *S3Service) Move(ctx context.Context, srcPath, dstPath string) error {
srcKey := s.s3Key(srcPath)
dstKey := s.s3Key(dstPath)
copySource := s.Bucket + "/" + srcKey
_, err := s.Client.CopyObject(ctx, &awss3.CopyObjectInput{
Bucket: &s.Bucket,
Key: &dstKey,
CopySource: &copySource,
})
if err != nil {
return fmt.Errorf("copy %s -> %s: %w", srcPath, dstPath, err)
}
_, err = s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
Bucket: &s.Bucket,
Key: &srcKey,
})
if err != nil {
return fmt.Errorf("delete source %s after copy: %w", srcPath, err)
}
return nil
}
// Delete removes the object at blobPath.
func (s *S3Service) Delete(ctx context.Context, blobPath string) error {
key := s.s3Key(blobPath)
_, err := s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
Bucket: &s.Bucket,
Key: &key,
})
return err
}
// WalkBlobs paginates ListObjectsV2 under prefix and calls fn for each object.
// Keys passed to fn have the PathPrefix stripped (same format as BlobPath output).
func (s *S3Service) WalkBlobs(ctx context.Context, prefix string, fn func(key string, size int64) error) error {
s3Prefix := s.s3Key(prefix)
if !strings.HasSuffix(s3Prefix, "/") {
s3Prefix += "/"
}
var continuationToken *string
for {
out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
Bucket: &s.Bucket,
Prefix: &s3Prefix,
ContinuationToken: continuationToken,
})
if err != nil {
return fmt.Errorf("list objects under %s: %w", prefix, err)
}
for _, obj := range out.Contents {
if obj.Key == nil {
continue
}
// Strip PathPrefix to get the logical path
key := *obj.Key
if s.PathPrefix != "" {
key = strings.TrimPrefix(key, s.PathPrefix+"/")
}
// Restore leading / for consistency with BlobPath
key = "/" + key
var size int64
if obj.Size != nil {
size = *obj.Size
}
if err := fn(key, size); err != nil {
return err
}
}
if out.IsTruncated == nil || !*out.IsTruncated {
break
}
continuationToken = out.NextContinuationToken
}
return nil
}
// ListPrefix returns immediate children (common prefixes) under blobPath using Delimiter="/".
func (s *S3Service) ListPrefix(ctx context.Context, blobPath string) ([]string, error) {
s3Prefix := s.s3Key(blobPath)
if !strings.HasSuffix(s3Prefix, "/") {
s3Prefix += "/"
}
delimiter := "/"
var results []string
var continuationToken *string
for {
out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
Bucket: &s.Bucket,
Prefix: &s3Prefix,
Delimiter: &delimiter,
ContinuationToken: continuationToken,
})
if err != nil {
return nil, fmt.Errorf("list prefix %s: %w", blobPath, err)
}
for _, cp := range out.CommonPrefixes {
if cp.Prefix == nil {
continue
}
// Strip the PathPrefix and reconstruct logical path
p := *cp.Prefix
if s.PathPrefix != "" {
p = strings.TrimPrefix(p, s.PathPrefix+"/")
}
p = "/" + strings.TrimSuffix(p, "/")
results = append(results, p)
}
if out.IsTruncated == nil || !*out.IsTruncated {
break
}
continuationToken = out.NextContinuationToken
}
return results, nil
}
// applyPullZone replaces the host in a presigned URL with the pull zone host.
// The signature is computed against the real S3 endpoint so that the origin
// can validate it; the CDN just proxies the request transparently.
func (r *RealS3Client) applyPullZone(presignedURL string) string {
if r.pullZone == "" {
return presignedURL
}
parsed, err := url.Parse(presignedURL)
if err != nil {
return presignedURL
}
pz, err := url.Parse(r.pullZone)
if err != nil {
return presignedURL
}
parsed.Scheme = pz.Scheme
parsed.Host = pz.Host
return parsed.String()
}
// BlobPath converts a digest (e.g., "sha256:abc123...") or temp path to a storage path
// Distribution stores blobs as: /docker/registry/v2/blobs/{algorithm}/{xx}/{hash}/data
// where xx is the first 2 characters of the hash for directory sharding