mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-05-29 19:40:21 +00:00
remove distribution from hold, add vulnerability scanning in appview.
1. Removing distribution/distribution from the Hold Service (biggest change) The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service: - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which broke SigV4 signatures) - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file) 2. Vulnerability Scan UI in AppView (new feature) Displays scan results from the hold's PDS on the repository page: - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table) - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links) - Repository page: Lazy-loads scan badges per manifest via HTMX - Tests: ~590 lines of test coverage for both handlers 3. S3 Diagnostic Tool New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output. 4. Deployment Tooling - New syncServiceUnit() for comparing/updating systemd units on servers - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload 5. DB Migration 0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration. 6. Documentation - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side 7. go.mod aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).
This commit is contained in:
Evan Jarrett
418
cmd/s3-test/main.go
Normal file
418
cmd/s3-test/main.go
Normal file
@@ -0,0 +1,418 @@
|
||||
// Command s3-test is a diagnostic tool that tests S3 connectivity using both
|
||||
// AWS SDK v1 (used by distribution's storage driver) and AWS SDK v2 (used by
|
||||
// ATCR's presigned URL service). It helps diagnose signature compatibility
|
||||
// issues with S3-compatible storage providers.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
awsv1 "github.com/aws/aws-sdk-go/aws"
|
||||
credentialsv1 "github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
s3v1 "github.com/aws/aws-sdk-go/service/s3"
|
||||
|
||||
awsv2 "github.com/aws/aws-sdk-go-v2/aws"
|
||||
configv2 "github.com/aws/aws-sdk-go-v2/config"
|
||||
credentialsv2 "github.com/aws/aws-sdk-go-v2/credentials"
|
||||
s3v2 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
envFile = flag.String("env-file", "", "Load environment variables from file (KEY=VALUE format)")
|
||||
accessKey = flag.String("access-key", "", "S3 access key (env: AWS_ACCESS_KEY_ID)")
|
||||
secretKey = flag.String("secret-key", "", "S3 secret key (env: AWS_SECRET_ACCESS_KEY)")
|
||||
region = flag.String("region", "", "S3 region (env: S3_REGION)")
|
||||
bucket = flag.String("bucket", "", "S3 bucket name (env: S3_BUCKET)")
|
||||
endpoint = flag.String("endpoint", "", "S3 endpoint URL (env: S3_ENDPOINT)")
|
||||
pullZone = flag.String("pull-zone", "", "CDN pull zone URL for presigned reads (env: PULL_ZONE)")
|
||||
prefix = flag.String("prefix", "docker/registry/v2/blobs", "Key prefix for list operations")
|
||||
verbose = flag.Bool("verbose", false, "Enable SDK debug signing logs")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
// Load env file first, then let flags and real env vars override
|
||||
if *envFile != "" {
|
||||
if err := loadEnvFile(*envFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error loading env file: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve: flag > env var > default
|
||||
if *accessKey == "" {
|
||||
*accessKey = os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
}
|
||||
if *secretKey == "" {
|
||||
*secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
}
|
||||
if *region == "" {
|
||||
*region = envOr("S3_REGION", "us-east-1")
|
||||
}
|
||||
if *bucket == "" {
|
||||
*bucket = os.Getenv("S3_BUCKET")
|
||||
}
|
||||
if *endpoint == "" {
|
||||
*endpoint = os.Getenv("S3_ENDPOINT")
|
||||
}
|
||||
if *pullZone == "" {
|
||||
*pullZone = os.Getenv("PULL_ZONE")
|
||||
}
|
||||
|
||||
if *accessKey == "" || *secretKey == "" || *bucket == "" {
|
||||
fmt.Fprintln(os.Stderr, "Usage: s3-test [--env-file FILE] [--access-key KEY] [--secret-key KEY] [--bucket BUCKET] [--endpoint URL] [--region REGION] [--prefix PREFIX] [--verbose]")
|
||||
fmt.Fprintln(os.Stderr, "Env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET, S3_REGION, S3_ENDPOINT")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println("S3 Connectivity Diagnostic")
|
||||
fmt.Println("==========================")
|
||||
fmt.Printf("Endpoint: %s\n", valueOr(*endpoint, "(default AWS)"))
|
||||
fmt.Printf("Pull Zone: %s\n", valueOr(*pullZone, "(none)"))
|
||||
fmt.Printf("Region: %s\n", *region)
|
||||
fmt.Printf("AccessKey: %s...%s (%d chars)\n", (*accessKey)[:3], (*accessKey)[len(*accessKey)-3:], len(*accessKey))
|
||||
fmt.Printf("SecretKey: %s...%s (%d chars)\n", (*secretKey)[:3], (*secretKey)[len(*secretKey)-3:], len(*secretKey))
|
||||
fmt.Printf("Bucket: %s\n", *bucket)
|
||||
fmt.Printf("Prefix: %s\n", *prefix)
|
||||
fmt.Println()
|
||||
|
||||
ctx := context.Background()
|
||||
results := make([]result, 0, 6)
|
||||
|
||||
// Build SDK v1 client (SigV4) — matches distribution driver's New()
|
||||
v1Client := buildV1Client(*accessKey, *secretKey, *region, *endpoint, *verbose)
|
||||
|
||||
// Test 1: SDK v1 SigV4 HeadBucket
|
||||
results = append(results, runTest("SDK v1 / SigV4 / HeadBucket", func() error {
|
||||
_, err := v1Client.HeadBucketWithContext(ctx, &s3v1.HeadBucketInput{
|
||||
Bucket: awsv1.String(*bucket),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Test 2: SDK v1 SigV4 ListObjectsV2
|
||||
results = append(results, runTest("SDK v1 / SigV4 / ListObjectsV2", func() error {
|
||||
_, err := v1Client.ListObjectsV2WithContext(ctx, &s3v1.ListObjectsV2Input{
|
||||
Bucket: awsv1.String(*bucket),
|
||||
Prefix: awsv1.String(*prefix),
|
||||
MaxKeys: awsv1.Int64(5),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Test 3: SDK v1 SigV4 ListObjectsV2Pages (paginated, matches doWalk)
|
||||
results = append(results, runTest("SDK v1 / SigV4 / ListObjectsV2Pages", func() error {
|
||||
return v1Client.ListObjectsV2PagesWithContext(ctx, &s3v1.ListObjectsV2Input{
|
||||
Bucket: awsv1.String(*bucket),
|
||||
Prefix: awsv1.String(*prefix),
|
||||
MaxKeys: awsv1.Int64(5),
|
||||
}, func(page *s3v1.ListObjectsV2Output, lastPage bool) bool {
|
||||
return false // stop after first page
|
||||
})
|
||||
}))
|
||||
|
||||
// Build SDK v2 client — matches NewS3Service()
|
||||
v2Client := buildV2Client(ctx, *accessKey, *secretKey, *region, *endpoint)
|
||||
|
||||
// Test 5: SDK v2 SigV4 HeadBucket
|
||||
results = append(results, runTest("SDK v2 / SigV4 / HeadBucket", func() error {
|
||||
_, err := v2Client.HeadBucket(ctx, &s3v2.HeadBucketInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Test 6: SDK v2 SigV4 ListObjectsV2
|
||||
results = append(results, runTest("SDK v2 / SigV4 / ListObjectsV2", func() error {
|
||||
_, err := v2Client.ListObjectsV2(ctx, &s3v2.ListObjectsV2Input{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Prefix: awsv2.String(*prefix),
|
||||
MaxKeys: awsv2.Int32(5),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Find a real object key for GetObject / presigned URL tests
|
||||
var testKey string
|
||||
listOut, err := v2Client.ListObjectsV2(ctx, &s3v2.ListObjectsV2Input{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Prefix: awsv2.String(*prefix),
|
||||
MaxKeys: awsv2.Int32(1),
|
||||
})
|
||||
if err == nil && len(listOut.Contents) > 0 {
|
||||
testKey = *listOut.Contents[0].Key
|
||||
}
|
||||
|
||||
if testKey == "" {
|
||||
fmt.Printf("\n (Skipping GetObject/Presigned tests — no objects found under prefix %q)\n", *prefix)
|
||||
} else {
|
||||
fmt.Printf("\n Test object: %s\n\n", testKey)
|
||||
|
||||
// Test 7: SDK v1 GetObject (HEAD only)
|
||||
results = append(results, runTest("SDK v1 / SigV4 / HeadObject", func() error {
|
||||
_, err := v1Client.HeadObjectWithContext(ctx, &s3v1.HeadObjectInput{
|
||||
Bucket: awsv1.String(*bucket),
|
||||
Key: awsv1.String(testKey),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Test 8: SDK v2 GetObject (HEAD only)
|
||||
results = append(results, runTest("SDK v2 / SigV4 / HeadObject", func() error {
|
||||
_, err := v2Client.HeadObject(ctx, &s3v2.HeadObjectInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Key: awsv2.String(testKey),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
|
||||
// Test 9: SDK v2 Presigned GET URL (generate + fetch)
|
||||
presignClient := s3v2.NewPresignClient(v2Client)
|
||||
results = append(results, runTest("SDK v2 / Presigned GET URL", func() error {
|
||||
presigned, err := presignClient.PresignGetObject(ctx, &s3v2.GetObjectInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Key: awsv2.String(testKey),
|
||||
}, func(opts *s3v2.PresignOptions) {
|
||||
opts.Expires = 5 * time.Minute
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
if *verbose {
|
||||
// Show host + query params (no path to avoid leaking key structure)
|
||||
u, _ := url.Parse(presigned.URL)
|
||||
fmt.Printf("\n Presigned host: %s\n", u.Host)
|
||||
fmt.Printf(" Signed headers: %s\n", presigned.SignedHeader)
|
||||
}
|
||||
resp, err := http.Get(presigned.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch: %w", err)
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("presigned URL returned %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
// Pull zone presigned tests — sign against real endpoint, swap host to pull zone
|
||||
if *pullZone != "" {
|
||||
results = append(results, runTest("SDK v2 / Presigned GET via Pull Zone", func() error {
|
||||
presigned, err := presignClient.PresignGetObject(ctx, &s3v2.GetObjectInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Key: awsv2.String(testKey),
|
||||
}, func(opts *s3v2.PresignOptions) {
|
||||
opts.Expires = 5 * time.Minute
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
pzURL := swapHost(presigned.URL, *pullZone)
|
||||
if *verbose {
|
||||
fmt.Printf("\n Signed against: %s\n", presigned.URL[:40]+"...")
|
||||
fmt.Printf(" Fetching from: %s\n", pzURL[:40]+"...")
|
||||
}
|
||||
resp, err := http.Get(pzURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch: %w", err)
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("pull zone GET returned %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
}
|
||||
|
||||
// Test 10: SDK v2 Presigned PUT URL (generate + upload empty)
|
||||
results = append(results, runTest("SDK v2 / Presigned PUT URL", func() error {
|
||||
putKey := *prefix + "/_s3-test-probe"
|
||||
presigned, err := presignClient.PresignPutObject(ctx, &s3v2.PutObjectInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Key: awsv2.String(putKey),
|
||||
}, func(opts *s3v2.PresignOptions) {
|
||||
opts.Expires = 5 * time.Minute
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, presigned.URL, strings.NewReader(""))
|
||||
if err != nil {
|
||||
return fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Length", "0")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("presigned PUT returned %d", resp.StatusCode)
|
||||
}
|
||||
// Clean up
|
||||
_, _ = v2Client.DeleteObject(ctx, &s3v2.DeleteObjectInput{
|
||||
Bucket: awsv2.String(*bucket),
|
||||
Key: awsv2.String(putKey),
|
||||
})
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
// Print summary
|
||||
fmt.Println()
|
||||
fmt.Println("Summary")
|
||||
fmt.Println("=======")
|
||||
|
||||
allPass := true
|
||||
for _, r := range results {
|
||||
status := "PASS"
|
||||
if !r.ok {
|
||||
status = "FAIL"
|
||||
allPass = false
|
||||
}
|
||||
fmt.Printf(" [%s] %s (%s)\n", status, r.name, r.duration.Round(time.Millisecond))
|
||||
if !r.ok {
|
||||
fmt.Printf(" Error: %s\n", r.err)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
if allPass {
|
||||
fmt.Println("Diagnosis: All tests passed. S3 connectivity is working with both SDKs.")
|
||||
} else {
|
||||
fmt.Println("Diagnosis: Some tests failed. Review errors above.")
|
||||
}
|
||||
}
|
||||
|
||||
type result struct {
|
||||
name string
|
||||
ok bool
|
||||
err error
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
func runTest(name string, fn func() error) result {
|
||||
fmt.Printf(" Testing: %s ... ", name)
|
||||
start := time.Now()
|
||||
err := fn()
|
||||
d := time.Since(start)
|
||||
if err != nil {
|
||||
fmt.Printf("FAIL (%s)\n", d.Round(time.Millisecond))
|
||||
return result{name: name, ok: false, err: err, duration: d}
|
||||
}
|
||||
fmt.Printf("PASS (%s)\n", d.Round(time.Millisecond))
|
||||
return result{name: name, ok: true, duration: d}
|
||||
}
|
||||
|
||||
func loadEnvFile(path string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
line = strings.TrimPrefix(line, "export ")
|
||||
k, v, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v = strings.Trim(v, `"'`)
|
||||
os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v))
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func envOr(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func swapHost(presignedURL, pullZone string) string {
|
||||
parsed, err := url.Parse(presignedURL)
|
||||
if err != nil {
|
||||
return presignedURL
|
||||
}
|
||||
pz, err := url.Parse(pullZone)
|
||||
if err != nil {
|
||||
return presignedURL
|
||||
}
|
||||
parsed.Scheme = pz.Scheme
|
||||
parsed.Host = pz.Host
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func valueOr(s, fallback string) string {
|
||||
if s == "" {
|
||||
return fallback
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// buildV1Client constructs an SDK v1 S3 client identically to
|
||||
// distribution/distribution's s3-aws driver New() function.
|
||||
func buildV1Client(accessKey, secretKey, region, endpoint string, verbose bool) *s3v1.S3 {
|
||||
awsConfig := awsv1.NewConfig()
|
||||
|
||||
if verbose {
|
||||
awsConfig.WithLogLevel(awsv1.LogDebugWithSigning)
|
||||
}
|
||||
|
||||
awsConfig.WithCredentials(credentialsv1.NewStaticCredentials(accessKey, secretKey, ""))
|
||||
awsConfig.WithRegion(region)
|
||||
|
||||
if endpoint != "" {
|
||||
awsConfig.WithEndpoint(endpoint)
|
||||
awsConfig.WithS3ForcePathStyle(true)
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(awsConfig)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create SDK v1 session: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return s3v1.New(sess)
|
||||
}
|
||||
|
||||
// buildV2Client constructs an SDK v2 S3 client identically to
|
||||
// ATCR's NewS3Service() in pkg/s3/types.go.
|
||||
func buildV2Client(ctx context.Context, accessKey, secretKey, region, endpoint string) *s3v2.Client {
|
||||
cfg, err := configv2.LoadDefaultConfig(ctx,
|
||||
configv2.WithRegion(region),
|
||||
configv2.WithCredentialsProvider(
|
||||
credentialsv2.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to load SDK v2 config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
|
||||
if endpoint != "" {
|
||||
o.BaseEndpoint = awsv2.String(endpoint)
|
||||
o.UsePathStyle = true
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -192,6 +192,39 @@ func generateCloudInit(p cloudInitParams) (string, error) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// syncServiceUnit compares a rendered systemd service unit against what's on
|
||||
// the server. If they differ, it writes the new unit file. Returns true if the
|
||||
// unit was updated (caller should daemon-reload before restart).
|
||||
func syncServiceUnit(name, ip, serviceName, renderedUnit string) (bool, error) {
|
||||
unitPath := "/etc/systemd/system/" + serviceName + ".service"
|
||||
|
||||
remote, err := runSSH(ip, fmt.Sprintf("cat %s 2>/dev/null || echo '__MISSING__'", unitPath), false)
|
||||
if err != nil {
|
||||
fmt.Printf(" service unit sync: could not reach %s (%v)\n", name, err)
|
||||
return false, nil
|
||||
}
|
||||
remote = strings.TrimSpace(remote)
|
||||
rendered := strings.TrimSpace(renderedUnit)
|
||||
|
||||
if remote == "__MISSING__" {
|
||||
fmt.Printf(" service unit: %s not found (cloud-init will handle it)\n", name)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if remote == rendered {
|
||||
fmt.Printf(" service unit: %s up to date\n", name)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Write the updated unit file
|
||||
script := fmt.Sprintf("cat > %s << 'SVCEOF'\n%s\nSVCEOF", unitPath, rendered)
|
||||
if _, err := runSSH(ip, script, false); err != nil {
|
||||
return false, fmt.Errorf("write service unit: %w", err)
|
||||
}
|
||||
fmt.Printf(" service unit: %s updated\n", name)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// syncConfigKeys fetches the existing config from a server and merges in any
|
||||
// missing keys from the rendered template. Existing values are never overwritten.
|
||||
func syncConfigKeys(name, ip, configPath, templateYAML string) error {
|
||||
|
||||
@@ -51,3 +51,4 @@ quota:
|
||||
new_crew_tier: deckhand
|
||||
scanner:
|
||||
secret: ""
|
||||
|
||||
|
||||
@@ -55,12 +55,17 @@ func cmdUpdate(target string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
vals := configValsFromState(state)
|
||||
|
||||
targets := map[string]struct {
|
||||
ip string
|
||||
binaryName string
|
||||
buildCmd string
|
||||
serviceName string
|
||||
healthURL string
|
||||
configTmpl string
|
||||
configPath string
|
||||
unitTmpl string
|
||||
}{
|
||||
"appview": {
|
||||
ip: state.Appview.PublicIP,
|
||||
@@ -68,6 +73,9 @@ func cmdUpdate(target string) error {
|
||||
buildCmd: "appview",
|
||||
serviceName: naming.Appview(),
|
||||
healthURL: "http://localhost:5000/health",
|
||||
configTmpl: appviewConfigTmpl,
|
||||
configPath: naming.AppviewConfigPath(),
|
||||
unitTmpl: appviewServiceTmpl,
|
||||
},
|
||||
"hold": {
|
||||
ip: state.Hold.PublicIP,
|
||||
@@ -75,6 +83,9 @@ func cmdUpdate(target string) error {
|
||||
buildCmd: "hold",
|
||||
serviceName: naming.Hold(),
|
||||
healthURL: "http://localhost:8080/xrpc/_health",
|
||||
configTmpl: holdConfigTmpl,
|
||||
configPath: naming.HoldConfigPath(),
|
||||
unitTmpl: holdServiceTmpl,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -92,6 +103,37 @@ func cmdUpdate(target string) error {
|
||||
t := targets[name]
|
||||
fmt.Printf("Updating %s (%s)...\n", name, t.ip)
|
||||
|
||||
// Sync config keys (adds missing keys from template, never overwrites)
|
||||
configYAML, err := renderConfig(t.configTmpl, vals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("render %s config: %w", name, err)
|
||||
}
|
||||
if err := syncConfigKeys(name, t.ip, t.configPath, configYAML); err != nil {
|
||||
return fmt.Errorf("%s config sync: %w", name, err)
|
||||
}
|
||||
|
||||
// Sync systemd service unit
|
||||
renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName,
|
||||
ConfigPath: t.configPath,
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: t.serviceName,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("render %s service unit: %w", name, err)
|
||||
}
|
||||
unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s service unit sync: %w", name, err)
|
||||
}
|
||||
|
||||
daemonReload := ""
|
||||
if unitChanged {
|
||||
daemonReload = "systemctl daemon-reload"
|
||||
}
|
||||
|
||||
updateScript := fmt.Sprintf(`set -euo pipefail
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
export GOTMPDIR=/var/tmp
|
||||
@@ -113,11 +155,12 @@ CGO_ENABLED=1 go build \
|
||||
-ldflags="-s -w -linkmode external -extldflags '-static'" \
|
||||
-tags sqlite_omit_load_extension -trimpath \
|
||||
-o bin/%s ./cmd/%s
|
||||
%s
|
||||
systemctl restart %s
|
||||
|
||||
sleep 2
|
||||
curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
|
||||
`, goVersion, naming.InstallDir(), branch, t.binaryName, t.buildCmd, t.serviceName, t.healthURL)
|
||||
`, goVersion, naming.InstallDir(), branch, t.binaryName, t.buildCmd, daemonReload, t.serviceName, t.healthURL)
|
||||
|
||||
output, err := runSSH(t.ip, updateScript, true)
|
||||
if err != nil {
|
||||
@@ -139,6 +182,27 @@ curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
|
||||
return nil
|
||||
}
|
||||
|
||||
// configValsFromState builds ConfigValues from persisted state.
|
||||
// S3SecretKey is intentionally left empty — syncConfigKeys only adds missing
|
||||
// keys and never overwrites, so the server's existing secret is preserved.
|
||||
func configValsFromState(state *InfraState) *ConfigValues {
|
||||
naming := state.Naming()
|
||||
_, baseDomain, _, _ := extractFromAppviewTemplate()
|
||||
holdDomain := state.Zone + ".cove." + baseDomain
|
||||
|
||||
return &ConfigValues{
|
||||
S3Endpoint: state.ObjectStorage.Endpoint,
|
||||
S3Region: state.ObjectStorage.Region,
|
||||
S3Bucket: state.ObjectStorage.Bucket,
|
||||
S3AccessKey: state.ObjectStorage.AccessKeyID,
|
||||
S3SecretKey: "", // not persisted in state; existing value on server is preserved
|
||||
Zone: state.Zone,
|
||||
HoldDomain: holdDomain,
|
||||
HoldDid: "did:web:" + holdDomain,
|
||||
BasePath: naming.BasePath(),
|
||||
}
|
||||
}
|
||||
|
||||
func cmdSSH(target string) error {
|
||||
state, err := loadState()
|
||||
if err != nil {
|
||||
|
||||
@@ -1,23 +1,51 @@
|
||||
# ATCR AppView UI - Future Features
|
||||
# ATCR UI - Feature Roadmap
|
||||
|
||||
This document outlines potential features for future versions of the ATCR AppView UI, beyond the V1 MVP. These are ideas to consider as the project matures and user needs evolve.
|
||||
This document tracks the status of ATCR features beyond the V1 MVP. Features are marked with their current status:
|
||||
|
||||
- **DONE** — Fully implemented and shipping
|
||||
- **PARTIAL** — Some parts implemented
|
||||
- **BACKEND ONLY** — Backend exists, no UI yet
|
||||
- **NOT STARTED** — Future work
|
||||
- **BLOCKED** — Waiting on external dependency
|
||||
|
||||
---
|
||||
|
||||
## What's Already Built (not in original roadmap)
|
||||
|
||||
These features were implemented but weren't in the original future features list:
|
||||
|
||||
| Feature | Location | Notes |
|
||||
|---------|----------|-------|
|
||||
| **Billing (Stripe)** | `pkg/hold/billing/` | Checkout sessions, customer portal, subscription webhooks, tier upgrades. Build with `-tags billing`. |
|
||||
| **Garbage collection** | `pkg/hold/gc/` | Mark-and-sweep for orphaned blobs. Preview (dry-run) and execute modes. Triggered from hold admin UI. |
|
||||
| **libSQL embedded replicas** | AppView + Hold | Sync to Turso, Bunny DB, or self-hosted libsql-server. Configurable sync interval. |
|
||||
| **Hold successor/migration** | `pkg/hold/` | Promote a hold as successor to migrate users to new storage. |
|
||||
| **Relay management** | Hold admin | Manage firehose relay connections from admin panel. |
|
||||
| **Data export** | `pkg/appview/handlers/export.go` | GDPR-compliant export of all user data from AppView + all holds where user is member/captain. |
|
||||
| **Dark/light mode** | AppView UI | System preference detection, toggle, localStorage persistence. |
|
||||
| **Credential helper install page** | `/install` | Install scripts for macOS/Linux/Windows, version API. |
|
||||
| **Stars** | AppView UI | Star/unstar repos stored as `io.atcr.star` ATProto records, counts displayed. |
|
||||
|
||||
---
|
||||
|
||||
## Advanced Image Management
|
||||
|
||||
### Multi-Architecture Image Support
|
||||
### Multi-Architecture Image Support — DONE (display) / NOT STARTED (creation)
|
||||
|
||||
**Display image indexes:**
|
||||
- Show when a tag points to an image index (multi-arch manifest)
|
||||
- Display all architectures/platforms in the index (linux/amd64, linux/arm64, darwin/arm64, etc.)
|
||||
**Display image indexes — DONE:**
|
||||
- Show when a tag points to an image index (multi-arch manifest) — `IsMultiArch` flag, "Multi-arch" badge
|
||||
- Display all architectures/platforms in the index — platform badges (e.g., linux/amd64, linux/arm64)
|
||||
- Allow viewing individual manifests within the index
|
||||
- Show platform-specific layer details
|
||||
- Show platform-specific details
|
||||
|
||||
**Image index creation:**
|
||||
**Image index creation — NOT STARTED:**
|
||||
- UI for combining multiple single-arch manifests into an image index
|
||||
- Automatic platform detection from manifest metadata
|
||||
- Validate that all manifests are for the same image (different platforms)
|
||||
|
||||
### Layer Inspection & Visualization
|
||||
### Layer Inspection & Visualization — NOT STARTED
|
||||
|
||||
DB stores layer metadata (digest, size, media type, layer index) but there's no UI for any of this.
|
||||
|
||||
**Layer details page:**
|
||||
- Show Dockerfile command that created each layer (if available in history)
|
||||
@@ -30,594 +58,409 @@ This document outlines potential features for future versions of the ATCR AppVie
|
||||
- Calculate storage savings from layer sharing
|
||||
- Identify duplicate layers with different digests (potential optimization)
|
||||
|
||||
### Image Operations
|
||||
### Image Operations — PARTIAL (delete only)
|
||||
|
||||
**Tag Management:**
|
||||
- **Tag promotion workflow:** dev → staging → prod with one click
|
||||
- **Tag aliases:** Create multiple tags pointing to same digest
|
||||
- **Tag patterns:** Auto-tag based on git commit, semantic version, date
|
||||
- **Tag protection:** Mark tags as immutable (prevent deletion/re-pointing)
|
||||
**Tag/manifest deletion — DONE:**
|
||||
- Delete tags with `DeleteTagHandler` (cascade + confirmation modal)
|
||||
- Delete manifests with `DeleteManifestHandler` (handles tagged manifests gracefully)
|
||||
|
||||
**Image Copying:**
|
||||
**Tag Management — NOT STARTED:**
|
||||
- Tag promotion workflow (dev → staging → prod)
|
||||
- Tag aliases (multiple tags → same digest)
|
||||
- Tag patterns (auto-tag based on git commit, semantic version, date)
|
||||
- Tag protection (mark tags as immutable)
|
||||
|
||||
**Image Copying — NOT STARTED:**
|
||||
- Copy image from one repository to another
|
||||
- Copy image from another user's repository (fork)
|
||||
- Bulk copy operations (copy all tags, copy all manifests)
|
||||
- Bulk copy operations
|
||||
|
||||
**Image History:**
|
||||
- Timeline view of tag changes (what digest did "latest" point to over time)
|
||||
- Rollback functionality (revert tag to previous digest)
|
||||
- Audit log of all image operations (push, delete, tag changes)
|
||||
**Image History — NOT STARTED:**
|
||||
- Timeline view of tag changes
|
||||
- Rollback functionality
|
||||
- Audit log of image operations
|
||||
|
||||
### Vulnerability Scanning
|
||||
### Vulnerability Scanning — DONE (backend) / NOT STARTED (UI)
|
||||
|
||||
**Integration with security scanners:**
|
||||
- **Trivy** - Comprehensive vulnerability scanner
|
||||
- **Grype** - Anchore's vulnerability scanner
|
||||
- **Clair** - CoreOS vulnerability scanner
|
||||
**Backend — DONE:**
|
||||
- Separate scanner service (`scanner/` module) with Syft (SBOM) + Grype (vulnerabilities)
|
||||
- WebSocket-based job queue connecting scanner to hold service
|
||||
- Priority queue with tier-based scheduling (quartermaster > bosun > deckhand)
|
||||
- Scan results stored as ORAS artifacts in S3, referenced in hold PDS
|
||||
- Automatic scanning dispatched by hold on manifest push
|
||||
- See `docs/SBOM_SCANNING.md`
|
||||
|
||||
**Features:**
|
||||
- Automatic scanning on image push
|
||||
**AppView UI — NOT STARTED:**
|
||||
- Display CVE count by severity (critical, high, medium, low)
|
||||
- Show detailed CVE information (description, CVSS score, affected packages)
|
||||
- Filter images by vulnerability status
|
||||
- Subscribe to CVE notifications for your images
|
||||
- Compare vulnerability status across tags/versions
|
||||
|
||||
### Image Signing & Verification
|
||||
### Image Signing & Verification — NOT STARTED
|
||||
|
||||
**Cosign/Sigstore integration:**
|
||||
- Sign images with Cosign
|
||||
Concept doc exists at `docs/SIGNATURE_INTEGRATION.md` but no implementation.
|
||||
|
||||
- Sign images
|
||||
- Display signature verification status
|
||||
- Show keyless signing certificate chains
|
||||
- Integrate with transparency log (Rekor)
|
||||
|
||||
**Features:**
|
||||
- UI for signing images (generate key, sign manifest)
|
||||
- Verify signatures before pull (browser-based verification)
|
||||
- Display signature metadata (signer, timestamp, transparency log entry)
|
||||
- Display signature metadata
|
||||
- Require signatures for protected repositories
|
||||
|
||||
### SBOM (Software Bill of Materials)
|
||||
### SBOM (Software Bill of Materials) — DONE (backend) / NOT STARTED (UI)
|
||||
|
||||
**SBOM generation and display:**
|
||||
- Generate SBOM on push (SPDX or CycloneDX format)
|
||||
**Backend — DONE:**
|
||||
- Syft generates SPDX JSON format SBOMs
|
||||
- Stored as ORAS artifacts (referenced via `artifactType: "application/spdx+json"`)
|
||||
- Blobs in S3, metadata in hold's PDS
|
||||
- Accessible via ORAS CLI and hold XRPC endpoints
|
||||
|
||||
**UI — NOT STARTED:**
|
||||
- Display package list from SBOM
|
||||
- Show license information
|
||||
- Link to upstream package sources
|
||||
- Compare SBOMs across versions (what packages changed)
|
||||
- Compare SBOMs across versions
|
||||
|
||||
**SBOM attestation:**
|
||||
- Store SBOM as attestation (in-toto format)
|
||||
- Link SBOM to image signature
|
||||
- Verify SBOM integrity
|
||||
---
|
||||
|
||||
## Hold Management Dashboard
|
||||
## Hold Management Dashboard — DONE (on hold admin panel)
|
||||
|
||||
### Hold Discovery & Registration
|
||||
Hold management is implemented as a separate admin panel on the hold service itself (`pkg/hold/admin/`), not in the AppView UI. This makes sense architecturally — hold owners manage their own holds.
|
||||
|
||||
**Create hold:**
|
||||
### Hold Discovery & Registration — PARTIAL
|
||||
|
||||
**Hold registration — DONE:**
|
||||
- Automatic registration on hold startup (captain + crew records created in embedded PDS)
|
||||
- Auto-detection of region from cloud metadata
|
||||
|
||||
**NOT STARTED:**
|
||||
- UI wizard for deploying hold service
|
||||
- One-click deployment to Fly.io, Railway, Render
|
||||
- Configuration generator (environment variables, docker-compose)
|
||||
- Test connectivity after deployment
|
||||
- One-click deployment to cloud platforms
|
||||
- Configuration generator
|
||||
- Test connectivity UI
|
||||
|
||||
**Hold registration:**
|
||||
- Automatic registration via OAuth (already implemented)
|
||||
- Manual registration form (for existing holds)
|
||||
- Bulk import holds from JSON/YAML
|
||||
### Hold Configuration — DONE (admin panel)
|
||||
|
||||
### Hold Configuration
|
||||
|
||||
**Hold settings page:**
|
||||
- Edit hold metadata (name, description, icon)
|
||||
**Hold settings — DONE (hold admin):**
|
||||
- Toggle public/private flag
|
||||
- Configure storage backend (S3, Storj, Minio, filesystem)
|
||||
- Set storage quotas and limits
|
||||
- Configure retention policies (auto-delete old blobs)
|
||||
- Toggle allow-all-crew
|
||||
- Toggle Bluesky post announcements
|
||||
- Set successor hold DID for migration
|
||||
- Writes changes back to YAML config file
|
||||
|
||||
**Hold credentials:**
|
||||
- Rotate S3 access keys
|
||||
- Test hold connectivity
|
||||
- View hold service logs (if accessible)
|
||||
**Storage config — YAML-only:**
|
||||
- S3 credentials, region, bucket, endpoint, CDN pull zone all configured via YAML
|
||||
- No UI for editing S3 credentials or rotating keys
|
||||
|
||||
### Crew Management
|
||||
**Quotas — DONE (read-only UI):**
|
||||
- Tier-based limits (deckhand 5GB, bosun 50GB, quartermaster 100GB)
|
||||
- Per-user quota tracking and display in admin
|
||||
- Not editable via UI (requires YAML change)
|
||||
|
||||
**Invite crew members:**
|
||||
- Send invitation links (OAuth-based)
|
||||
- Invite by handle or DID
|
||||
- Set crew permissions (read-only, read-write, admin)
|
||||
- Bulk invite (upload CSV)
|
||||
**NOT STARTED:**
|
||||
- Retention policies (auto-delete old blobs)
|
||||
- Hold service log viewer
|
||||
|
||||
**Crew list:**
|
||||
- Display all crew members
|
||||
- Show last activity (last push, last pull)
|
||||
### Crew Management — DONE (hold admin panel)
|
||||
|
||||
**Implemented in `pkg/hold/admin/handlers_crew.go`:**
|
||||
- Add crew by DID with role, permissions (`blob:read`, `blob:write`, `crew:admin`), and tier
|
||||
- Crew list showing handle, role, permissions, tier, usage, quota
|
||||
- Edit crew permissions and tier
|
||||
- Remove crew members
|
||||
- Change crew permissions
|
||||
- Bulk JSON import/export with deduplication (`handlers_crew_io.go`)
|
||||
|
||||
**Crew request workflow:**
|
||||
- Allow users to request access to a hold
|
||||
- Hold owner approves/rejects requests
|
||||
- Notification system for requests
|
||||
**NOT STARTED:**
|
||||
- Invitation links (OAuth-based, currently must know DID)
|
||||
- Invite by handle (currently DID-only)
|
||||
- Crew request workflow (users can't self-request access)
|
||||
- Approval/rejection flow
|
||||
|
||||
### Hold Analytics
|
||||
### Hold Analytics — PARTIAL
|
||||
|
||||
**Storage metrics:**
|
||||
- Total storage used (bytes)
|
||||
- Blob count
|
||||
- Largest blobs
|
||||
- Growth over time (chart)
|
||||
- Deduplication savings
|
||||
**Storage metrics — DONE (hold admin):**
|
||||
- Total blobs, total size, unique digests
|
||||
- Per-user quota stats (total size, blob count)
|
||||
- Top users by storage (lazy-loaded HTMX partial)
|
||||
- Crew count and tier distribution
|
||||
|
||||
**Access metrics:**
|
||||
- Total downloads (pulls)
|
||||
- Bandwidth used
|
||||
- Popular images (most pulled)
|
||||
- Geographic distribution (if available)
|
||||
- Access logs (who pulled what, when)
|
||||
**NOT STARTED:**
|
||||
- Access metrics (downloads, pulls, bandwidth)
|
||||
- Growth over time charts
|
||||
- Cost estimation
|
||||
- Geographic distribution
|
||||
- Access logs
|
||||
|
||||
**Cost estimation:**
|
||||
- Calculate S3 storage costs
|
||||
- Calculate bandwidth costs
|
||||
- Compare costs across storage backends
|
||||
- Budget alerts (notify when approaching limit)
|
||||
---
|
||||
|
||||
## Discovery & Social Features
|
||||
|
||||
### Federated Browse & Search
|
||||
### Federated Browse & Search — PARTIAL
|
||||
|
||||
**Enhanced discovery:**
|
||||
- Full-text search across all ATCR images (repository name, tag, description)
|
||||
**Basic search — DONE:**
|
||||
- Full-text search across handles, DIDs, repo names, and annotations
|
||||
- Search UI with HTMX lazy loading and pagination
|
||||
- Navigation bar search component
|
||||
|
||||
**NOT STARTED:**
|
||||
- Filter by user, hold, architecture, date range
|
||||
- Sort by popularity, recency, size
|
||||
- Advanced query syntax (e.g., "user:alice tag:latest arch:arm64")
|
||||
- Advanced query syntax
|
||||
- Popular/trending images
|
||||
- Categories and user-defined tags
|
||||
|
||||
**Popular/Trending:**
|
||||
- Most pulled images (past day, week, month)
|
||||
- Fastest growing images (new pulls)
|
||||
- Recently updated images (new tags)
|
||||
- Community favorites (curated list)
|
||||
### Sailor Profiles — PARTIAL
|
||||
|
||||
**Categories & Tags:**
|
||||
- User-defined categories (web, database, ml, etc.)
|
||||
- Tag images with keywords (nginx, proxy, reverse-proxy)
|
||||
- Browse by category
|
||||
- Tag cloud visualization
|
||||
**Public profile page — DONE:**
|
||||
- `/u/{handle}` shows user's avatar, handle, DID, and all public repositories
|
||||
- OpenGraph meta tags and JSON-LD structured data
|
||||
|
||||
### Sailor Profiles (Public)
|
||||
|
||||
**Public profile page:**
|
||||
- `/ui/@alice` shows alice's public repositories
|
||||
- Bio, avatar, website links
|
||||
**NOT STARTED:**
|
||||
- Bio/description field
|
||||
- Website links
|
||||
- Statistics (total images, total pulls, joined date)
|
||||
- Pinned repositories (showcase best images)
|
||||
- Pinned/featured repositories
|
||||
|
||||
**Social features:**
|
||||
- Follow other sailors (get notified of their pushes)
|
||||
- Star repositories (bookmark favorites)
|
||||
- Comment on images (feedback, questions)
|
||||
### Social Features — PARTIAL (stars only)
|
||||
|
||||
**Stars — DONE:**
|
||||
- Star/unstar repositories stored as `io.atcr.star` ATProto records
|
||||
- Star counts displayed on repository pages
|
||||
|
||||
**NOT STARTED:**
|
||||
- Follow other sailors
|
||||
- Comment on images
|
||||
- Like/upvote images
|
||||
- Activity feed
|
||||
- Federated timeline / custom feeds
|
||||
- Sharing to Bluesky/ATProto social apps
|
||||
|
||||
**Activity feed:**
|
||||
- Timeline of followed sailors' activity
|
||||
- Recent pushes from community
|
||||
- Popular images from followed users
|
||||
|
||||
### Federated Timeline
|
||||
|
||||
**ATProto-native feed:**
|
||||
- Real-time feed of container pushes (like Bluesky's timeline)
|
||||
- Filter by follows, community, or global
|
||||
- React to pushes (like, share, comment)
|
||||
- Share images to Bluesky/ATProto social apps
|
||||
|
||||
**Custom feeds:**
|
||||
- Create algorithmic feeds (e.g., "Show me all ML images")
|
||||
- Subscribe to curated feeds
|
||||
- Publish feeds for others to subscribe
|
||||
---
|
||||
|
||||
## Access Control & Permissions
|
||||
|
||||
### Repository-Level Permissions
|
||||
### Hold-Level Access Control — DONE
|
||||
|
||||
**Private repositories:**
|
||||
- Mark repositories as private (only owner + collaborators can pull)
|
||||
- Invite collaborators by handle/DID
|
||||
- Set permissions (read-only, read-write, admin)
|
||||
- Public/private hold toggle (admin UI + OCI enforcement)
|
||||
- Crew permissions: `blob:read`, `blob:write`, `crew:admin`
|
||||
- `blob:write` implicitly grants `blob:read`
|
||||
- Captain has all permissions implicitly
|
||||
- See `docs/BYOS.md`
|
||||
|
||||
**Public repositories:**
|
||||
- Default: public (anyone can pull)
|
||||
- Require authentication for private repos
|
||||
- Generate read-only tokens (for CI/CD)
|
||||
### Repository-Level Permissions — BLOCKED
|
||||
|
||||
**Implementation challenge:**
|
||||
- ATProto doesn't support private records yet
|
||||
- May require proxy layer for access control
|
||||
- Or use encrypted blobs with shared keys
|
||||
- **Private repositories blocked by ATProto** — no private records support yet
|
||||
- Repository-level permissions, collaborator invites, read-only tokens all depend on this
|
||||
- May require proxy layer or encrypted blobs when ATProto adds private record support
|
||||
|
||||
### Team/Organization Accounts
|
||||
### Team/Organization Accounts — NOT STARTED
|
||||
|
||||
**Multi-user organizations:**
|
||||
- Create organization account (e.g., `@acme-corp`)
|
||||
- Add members with roles (owner, maintainer, member)
|
||||
- Organization-owned repositories
|
||||
- Billing and quotas at org level
|
||||
- Organization accounts, RBAC, SSO, audit logs
|
||||
- Likely a later-stage feature
|
||||
|
||||
**Features:**
|
||||
- Team-based access control
|
||||
- Shared hold for organization
|
||||
- Audit logs for all org activity
|
||||
- Single sign-on (SSO) integration
|
||||
---
|
||||
|
||||
## Analytics & Monitoring
|
||||
|
||||
### Dashboard
|
||||
### Dashboard — PARTIAL
|
||||
|
||||
**Personal dashboard:**
|
||||
**Hold dashboard — DONE (hold admin):**
|
||||
- Storage usage, crew count, tier distribution
|
||||
|
||||
**Personal dashboard — NOT STARTED:**
|
||||
- Overview of your images, holds, activity
|
||||
- Quick stats (total size, pull count, last push)
|
||||
- Recent activity (your pushes, pulls)
|
||||
- Alerts and notifications
|
||||
- Quick stats, recent activity, alerts
|
||||
|
||||
**Hold dashboard:**
|
||||
- Storage usage, bandwidth, costs
|
||||
- Active crew members
|
||||
- Recent uploads/downloads
|
||||
- Health status of hold service
|
||||
### Pull Analytics — NOT STARTED
|
||||
|
||||
### Pull Analytics
|
||||
|
||||
**Detailed metrics:**
|
||||
- Pull count per image/tag
|
||||
- Pull count by client (Docker, containerd, podman)
|
||||
- Pull count by geography (country, region)
|
||||
- Pull count over time (chart)
|
||||
- Failed pulls (errors, retries)
|
||||
- Pull count by client, geography, over time
|
||||
- User analytics (authenticated vs anonymous)
|
||||
|
||||
**User analytics:**
|
||||
- Who is pulling your images (if authenticated)
|
||||
- Anonymous vs authenticated pulls
|
||||
- Repeat users vs new users
|
||||
### Alerts & Notifications — NOT STARTED
|
||||
|
||||
### Alerts & Notifications
|
||||
- Alert types (quota exceeded, vulnerability detected, hold down, etc.)
|
||||
- Notification channels (email, webhook, ATProto, Slack/Discord)
|
||||
|
||||
**Alert types:**
|
||||
- Storage quota exceeded
|
||||
- High bandwidth usage
|
||||
- New vulnerability detected
|
||||
- Image signature invalid
|
||||
- Hold service down
|
||||
- Crew member joined/left
|
||||
|
||||
**Notification channels:**
|
||||
- Email
|
||||
- Webhook (POST to custom URL)
|
||||
- ATProto app notification (future: in-app notifications in Bluesky)
|
||||
- Slack, Discord, Telegram integrations
|
||||
---
|
||||
|
||||
## Developer Tools & Integrations
|
||||
|
||||
### API Documentation
|
||||
### Credential Helper — DONE
|
||||
|
||||
**Interactive API docs:**
|
||||
- Swagger/OpenAPI spec for OCI API
|
||||
- Swagger/OpenAPI spec for UI API
|
||||
- Interactive API explorer (try API calls in browser)
|
||||
- Code examples in multiple languages (curl, Go, Python, JavaScript)
|
||||
- Install page at `/install` with shell scripts
|
||||
- Version API endpoint for automatic updates
|
||||
|
||||
**SDK/Client Libraries:**
|
||||
- Official Go client library
|
||||
- JavaScript/TypeScript client
|
||||
- Python client
|
||||
- Rust client
|
||||
### API Documentation — NOT STARTED
|
||||
|
||||
### Webhooks
|
||||
- Swagger/OpenAPI specs
|
||||
- Interactive API explorer
|
||||
- Code examples, SDKs
|
||||
|
||||
**Webhook configuration:**
|
||||
- Register webhook URLs per repository
|
||||
- Select events to trigger (push, delete, tag update)
|
||||
- Test webhooks (send test payload)
|
||||
- View webhook delivery history
|
||||
- Retry failed deliveries
|
||||
### Webhooks — NOT STARTED
|
||||
|
||||
**Webhook events:**
|
||||
- `manifest.pushed`
|
||||
- `manifest.deleted`
|
||||
- `tag.created`
|
||||
- `tag.updated`
|
||||
- `tag.deleted`
|
||||
- `scan.completed` (vulnerability scan finished)
|
||||
- Repository-level webhook registration
|
||||
- Events: manifest.pushed, tag.created, scan.completed, etc.
|
||||
- Test, retry, delivery history
|
||||
|
||||
### CI/CD Integration Guides
|
||||
### CI/CD Integration — NOT STARTED
|
||||
|
||||
**Documentation for popular CI/CD platforms:**
|
||||
- GitHub Actions (example workflows)
|
||||
- GitLab CI (.gitlab-ci.yml examples)
|
||||
- CircleCI (config.yml examples)
|
||||
- Jenkins (Jenkinsfile examples)
|
||||
- Drone CI
|
||||
- GitHub Actions, GitLab CI, CircleCI example workflows
|
||||
- Pre-built actions/plugins
|
||||
- Build status badges
|
||||
|
||||
**Features:**
|
||||
- One-click workflow generation
|
||||
- Pre-built actions/plugins for ATCR
|
||||
- Cache layer optimization for faster builds
|
||||
- Build status badges (show build status in README)
|
||||
### Infrastructure as Code — PARTIAL
|
||||
|
||||
### Infrastructure as Code
|
||||
**DONE:**
|
||||
- Custom UpCloud deployment tool (`deploy/upcloud/`) with Go-based provisioning, cloud-init, systemd, config templates
|
||||
- Docker Compose for dev and production
|
||||
|
||||
**IaC examples:**
|
||||
- Terraform module for deploying hold service
|
||||
- Pulumi program for ATCR infrastructure
|
||||
- Kubernetes manifests for hold service
|
||||
- Docker Compose for local development
|
||||
- Helm chart for AppView + hold
|
||||
**NOT STARTED:**
|
||||
- Terraform modules
|
||||
- Helm charts
|
||||
- Kubernetes manifests (only an example verification webhook exists)
|
||||
- GitOps integrations (ArgoCD, FluxCD)
|
||||
|
||||
**GitOps workflows:**
|
||||
- ArgoCD integration (deploy images from ATCR)
|
||||
- FluxCD integration
|
||||
- Automated deployments on tag push
|
||||
---
|
||||
|
||||
## Documentation & Onboarding
|
||||
## Documentation & Onboarding — PARTIAL
|
||||
|
||||
### Interactive Getting Started
|
||||
**DONE:**
|
||||
- Install page with credential helper setup
|
||||
- Learn more page
|
||||
- Internal developer docs (`docs/`)
|
||||
|
||||
**Onboarding wizard:**
|
||||
- Step-by-step guide for first-time users
|
||||
- Interactive tutorial (push your first image)
|
||||
- Verify setup (test authentication, test push/pull)
|
||||
- Completion checklist
|
||||
|
||||
**Guided tours:**
|
||||
- Product tour of UI features
|
||||
- Tooltips and hints for new users
|
||||
**NOT STARTED:**
|
||||
- Interactive onboarding wizard
|
||||
- Product tour / tooltips
|
||||
- Help center with FAQs
|
||||
- Video tutorials
|
||||
- Comprehensive user-facing documentation site
|
||||
|
||||
### Comprehensive Documentation
|
||||
|
||||
**Documentation sections:**
|
||||
- Quickstart guide
|
||||
- Detailed user manual
|
||||
- API reference
|
||||
- ATProto record schemas
|
||||
- Deployment guides (hold service, AppView)
|
||||
- Troubleshooting guide
|
||||
- Security best practices
|
||||
|
||||
**Video tutorials:**
|
||||
- YouTube channel with how-to videos
|
||||
- Screen recordings of common tasks
|
||||
- Conference talks and demos
|
||||
|
||||
### Community & Support
|
||||
|
||||
**Community features:**
|
||||
- Discussion forum (or integrate with Discourse)
|
||||
- GitHub Discussions for ATCR project
|
||||
- Discord/Slack community
|
||||
- Monthly community calls
|
||||
|
||||
**Support channels:**
|
||||
- Email support
|
||||
- Live chat (for paid tiers)
|
||||
- Priority support (for enterprise)
|
||||
---
|
||||
|
||||
## Advanced ATProto Integration
|
||||
|
||||
### Record Viewer
|
||||
### Data Export — DONE
|
||||
|
||||
**ATProto record browser:**
|
||||
- Browse all your `io.atcr.*` records
|
||||
- Raw JSON view with ATProto metadata (CID, commit info, timestamp)
|
||||
- Diff viewer for record updates
|
||||
- History view (see all versions of a record)
|
||||
- Link to ATP URI (`at://did/collection/rkey`)
|
||||
- GDPR-compliant data export (`ExportUserDataHandler`)
|
||||
- Fetches data from AppView DB + all holds where user is member/captain
|
||||
|
||||
**Export/Import:**
|
||||
- Export all records as JSON (backup)
|
||||
- Import records from JSON (restore, migration)
|
||||
- CAR file export (ATProto native format)
|
||||
### Record Viewer — NOT STARTED
|
||||
|
||||
### PDS Integration
|
||||
- Browse `io.atcr.*` records with raw JSON view
|
||||
- Record history, diff viewer
|
||||
- ATP URI links
|
||||
|
||||
**Multi-PDS support:**
|
||||
- Switch between multiple PDS accounts
|
||||
- Manage images across different PDSs
|
||||
- Unified view of all your images (across PDSs)
|
||||
### PDS Integration — NOT STARTED
|
||||
|
||||
**PDS health monitoring:**
|
||||
- Show PDS connection status
|
||||
- Alert if PDS is unreachable
|
||||
- Fallback to alternate PDS (if configured)
|
||||
- Multi-PDS support, PDS health monitoring
|
||||
- PDS migration tools
|
||||
- "Verify on PDS" button
|
||||
|
||||
**PDS migration tools:**
|
||||
- Migrate images from one PDS to another
|
||||
- Bulk update hold endpoints
|
||||
- Re-sign OAuth tokens for new PDS
|
||||
### Federation — NOT STARTED
|
||||
|
||||
### Decentralization Features
|
||||
- Cross-AppView image pulls
|
||||
- AppView discovery
|
||||
- Federated search
|
||||
|
||||
**Data sovereignty:**
|
||||
- "Verify on PDS" button (proves manifest is in your PDS)
|
||||
- "Clone my registry" guide (backup to another PDS)
|
||||
- "Export registry" (download all manifests + metadata)
|
||||
|
||||
**Federation:**
|
||||
- Cross-AppView image pulls (pull from other ATCR AppViews)
|
||||
- AppView discovery (find other ATCR instances)
|
||||
- Federated search (search across multiple AppViews)
|
||||
|
||||
## Enterprise Features (Future Commercial Offering)
|
||||
|
||||
### Team Collaboration
|
||||
|
||||
**Organizations:**
|
||||
- Enterprise org accounts with unlimited members
|
||||
- RBAC (role-based access control)
|
||||
- SSO integration (SAML, OIDC)
|
||||
- Audit logs for compliance
|
||||
|
||||
### Compliance & Security
|
||||
|
||||
**Compliance tools:**
|
||||
- SOC 2 compliance reporting
|
||||
- HIPAA-compliant storage options
|
||||
- GDPR data export/deletion
|
||||
- Retention policies (auto-delete after N days)
|
||||
|
||||
**Security features:**
|
||||
- Image scanning with policy enforcement (block vulnerable images)
|
||||
- Malware scanning (scan blobs for malware)
|
||||
- Secrets scanning (detect leaked credentials in layers)
|
||||
- Content trust (require signed images)
|
||||
|
||||
### SLA & Support
|
||||
|
||||
**Paid tiers:**
|
||||
- Free tier: 5GB storage, community support
|
||||
- Pro tier: 100GB storage, email support, SLA
|
||||
- Enterprise tier: Unlimited storage, priority support, dedicated instance
|
||||
|
||||
**Features:**
|
||||
- Guaranteed uptime (99.9%)
|
||||
- Premium support (24/7, faster response)
|
||||
- Dedicated account manager
|
||||
- Custom contract terms
|
||||
---
|
||||
|
||||
## UI/UX Enhancements
|
||||
|
||||
### Design System
|
||||
### Theming — PARTIAL
|
||||
|
||||
**Theming:**
|
||||
- Light and dark modes (system preference)
|
||||
- Custom themes (nautical, cyberpunk, minimalist)
|
||||
- Accessibility (WCAG 2.1 AA compliance)
|
||||
**DONE:**
|
||||
- Light/dark mode with system preference detection and toggle
|
||||
- Responsive design (Tailwind/DaisyUI, mobile-friendly)
|
||||
- PWA manifest with icons (no service worker yet)
|
||||
|
||||
**NOT STARTED:**
|
||||
- Custom themes
|
||||
- WCAG 2.1 AA accessibility audit
|
||||
- High contrast mode
|
||||
- Internationalization (i18n)
|
||||
- Native mobile apps
|
||||
|
||||
**Responsive design:**
|
||||
- Mobile-first design
|
||||
- Progressive web app (PWA) with offline support
|
||||
- Native mobile apps (iOS, Android)
|
||||
### Performance — PARTIAL
|
||||
|
||||
### Performance Optimizations
|
||||
**DONE:**
|
||||
- HTMX lazy loading for data-heavy partials
|
||||
- Efficient server-side rendering
|
||||
|
||||
**Frontend optimizations:**
|
||||
- Lazy loading for images and data
|
||||
**NOT STARTED:**
|
||||
- Service worker for offline caching
|
||||
- Virtual scrolling for large lists
|
||||
- Service worker for caching
|
||||
- Code splitting (load only what's needed)
|
||||
- GraphQL API
|
||||
- Real-time WebSocket updates in UI
|
||||
|
||||
**Backend optimizations:**
|
||||
- GraphQL API (fetch only required fields)
|
||||
- Real-time updates via WebSocket
|
||||
- Server-sent events for firehose
|
||||
- Edge caching (CloudFlare, Fastly)
|
||||
---
|
||||
|
||||
### Internationalization
|
||||
## Enterprise Features — NOT STARTED (except billing)
|
||||
|
||||
**Multi-language support:**
|
||||
- UI translations (English, Spanish, French, German, Japanese, Chinese, etc.)
|
||||
- RTL (right-to-left) language support
|
||||
- Localized date/time formats
|
||||
- Locale-specific formatting (numbers, currencies)
|
||||
### Billing — DONE
|
||||
|
||||
## Miscellaneous Ideas
|
||||
- Stripe integration (`pkg/hold/billing/`, requires `-tags billing` build tag)
|
||||
- Checkout sessions, customer portal, subscription webhooks
|
||||
- Tier upgrades/downgrades
|
||||
|
||||
### Image Build Service
|
||||
### Everything Else — NOT STARTED
|
||||
|
||||
**Cloud-based builds:**
|
||||
- Build images from Dockerfile in the UI
|
||||
- Multi-stage build support
|
||||
- Build cache optimization
|
||||
- Build logs and status
|
||||
- Organization accounts with SSO (SAML, OIDC)
|
||||
- RBAC, audit logs for compliance
|
||||
- SOC 2, HIPAA, GDPR compliance tooling (data export exists, see above)
|
||||
- Image scanning policy enforcement
|
||||
- Paid tier SLAs
|
||||
|
||||
**Automated builds:**
|
||||
- Connect GitHub/GitLab repository
|
||||
- Auto-build on git push
|
||||
- Build matrix (multiple architectures, versions)
|
||||
- Build notifications
|
||||
---
|
||||
|
||||
### Image Registry Mirroring
|
||||
## Miscellaneous Ideas — NOT STARTED
|
||||
|
||||
**Mirror external registries:**
|
||||
- Cache images from Docker Hub, ghcr.io, quay.io
|
||||
- Transparent proxy (pull-through cache)
|
||||
- Reduce external bandwidth costs
|
||||
- Faster pulls (cache locally)
|
||||
These remain future ideas with no implementation:
|
||||
|
||||
**Features:**
|
||||
- Configurable cache retention
|
||||
- Whitelist/blacklist registries
|
||||
- Statistics (cache hit rate, savings)
|
||||
- **Image build service** — Cloud-based Dockerfile builds
|
||||
- **Registry mirroring** — Pull-through cache for Docker Hub, ghcr.io, etc.
|
||||
- **Deployment tools** — One-click deploy to K8s, ECS, Fly.io
|
||||
- **Image recommendations** — ML-based "similar images" and "people also pulled"
|
||||
- **Gamification** — Achievement badges, leaderboards
|
||||
- **Advanced search** — Semantic/AI-powered search, saved searches
|
||||
|
||||
### Deployment Tools
|
||||
---
|
||||
|
||||
**One-click deployments:**
|
||||
- Deploy image to Kubernetes
|
||||
- Deploy to Docker Swarm
|
||||
- Deploy to AWS ECS/Fargate
|
||||
- Deploy to Fly.io, Railway, Render
|
||||
## Updated Priority List
|
||||
|
||||
**Deployment tracking:**
|
||||
- Track where images are deployed
|
||||
- Show running versions (which environments use which tags)
|
||||
- Notify on new deployments
|
||||
**Already done (was "High Priority"):**
|
||||
1. ~~Multi-architecture image support~~ — display working
|
||||
2. ~~Vulnerability scanning integration~~ — backend complete
|
||||
3. ~~Hold management dashboard~~ — implemented on hold admin panel
|
||||
4. ~~Basic search~~ — working
|
||||
|
||||
### Image Recommendations
|
||||
**Remaining high priority:**
|
||||
1. Scan results UI in AppView (backend exists, just needs frontend)
|
||||
2. SBOM display UI in AppView (backend exists, just needs frontend)
|
||||
3. Webhooks for CI/CD integration
|
||||
4. Enhanced search (filters, sorting, advanced queries)
|
||||
5. Richer sailor profiles (bio, stats, pinned repos)
|
||||
|
||||
**ML-based recommendations:**
|
||||
- "Similar images" (based on layers, packages, tags)
|
||||
- "People who pulled this also pulled..." (collaborative filtering)
|
||||
- "Recommended for you" (personalized based on history)
|
||||
**Medium priority:**
|
||||
1. Layer inspection UI
|
||||
2. Pull analytics and monitoring
|
||||
3. API documentation (Swagger/OpenAPI)
|
||||
4. Tag management (promotion, protection, aliases)
|
||||
5. Onboarding wizard / getting started guide
|
||||
|
||||
### Gamification
|
||||
|
||||
**Achievements:**
|
||||
- Badges for milestones (first push, 100 pulls, 1GB storage, etc.)
|
||||
- Leaderboards (most popular images, most active sailors)
|
||||
- Community contributions (points for helping others)
|
||||
|
||||
### Advanced Search
|
||||
|
||||
**Semantic search:**
|
||||
- Search by description, README, labels
|
||||
- Natural language queries ("show me nginx images with SSL")
|
||||
- AI-powered search (GPT-based understanding)
|
||||
|
||||
**Saved searches:**
|
||||
- Save frequently used queries
|
||||
- Subscribe to search results (get notified of new matches)
|
||||
- Share searches with team
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
If implementing these features, suggested priority order:
|
||||
|
||||
**High Priority (Next 6 months):**
|
||||
1. Multi-architecture image support
|
||||
2. Vulnerability scanning integration
|
||||
3. Hold management dashboard
|
||||
4. Enhanced search and filtering
|
||||
5. Webhooks for CI/CD integration
|
||||
|
||||
**Medium Priority (6-12 months):**
|
||||
**Low priority / long-term:**
|
||||
1. Team/organization accounts
|
||||
2. Repository-level permissions
|
||||
3. Image signing and verification
|
||||
4. Pull analytics and monitoring
|
||||
5. API documentation and SDKs
|
||||
|
||||
**Low Priority (12+ months):**
|
||||
1. Enterprise features (SSO, compliance, SLA)
|
||||
2. Image build service
|
||||
3. Registry mirroring
|
||||
4. Mobile apps
|
||||
5. ML-based recommendations
|
||||
4. Federation features
|
||||
5. Internationalization
|
||||
|
||||
**Research/Experimental:**
|
||||
**Blocked on external dependencies:**
|
||||
1. Private repositories (requires ATProto private records)
|
||||
2. Federated timeline (requires ATProto feed infrastructure)
|
||||
3. Deployment tools integration
|
||||
4. Semantic search
|
||||
|
||||
---
|
||||
|
||||
**Note:** This is a living document. Features may be added, removed, or reprioritized based on user feedback, technical feasibility, and ATProto ecosystem evolution.
|
||||
|
||||
*Last audited: 2026-02-12*
|
||||
|
||||
480
docs/REMOVING_DISTRIBUTION.md
Normal file
480
docs/REMOVING_DISTRIBUTION.md
Normal file
@@ -0,0 +1,480 @@
|
||||
# Removing distribution/distribution
|
||||
|
||||
This document analyzes what it would take to remove the `github.com/distribution/distribution/v3` library and implement ATCR's own OCI Distribution Spec HTTP endpoints.
|
||||
|
||||
## Why Consider Removing It
|
||||
|
||||
1. **Impedance mismatch** -- Distribution assumes manifests and blobs live in the same storage backend. ATCR routes manifests to ATProto PDS and blobs to hold/S3. Every storage interface is overridden.
|
||||
2. **Context value workaround** -- `Repository()` receives only `context.Context` from distribution's interface, forcing auth/identity data through context keys into `RegistryContext`.
|
||||
3. **Per-request repository creation** -- `RoutingRepository` is recreated on every request because distribution's caching assumptions conflict with ATCR's OAuth session model.
|
||||
4. **Stale transitive dependencies** -- Distribution pulls in AWS SDK v1 (EOL) via its S3 storage driver, even though ATCR doesn't use that driver.
|
||||
5. **Unused features** -- GC, notifications, storage drivers, replication -- none are used. ATCR has its own GC, its own event dispatch (`processManifest` XRPC), and its own S3 integration.
|
||||
6. **Upstream maintenance pace** -- Slow to merge dependency updates and bug fixes.
|
||||
|
||||
## What Distribution Currently Provides
|
||||
|
||||
Only these pieces are actually used:
|
||||
|
||||
| What | Distribution Package | ATCR Usage |
|
||||
|------|---------------------|------------|
|
||||
| HTTP endpoint routing | `registry/handlers` | `handlers.NewApp()` creates the `/v2/` handler |
|
||||
| OCI error responses | `registry/api/errcode` | `ErrorCodeUnauthorized`, `ErrorCodeDenied`, `ErrorCodeUnsupported` |
|
||||
| Middleware registration | `registry/middleware/registry` | `Register("atproto-resolver", ...)` |
|
||||
| Repository interface | `distribution` (root) | `Repository`, `ManifestService`, `BlobStore`, `TagService` |
|
||||
| Reference parsing | `distribution/reference` | `reference.Named` for `identity/image` parsing |
|
||||
| Token auth | `registry/auth/token` | Blank import for registration |
|
||||
| In-memory driver | `registry/storage/driver/inmemory` | Blank import; placeholder since real storage is external |
|
||||
| Configuration types | `configuration` | `configuration.Configuration` struct |
|
||||
|
||||
Everything else (S3 driver, GC, notifications, replication, schema validation) is dead weight.
|
||||
|
||||
## Files That Import Distribution
|
||||
|
||||
All in `pkg/appview/` -- hold and scanner are unaffected.
|
||||
|
||||
**Core implementation (8 files):**
|
||||
- `storage/routing_repository.go` -- `distribution.Repository` wrapper
|
||||
- `storage/manifest_store.go` -- `distribution.ManifestService` impl
|
||||
- `storage/proxy_blob_store.go` -- `distribution.BlobStore` + `BlobWriter` impl
|
||||
- `storage/tag_store.go` -- `distribution.TagService` impl
|
||||
- `middleware/registry.go` -- `distribution.Namespace` + middleware registration
|
||||
- `config.go` -- Builds `configuration.Configuration`
|
||||
- `server.go` -- `handlers.NewApp()`, `errcode` for error responses
|
||||
- `cmd/appview/main.go` -- Blank imports for driver/auth registration
|
||||
|
||||
**Tests (6 files):**
|
||||
- `storage/routing_repository_test.go`
|
||||
- `storage/manifest_store_test.go`
|
||||
- `storage/proxy_blob_store_test.go`
|
||||
- `storage/tag_store_test.go`
|
||||
- `middleware/registry_test.go`
|
||||
|
||||
## OCI Distribution Spec Endpoints to Implement
|
||||
|
||||
The spec defines these HTTP endpoints. ATCR would need handlers for each.
|
||||
|
||||
### Version Check
|
||||
|
||||
```
|
||||
GET /v2/
|
||||
200 OK (confirms OCI compliance)
|
||||
401 Unauthorized (triggers auth flow)
|
||||
```
|
||||
|
||||
Docker clients hit this first. Must return 200 for authenticated requests. A 401 response with `WWW-Authenticate` header triggers the Docker auth handshake.
|
||||
|
||||
### Manifests
|
||||
|
||||
```
|
||||
GET /v2/<name>/manifests/<reference> -> 200 + manifest body
|
||||
HEAD /v2/<name>/manifests/<reference> -> 200 + headers only
|
||||
PUT /v2/<name>/manifests/<reference> -> 201 Created
|
||||
DELETE /v2/<name>/manifests/<reference> -> 202 Accepted
|
||||
```
|
||||
|
||||
`<reference>` is either a tag (`latest`) or digest (`sha256:abc...`).
|
||||
|
||||
**Required headers:**
|
||||
- Request `Accept`: manifest media types the client supports
|
||||
- Response `Content-Type`: actual manifest media type
|
||||
- Response `Docker-Content-Digest`: canonical digest of manifest
|
||||
|
||||
**Media types to support:**
|
||||
- `application/vnd.oci.image.manifest.v1+json`
|
||||
- `application/vnd.oci.image.index.v1+json`
|
||||
- `application/vnd.docker.distribution.manifest.v2+json`
|
||||
- `application/vnd.docker.distribution.manifest.list.v2+json`
|
||||
|
||||
### Blobs
|
||||
|
||||
```
|
||||
GET /v2/<name>/blobs/<digest> -> 200 + blob body (or 307 redirect)
|
||||
HEAD /v2/<name>/blobs/<digest> -> 200 + headers only
|
||||
DELETE /v2/<name>/blobs/<digest> -> 202 Accepted
|
||||
```
|
||||
|
||||
ATCR already redirects to presigned S3 URLs via `ServeBlob()` -- this would become a direct 307 redirect in the handler.
|
||||
|
||||
### Blob Uploads (Chunked/Resumable)
|
||||
|
||||
**Initiate:**
|
||||
```
|
||||
POST /v2/<name>/blobs/uploads/
|
||||
202 Accepted
|
||||
Location: /v2/<name>/blobs/uploads/<uuid>
|
||||
```
|
||||
|
||||
**Monolithic (single request):**
|
||||
```
|
||||
POST /v2/<name>/blobs/uploads/?digest=sha256:...
|
||||
Content-Type: application/octet-stream
|
||||
Body: <entire blob>
|
||||
201 Created
|
||||
```
|
||||
|
||||
**Chunked:**
|
||||
```
|
||||
PATCH /v2/<name>/blobs/uploads/<uuid>
|
||||
Content-Type: application/octet-stream
|
||||
Content-Range: <start>-<end>
|
||||
Body: <chunk data>
|
||||
202 Accepted
|
||||
Range: 0-<end>
|
||||
|
||||
(repeat PATCH for each chunk)
|
||||
|
||||
PUT /v2/<name>/blobs/uploads/<uuid>?digest=sha256:...
|
||||
201 Created
|
||||
Location: /v2/<name>/blobs/<digest>
|
||||
```
|
||||
|
||||
**Check progress:**
|
||||
```
|
||||
GET /v2/<name>/blobs/uploads/<uuid>
|
||||
204 No Content
|
||||
Range: 0-<bytes received>
|
||||
```
|
||||
|
||||
**Cancel:**
|
||||
```
|
||||
DELETE /v2/<name>/blobs/uploads/<uuid>
|
||||
204 No Content
|
||||
```
|
||||
|
||||
**Cross-repo mount:**
|
||||
```
|
||||
POST /v2/<name>/blobs/uploads/?mount=<digest>&from=<other-repo>
|
||||
201 Created (if blob exists in source repo)
|
||||
202 Accepted (fall back to regular upload)
|
||||
```
|
||||
|
||||
### Tags
|
||||
|
||||
```
|
||||
GET /v2/<name>/tags/list
|
||||
200 OK
|
||||
{
|
||||
"name": "<name>",
|
||||
"tags": ["latest", "v1.0"]
|
||||
}
|
||||
```
|
||||
|
||||
Supports pagination via `n` (count) and `last` (cursor) query params.
|
||||
|
||||
### Referrers (OCI v1.1)
|
||||
|
||||
```
|
||||
GET /v2/<name>/referrers/<digest>
|
||||
200 OK
|
||||
Content-Type: application/vnd.oci.image.index.v1+json
|
||||
Body: image index of referring manifests
|
||||
```
|
||||
|
||||
Supports `artifactType` query filter. Returns manifests whose `subject` field points to the given digest.
|
||||
|
||||
### Catalog (Optional)
|
||||
|
||||
```
|
||||
GET /v2/_catalog
|
||||
200 OK
|
||||
{ "repositories": ["alice/app", "bob/tool"] }
|
||||
```
|
||||
|
||||
Pagination via `n` and `last`. ATCR may choose not to implement this (many registries don't).
|
||||
|
||||
## Error Response Format
|
||||
|
||||
All 4xx/5xx responses must use the OCI error envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"errors": [
|
||||
{
|
||||
"code": "MANIFEST_UNKNOWN",
|
||||
"message": "manifest not found",
|
||||
"detail": { "tag": "latest" }
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Standard error codes:**
|
||||
|
||||
| Code | HTTP Status | Meaning |
|
||||
|------|-------------|---------|
|
||||
| `BLOB_UNKNOWN` | 404 | Blob not found |
|
||||
| `BLOB_UPLOAD_INVALID` | 400 | Bad digest or size mismatch |
|
||||
| `BLOB_UPLOAD_UNKNOWN` | 404 | Upload session expired/missing |
|
||||
| `DIGEST_INVALID` | 400 | Digest doesn't match content |
|
||||
| `MANIFEST_BLOB_UNKNOWN` | 404 | Manifest references missing blob |
|
||||
| `MANIFEST_INVALID` | 400 | Malformed manifest |
|
||||
| `MANIFEST_UNKNOWN` | 404 | Manifest not found |
|
||||
| `NAME_INVALID` | 400 | Bad repository name |
|
||||
| `NAME_UNKNOWN` | 404 | Repository doesn't exist |
|
||||
| `SIZE_INVALID` | 400 | Content-Length mismatch |
|
||||
| `UNAUTHORIZED` | 401 | Authentication required |
|
||||
| `DENIED` | 403 | Permission denied |
|
||||
| `UNSUPPORTED` | 405 | Operation not supported |
|
||||
| `TOOMANYREQUESTS` | 429 | Rate limited |
|
||||
|
||||
## What Exists Today vs What's New
|
||||
|
||||
For each handler, this breaks down what logic already exists in the storage layer (and just needs to be called) vs what new HTTP glue code must be written. Distribution's handler layer currently handles all the HTTP parsing, header validation, content negotiation, and response formatting -- all of that becomes our responsibility.
|
||||
|
||||
### Shared New Code
|
||||
|
||||
**Error helpers (~50 lines, new):**
|
||||
OCI error envelope formatting. Currently provided by `errcode.ErrorCodeUnauthorized` etc.
|
||||
|
||||
```go
|
||||
type RegistryError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Detail interface{} `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func WriteError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(struct {
|
||||
Errors []RegistryError `json:"errors"`
|
||||
}{Errors: []RegistryError{{Code: code, Message: message}}})
|
||||
}
|
||||
```
|
||||
|
||||
**Auth middleware (~80 lines, mostly exists):**
|
||||
`ExtractAuthMethod()` already exists in `middleware/registry.go`. Needs adaptation to work standalone (currently wraps distribution's app). Must also generate `WWW-Authenticate` header for 401 responses -- distribution's token auth handler currently does this via blank import of `registry/auth/token`.
|
||||
|
||||
**Identity resolution middleware (~250 lines, exists):**
|
||||
`NamespaceResolver.Repository()` in `middleware/registry.go` does identity resolution, hold discovery, service token acquisition, and ATProto client creation. This logic moves into an HTTP middleware but the code is the same -- resolves DID, finds hold, gets service token, builds `RegistryContext`. The validation cache (concurrent service token deduplication) comes along as-is.
|
||||
|
||||
**Router (~30 lines, new):**
|
||||
```go
|
||||
mux.HandleFunc("GET /v2/", handleVersionCheck)
|
||||
mux.HandleFunc("GET /v2/{name...}/manifests/{reference}", handleManifestGet)
|
||||
// ... etc
|
||||
```
|
||||
|
||||
### Handler-by-Handler Breakdown
|
||||
|
||||
---
|
||||
|
||||
**`handleVersionCheck`** -- `GET /v2/`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | None needed -- this is just a 200 OK response |
|
||||
| New code | ~10 lines. Return 200 with `Docker-Distribution-API-Version: registry/2.0` header. If unauthenticated, return 401 with `WWW-Authenticate` header to trigger Docker's auth flow |
|
||||
|
||||
---
|
||||
|
||||
**`handleManifestGet`** -- `GET /v2/<name>/manifests/<reference>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ManifestStore.Get()` fetches manifest from PDS (record lookup, optional blob download for new-format records). Returns media type + raw bytes. Also fires async pull notification to hold for stats. `TagStore.Get()` resolves tag → digest when reference is a tag |
|
||||
| New code (~40 lines) | Parse `<reference>` to determine tag vs digest. If tag, call `TagStore.Get()` first to resolve digest. Call `ManifestStore.Get()`. Set response headers: `Content-Type` (manifest media type), `Docker-Content-Digest` (canonical digest), `Content-Length`. Write body. Handle 404 (manifest not found → `MANIFEST_UNKNOWN` error) |
|
||||
| Subtle | Content negotiation: must check client's `Accept` header against the manifest's actual media type. Distribution handles this transparently. If client doesn't accept the type, return 404. In practice most clients accept everything, but `crane` and `skopeo` can be picky |
|
||||
|
||||
---
|
||||
|
||||
**`handleManifestHead`** -- `HEAD /v2/<name>/manifests/<reference>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ManifestStore.Exists()` checks PDS record existence. `ManifestStore.Get()` needed for full headers |
|
||||
| New code (~30 lines) | Same as GET but write headers only, no body. Needs `Content-Type`, `Docker-Content-Digest`, `Content-Length`. Could call `Exists()` for a fast path and `Get()` for full header population, or just call `Get()` and skip the body write |
|
||||
| Note | Some clients (Docker) use HEAD to check existence before pulling. Must return same headers as GET |
|
||||
|
||||
---
|
||||
|
||||
**`handleManifestPut`** -- `PUT /v2/<name>/manifests/<reference>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ManifestStore.Put()` does a LOT: calculates digest, uploads manifest bytes as blob to PDS, creates `ManifestRecord` with structured metadata, validates manifest list child references, extracts config labels, fetches README/icon, creates tag record, fires async notifications to hold, creates repo page records, handles successor migration |
|
||||
| New code (~50 lines) | Read request body. Extract `Content-Type` header as media type. Parse `<reference>` to determine if this is a tag push. Call `ManifestStore.Put()` with payload, media type, and optional tag. Set response headers: `Location` (`/v2/<name>/manifests/<digest>`), `Docker-Content-Digest`. Return 201 Created. Handle errors: `MANIFEST_INVALID` (bad JSON), `MANIFEST_BLOB_UNKNOWN` (missing child manifest in manifest list) |
|
||||
| Subtle | Distribution currently wraps the manifest in a `distribution.Manifest` interface (with `Payload()` and `References()` methods) before passing to `Put()`. Without distribution, we'd change `Put()` to accept raw `[]byte` + `mediaType` + optional tag directly -- simpler but requires updating the method signature and its internals |
|
||||
|
||||
---
|
||||
|
||||
**`handleManifestDelete`** -- `DELETE /v2/<name>/manifests/<reference>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ManifestStore.Delete()` calls `ATProtoClient.DeleteRecord()` |
|
||||
| New code (~15 lines) | Parse digest from `<reference>`. Call `ManifestStore.Delete()`. Return 202 Accepted. Handle 404 |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobGet`** -- `GET /v2/<name>/blobs/<digest>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobStore.ServeBlob()` checks read access, gets presigned URL from hold, and issues 307 redirect. This is already essentially an HTTP handler |
|
||||
| New code (~20 lines) | Parse digest from path. Call the presigned URL logic (read access check + hold XRPC call). Write 307 redirect with `Location` header pointing to presigned S3 URL |
|
||||
| Note | `ServeBlob()` currently takes `http.ResponseWriter` and `*http.Request` -- it's already doing the HTTP work. This handler is mostly just calling it. Could almost be used as-is |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobHead`** -- `HEAD /v2/<name>/blobs/<digest>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobStore.Stat()` checks read access, gets presigned HEAD URL, makes HEAD request to S3, returns size |
|
||||
| New code (~20 lines) | Parse digest. Call `Stat()`. Set `Content-Length`, `Docker-Content-Digest`, `Content-Type: application/octet-stream`. Return 200. Handle 404 (`BLOB_UNKNOWN`) |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobUploadInit`** -- `POST /v2/<name>/blobs/uploads/`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobStore.Create()` checks write access, generates upload ID, calls `startMultipartUpload()` XRPC to hold, creates `ProxyBlobWriter`, stores in `globalUploads` map |
|
||||
| New code (~50 lines) | Check for `?mount=<digest>&from=<repo>` query params (cross-repo mount). Check for `?digest=<digest>` (monolithic upload -- read body, write to store, complete in one shot). Otherwise, call `Create()` to start a new upload session. Return 202 Accepted with `Location: /v2/<name>/blobs/uploads/<uuid>` header, `Docker-Upload-UUID` header |
|
||||
| Subtle | Monolithic upload (single POST with digest and body) is a shortcut some clients use. Distribution handles this transparently. We'd need to handle it explicitly: read body, create writer, write, commit. Cross-repo mount is also handled here -- check if blob exists in source repo, skip upload if so |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobUploadChunk`** -- `PATCH /v2/<name>/blobs/uploads/<uuid>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobWriter.Write()` buffers data and auto-flushes 10MB chunks to S3 via presigned URLs. `flushPart()` handles the XRPC call to hold for part upload URLs and ETag tracking |
|
||||
| New code (~40 lines) | Look up writer from `globalUploads` by UUID. Parse `Content-Range` header (format: `<start>-<end>`). Read request body. Call `writer.Write(body)`. Return 202 Accepted with `Location` header (same upload URL), `Range: 0-<total bytes received>` header. Handle missing upload (`BLOB_UPLOAD_UNKNOWN`) |
|
||||
| Subtle | `Content-Range` validation: must verify start offset matches current writer position (no gaps, no out-of-order). Return 416 Range Not Satisfiable if misaligned. Distribution handles this; we'd need to track and validate |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobUploadComplete`** -- `PUT /v2/<name>/blobs/uploads/<uuid>?digest=sha256:...`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobWriter.Commit()` flushes remaining buffer, calls `completeMultipartUpload()` XRPC to hold, removes writer from `globalUploads` |
|
||||
| New code (~40 lines) | Look up writer from `globalUploads`. Parse `?digest=` query param. If request has body, write it to the writer (final chunk can be in the PUT). Call `writer.Commit()` with digest descriptor. Return 201 Created with `Location: /v2/<name>/blobs/<digest>`, `Docker-Content-Digest` header. Handle errors: `DIGEST_INVALID` (provided digest doesn't match), `BLOB_UPLOAD_UNKNOWN` (expired session) |
|
||||
| Subtle | Digest validation: distribution verifies the provided digest matches what was actually uploaded. Our writer doesn't currently track a running digest hash -- `Commit()` just passes the digest through to hold. Need to decide: trust the hold to validate, or add client-side validation. Currently hold does the final validation since it has all the parts |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobUploadStatus`** -- `GET /v2/<name>/blobs/uploads/<uuid>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobWriter.Size()` returns total bytes written |
|
||||
| New code (~15 lines) | Look up writer from `globalUploads`. Return 204 No Content with `Range: 0-<size - 1>`, `Docker-Upload-UUID`, `Location` headers. Handle missing upload |
|
||||
|
||||
---
|
||||
|
||||
**`handleBlobUploadCancel`** -- `DELETE /v2/<name>/blobs/uploads/<uuid>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `ProxyBlobWriter.Cancel()` calls `abortMultipartUpload()` XRPC to hold, removes from `globalUploads` |
|
||||
| New code (~15 lines) | Look up writer. Call `Cancel()`. Return 204 No Content. Handle missing upload |
|
||||
|
||||
---
|
||||
|
||||
**`handleTagsList`** -- `GET /v2/<name>/tags/list`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | `TagStore.All()` lists all tag records from PDS, filters by repository |
|
||||
| New code (~30 lines) | Call `TagStore.All()`. Parse `?n=` and `?last=` query params for pagination (slice the results). Return JSON: `{"name": "<name>", "tags": [...]}`. Set `Link` header for pagination if there are more results |
|
||||
| Note | Distribution handles pagination. We'd need to implement it ourselves -- sort tags, apply cursor, set Link header with next page URL |
|
||||
|
||||
---
|
||||
|
||||
**`handleReferrers`** -- `GET /v2/<name>/referrers/<digest>`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Existing logic | Not currently implemented in ATCR's storage layer. Distribution may return an empty index |
|
||||
| New code (~30 lines) | Query manifests that have a `subject` field pointing to the given digest. Return an OCI image index containing descriptors for each referrer. Support `?artifactType=` filter. If no referrers, return empty index |
|
||||
| Note | This is new functionality either way. ATCR would need to query PDS for manifests with matching subject digests. Could defer this (return empty index) and implement properly later |
|
||||
|
||||
---
|
||||
|
||||
### Interface Changes to Storage Layer
|
||||
|
||||
The existing stores would need their method signatures simplified. This is mostly mechanical -- removing distribution wrapper types:
|
||||
|
||||
**ManifestStore changes:**
|
||||
- `Get()`: returns `(distribution.Manifest, error)` → returns `(mediaType string, payload []byte, err error)`
|
||||
- `Put()`: accepts `distribution.Manifest` + `...distribution.ManifestServiceOption` → accepts `payload []byte, mediaType string, tag string`
|
||||
- `Exists()` and `Delete()`: signatures stay roughly the same (just `digest.Digest` in, error out)
|
||||
- Remove `rawManifest` struct (wrapper implementing `distribution.Manifest` interface)
|
||||
- Remove `distribution.WithTagOption` extraction logic in `Put()`
|
||||
|
||||
**ProxyBlobStore changes:**
|
||||
- `Stat()`: returns `distribution.Descriptor` → returns `(size int64, err error)`
|
||||
- `Get()`: stays the same (returns `[]byte`)
|
||||
- `ServeBlob()`: already takes `http.ResponseWriter`/`*http.Request` -- could become the handler itself
|
||||
- `Create()`: returns `distribution.BlobWriter` → returns `*ProxyBlobWriter` directly
|
||||
- `Resume()`: same change
|
||||
- Remove `distribution.BlobCreateOption` / `distribution.CreateOptions` parsing
|
||||
- `ProxyBlobWriter.Commit()`: accepts `distribution.Descriptor` → accepts `digest string, size int64`
|
||||
|
||||
**TagStore changes:**
|
||||
- `Get()`: returns `distribution.Descriptor` → returns `(digest string, err error)`
|
||||
- `Tag()`: accepts `distribution.Descriptor` → accepts `digest string`
|
||||
- `All()`, `Untag()`, `Lookup()`: minimal changes
|
||||
|
||||
**RoutingRepository:**
|
||||
- Removed entirely. Handlers call stores directly. The lazy initialization via `sync.Once` goes away since there's no interface requiring a `Repository` object.
|
||||
|
||||
**Estimated interface change work:** ~150 lines changed across storage files + ~150 lines changed across test files.
|
||||
|
||||
## What Stays
|
||||
|
||||
These dependencies are used directly and stay regardless:
|
||||
|
||||
- `github.com/opencontainers/go-digest` -- Digest parsing/validation (standard, lightweight)
|
||||
- `github.com/opencontainers/image-spec` -- OCI manifest/index structs (optional but useful for validation)
|
||||
- `github.com/distribution/reference` -- Could stay (lightweight, no heavy transitive deps) or replace with string splitting since ATCR's name format is always `<identity>/<image>`
|
||||
|
||||
## Revised Effort Estimate
|
||||
|
||||
| Component | New Lines | Changed Lines | Notes |
|
||||
|-----------|-----------|---------------|-------|
|
||||
| Router + version check | ~40 | 0 | Trivial |
|
||||
| Error helpers | ~50 | 0 | OCI error envelope, error code constants |
|
||||
| Auth middleware adaptation | ~30 | ~50 | `WWW-Authenticate` header generation is new; `ExtractAuthMethod` moves |
|
||||
| Identity resolution middleware | ~20 | ~30 | `NamespaceResolver.Repository()` logic moves to HTTP middleware; code is the same |
|
||||
| Manifest handlers (GET/HEAD/PUT/DELETE) | ~135 | 0 | Content negotiation, header writing, tag vs digest parsing |
|
||||
| Blob handlers (GET/HEAD/DELETE) | ~55 | 0 | Presigned URL redirect, stat, delete stub |
|
||||
| Blob upload handlers (POST/PATCH/PUT/GET/DELETE) | ~160 | 0 | Chunked upload protocol, Content-Range validation, monolithic upload, cross-repo mount |
|
||||
| Tags list handler | ~30 | 0 | Pagination logic |
|
||||
| Referrers handler | ~30 | 0 | Could defer with empty index |
|
||||
| Storage interface changes | 0 | ~150 | Remove distribution types from method signatures |
|
||||
| Test updates | 0 | ~150 | Update mocks and assertions for new signatures |
|
||||
| Config cleanup | 0 | ~80 | Remove `buildDistributionConfig()`, blank imports |
|
||||
| **Total** | **~550 new** | **~460 changed** | **~1010 lines total** |
|
||||
|
||||
This is not a trivial migration. The ~550 new lines are genuine new HTTP handler code that doesn't exist today -- distribution's handler layer provides all of it currently. The changed lines are mostly mechanical (removing distribution type wrappers) but still need care and test updates.
|
||||
|
||||
## Risk Assessment
|
||||
|
||||
**Low risk:**
|
||||
- Storage logic is unchanged -- same PDS calls, same hold XRPC calls, same presigned URLs
|
||||
- Auth flow is unchanged -- same JWT validation, same OAuth refresh
|
||||
- Tests can be adapted incrementally
|
||||
|
||||
**Medium risk:**
|
||||
- Subtle OCI spec compliance gaps (edge cases in content negotiation, digest validation, chunked upload semantics)
|
||||
- Docker client compatibility -- different clients (Docker, Podman, crane, skopeo) may exercise different code paths
|
||||
|
||||
**Mitigation:**
|
||||
- Use [OCI conformance tests](https://github.com/opencontainers/distribution-spec/tree/main/conformance) to validate
|
||||
- Test against Docker, Podman, crane, and skopeo before shipping
|
||||
- Can be done incrementally: build new router, test alongside distribution handler, swap when ready
|
||||
|
||||
## Dependencies Removed
|
||||
|
||||
Removing distribution eliminates ~30-40 transitive packages, notably:
|
||||
- `github.com/aws/aws-sdk-go` (v1, EOL)
|
||||
- Azure cloud SDK packages
|
||||
- Google Cloud Storage packages
|
||||
- Distribution-specific logging/metrics
|
||||
- Unused storage driver registrations
|
||||
|
||||
Most other transitive deps (gRPC, protobuf, OpenTelemetry, logrus) are also pulled by `bluesky-social/indigo` and would remain.
|
||||
2
go.mod
2
go.mod
@@ -3,6 +3,7 @@ module atcr.io
|
||||
go 1.25.7
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
@@ -52,7 +53,6 @@ require (
|
||||
github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect
|
||||
github.com/ajg/form v1.6.1 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
|
||||
77
lexicons/io/atcr/hold/scan.json
Normal file
77
lexicons/io/atcr/hold/scan.json
Normal file
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"lexicon": 1,
|
||||
"id": "io.atcr.hold.scan",
|
||||
"defs": {
|
||||
"main": {
|
||||
"type": "record",
|
||||
"key": "any",
|
||||
"description": "Vulnerability scan results for a container manifest. Stored in the hold's embedded PDS. Record key is deterministic: the manifest digest hex without the 'sha256:' prefix, so re-scans upsert the existing record.",
|
||||
"record": {
|
||||
"type": "object",
|
||||
"required": ["manifest", "repository", "userDid", "critical", "high", "medium", "low", "total", "scannerVersion", "scannedAt"],
|
||||
"properties": {
|
||||
"manifest": {
|
||||
"type": "string",
|
||||
"format": "at-uri",
|
||||
"description": "AT-URI of the scanned manifest (e.g., at://did:plc:xyz/io.atcr.manifest/abc123...)"
|
||||
},
|
||||
"repository": {
|
||||
"type": "string",
|
||||
"description": "Repository name (e.g., myapp)",
|
||||
"maxLength": 256
|
||||
},
|
||||
"userDid": {
|
||||
"type": "string",
|
||||
"format": "did",
|
||||
"description": "DID of the image owner"
|
||||
},
|
||||
"sbomBlob": {
|
||||
"type": "blob",
|
||||
"description": "SBOM blob (SPDX JSON format) uploaded to the hold's blob storage",
|
||||
"accept": ["application/spdx+json"]
|
||||
},
|
||||
"vulnReportBlob": {
|
||||
"type": "blob",
|
||||
"description": "Grype vulnerability report blob (JSON) with full CVE details",
|
||||
"accept": ["application/vnd.atcr.vulnerabilities+json"]
|
||||
},
|
||||
"critical": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Count of critical severity vulnerabilities"
|
||||
},
|
||||
"high": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Count of high severity vulnerabilities"
|
||||
},
|
||||
"medium": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Count of medium severity vulnerabilities"
|
||||
},
|
||||
"low": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Count of low severity vulnerabilities"
|
||||
},
|
||||
"total": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Total vulnerability count"
|
||||
},
|
||||
"scannerVersion": {
|
||||
"type": "string",
|
||||
"description": "Version of the scanner that produced this result (e.g., atcr-scanner-v1.0.0)",
|
||||
"maxLength": 64
|
||||
},
|
||||
"scannedAt": {
|
||||
"type": "string",
|
||||
"format": "datetime",
|
||||
"description": "RFC3339 timestamp of when the scan completed"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
description: Rebuild hold_captain_records to match schema.sql (provider→successor rename was missed when migration 0010 was recorded but not executed on a fresh DB)
|
||||
query: |
|
||||
-- Recreate table to match schema.sql exactly
|
||||
CREATE TABLE IF NOT EXISTS hold_captain_records_new (
|
||||
hold_did TEXT PRIMARY KEY,
|
||||
owner_did TEXT NOT NULL,
|
||||
public BOOLEAN NOT NULL,
|
||||
allow_all_crew BOOLEAN NOT NULL,
|
||||
deployed_at TEXT,
|
||||
region TEXT,
|
||||
successor TEXT,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Copy data (only guaranteed-common columns; successor will be NULL)
|
||||
INSERT OR IGNORE INTO hold_captain_records_new (hold_did, owner_did, public, allow_all_crew, deployed_at, region, updated_at)
|
||||
SELECT hold_did, owner_did, public, allow_all_crew, deployed_at, region, updated_at
|
||||
FROM hold_captain_records;
|
||||
|
||||
-- Swap tables
|
||||
DROP TABLE hold_captain_records;
|
||||
ALTER TABLE hold_captain_records_new RENAME TO hold_captain_records;
|
||||
|
||||
-- Recreate index
|
||||
CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at);
|
||||
125
pkg/appview/handlers/scan_result.go
Normal file
125
pkg/appview/handlers/scan_result.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
)
|
||||
|
||||
// ScanResultHandler handles HTMX requests for vulnerability scan badges.
|
||||
// Returns an HTML fragment (vuln-badge partial) that replaces the placeholder span.
|
||||
type ScanResultHandler struct {
|
||||
BaseUIHandler
|
||||
}
|
||||
|
||||
// vulnBadgeData is the template data for the vuln-badge partial.
|
||||
type vulnBadgeData struct {
|
||||
Critical int64
|
||||
High int64
|
||||
Medium int64
|
||||
Low int64
|
||||
Total int64
|
||||
ScannedAt string
|
||||
Found bool // true if scan record exists
|
||||
Error bool // true if hold unreachable or error
|
||||
Digest string // for the detail modal link
|
||||
HoldEndpoint string // for the detail modal link
|
||||
}
|
||||
|
||||
func (h *ScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
digest := r.URL.Query().Get("digest")
|
||||
holdEndpoint := r.URL.Query().Get("holdEndpoint")
|
||||
|
||||
if digest == "" || holdEndpoint == "" {
|
||||
// Missing params — render nothing
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
return
|
||||
}
|
||||
|
||||
// Derive hold DID from endpoint URL
|
||||
holdDID := atproto.ResolveHoldDIDFromURL(holdEndpoint)
|
||||
if holdDID == "" {
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
// Compute rkey from digest (strip sha256: prefix)
|
||||
rkey := strings.TrimPrefix(digest, "sha256:")
|
||||
|
||||
// Fetch scan record from hold's PDS
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
|
||||
holdEndpoint,
|
||||
url.QueryEscape(holdDID),
|
||||
url.QueryEscape(atproto.ScanCollection),
|
||||
url.QueryEscape(rkey),
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil)
|
||||
if err != nil {
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
// Hold unreachable or timeout — render nothing
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
// No scan record — scanning disabled or not yet scanned. Render nothing.
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
// Parse the getRecord response envelope
|
||||
var envelope struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil {
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
var scanRecord atproto.ScanRecord
|
||||
if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil {
|
||||
h.renderBadge(w, vulnBadgeData{Error: true})
|
||||
return
|
||||
}
|
||||
|
||||
h.renderBadge(w, vulnBadgeData{
|
||||
Critical: scanRecord.Critical,
|
||||
High: scanRecord.High,
|
||||
Medium: scanRecord.Medium,
|
||||
Low: scanRecord.Low,
|
||||
Total: scanRecord.Total,
|
||||
ScannedAt: scanRecord.ScannedAt,
|
||||
Found: true,
|
||||
Digest: digest,
|
||||
HoldEndpoint: holdEndpoint,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ScanResultHandler) renderBadge(w http.ResponseWriter, data vulnBadgeData) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
if err := h.Templates.ExecuteTemplate(w, "vuln-badge", data); err != nil {
|
||||
slog.Warn("Failed to render vuln badge", "error", err)
|
||||
}
|
||||
}
|
||||
255
pkg/appview/handlers/scan_result_test.go
Normal file
255
pkg/appview/handlers/scan_result_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"atcr.io/pkg/appview"
|
||||
"atcr.io/pkg/appview/handlers"
|
||||
)
|
||||
|
||||
// mockScanRecord returns a getRecord JSON envelope wrapping a scan record
|
||||
func mockScanRecord(critical, high, medium, low, total int64) string {
|
||||
record := map[string]any{
|
||||
"$type": "io.atcr.hold.scan",
|
||||
"manifest": "at://did:plc:test/io.atcr.manifest/abc123",
|
||||
"repository": "myapp",
|
||||
"userDid": "did:plc:test",
|
||||
"critical": critical,
|
||||
"high": high,
|
||||
"medium": medium,
|
||||
"low": low,
|
||||
"total": total,
|
||||
"scannerVersion": "atcr-scanner-v1.0.0",
|
||||
"scannedAt": "2025-01-15T10:30:00Z",
|
||||
}
|
||||
envelope := map[string]any{
|
||||
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
|
||||
"cid": "bafyreiabc123",
|
||||
"value": record,
|
||||
}
|
||||
b, _ := json.Marshal(envelope)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func setupScanResultHandler(t *testing.T, holdURL string) *handlers.ScanResultHandler {
|
||||
t.Helper()
|
||||
templates, err := appview.Templates(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load templates: %v", err)
|
||||
}
|
||||
return &handlers.ScanResultHandler{
|
||||
BaseUIHandler: handlers.BaseUIHandler{
|
||||
Templates: templates,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_WithVulnerabilities(t *testing.T) {
|
||||
// Mock hold that returns a scan record with vulnerabilities
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecord(2, 5, 10, 3, 20)))
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Errorf("Expected status %d, got %d", http.StatusOK, rr.Code)
|
||||
}
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
// Should contain severity badges
|
||||
if !strings.Contains(body, "badge-error") {
|
||||
t.Error("Expected body to contain badge-error for critical vulnerabilities")
|
||||
}
|
||||
if !strings.Contains(body, "C:2") {
|
||||
t.Error("Expected body to contain 'C:2' for critical count")
|
||||
}
|
||||
if !strings.Contains(body, "badge-warning") {
|
||||
t.Error("Expected body to contain badge-warning for high vulnerabilities")
|
||||
}
|
||||
if !strings.Contains(body, "H:5") {
|
||||
t.Error("Expected body to contain 'H:5' for high count")
|
||||
}
|
||||
if !strings.Contains(body, "M:10") {
|
||||
t.Error("Expected body to contain 'M:10' for medium count")
|
||||
}
|
||||
if !strings.Contains(body, "L:3") {
|
||||
t.Error("Expected body to contain 'L:3' for low count")
|
||||
}
|
||||
// Should be clickable (has openVulnDetails)
|
||||
if !strings.Contains(body, "openVulnDetails") {
|
||||
t.Error("Expected body to contain openVulnDetails click handler")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_Clean(t *testing.T) {
|
||||
// Mock hold that returns a scan record with zero vulnerabilities
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecord(0, 0, 0, 0, 0)))
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
if !strings.Contains(body, "Clean") {
|
||||
t.Error("Expected body to contain 'Clean' for zero-vulnerability scan")
|
||||
}
|
||||
if !strings.Contains(body, "badge-success") {
|
||||
t.Error("Expected body to contain badge-success for clean scan")
|
||||
}
|
||||
// Should NOT be clickable
|
||||
if strings.Contains(body, "openVulnDetails") {
|
||||
t.Error("Clean badge should not have openVulnDetails click handler")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_NotFound(t *testing.T) {
|
||||
// Mock hold that returns 404 (no scan record — scanning disabled or not yet scanned)
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "record not found", http.StatusNotFound)
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
// 404 = no scan record. Should render NOTHING — not "Scan pending".
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for 404, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_HoldError(t *testing.T) {
|
||||
// Mock hold that returns 500
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for hold error, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_HoldUnreachable(t *testing.T) {
|
||||
// Use a server that's already closed (unreachable)
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for unreachable hold, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_MissingParams(t *testing.T) {
|
||||
handler := setupScanResultHandler(t, "")
|
||||
|
||||
// No params at all
|
||||
req := httptest.NewRequest("GET", "/api/scan-result", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for missing params, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_MissingDigest(t *testing.T) {
|
||||
handler := setupScanResultHandler(t, "")
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?holdEndpoint=https://hold.example.com", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for missing digest, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_MissingHoldEndpoint(t *testing.T) {
|
||||
handler := setupScanResultHandler(t, "")
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := strings.TrimSpace(rr.Body.String())
|
||||
|
||||
if body != "" {
|
||||
t.Errorf("Expected empty body for missing holdEndpoint, got: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanResult_OnlyCriticalShown(t *testing.T) {
|
||||
// Only critical vulns, no high/medium/low
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecord(3, 0, 0, 0, 3)))
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupScanResultHandler(t, hold.URL)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scan-result?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
if !strings.Contains(body, "C:3") {
|
||||
t.Error("Expected body to contain 'C:3'")
|
||||
}
|
||||
// Zero-count badges should NOT appear
|
||||
if strings.Contains(body, "H:0") {
|
||||
t.Error("Should not contain 'H:0' for zero high count")
|
||||
}
|
||||
if strings.Contains(body, "M:0") {
|
||||
t.Error("Should not contain 'M:0' for zero medium count")
|
||||
}
|
||||
if strings.Contains(body, "L:0") {
|
||||
t.Error("Should not contain 'L:0' for zero low count")
|
||||
}
|
||||
}
|
||||
244
pkg/appview/handlers/vuln_details.go
Normal file
244
pkg/appview/handlers/vuln_details.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
)
|
||||
|
||||
// VulnDetailsHandler handles requests for the vulnerability detail modal content.
|
||||
// Returns an HTML fragment (vuln-details partial) for insertion into the modal body.
|
||||
type VulnDetailsHandler struct {
|
||||
BaseUIHandler
|
||||
}
|
||||
|
||||
// grypeReport is the minimal Grype JSON structure we need.
|
||||
type grypeReport struct {
|
||||
Matches []grypeMatch `json:"matches"`
|
||||
}
|
||||
|
||||
type grypeMatch struct {
|
||||
Vulnerability grypeVuln `json:"vulnerability"`
|
||||
Artifact grypeArtifact `json:"artifact"`
|
||||
}
|
||||
|
||||
type grypeVuln struct {
|
||||
ID string `json:"id"`
|
||||
Severity string `json:"severity"`
|
||||
Fix grypeFix `json:"fix"`
|
||||
}
|
||||
|
||||
type grypeFix struct {
|
||||
Versions []string `json:"versions"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
type grypeArtifact struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// vulnDetailsData is the template data for the vuln-details partial.
|
||||
type vulnDetailsData struct {
|
||||
Matches []vulnMatch
|
||||
Summary vulnSummary
|
||||
Error string // non-empty if something went wrong
|
||||
ScannedAt string
|
||||
}
|
||||
|
||||
type vulnMatch struct {
|
||||
CVEURL string
|
||||
CVEID string
|
||||
Severity string // Critical, High, Medium, Low, Negligible, Unknown
|
||||
Package string
|
||||
Version string
|
||||
FixedIn string
|
||||
Type string // deb, npm, gem, etc.
|
||||
}
|
||||
|
||||
type vulnSummary struct {
|
||||
Critical int64
|
||||
High int64
|
||||
Medium int64
|
||||
Low int64
|
||||
Total int64
|
||||
}
|
||||
|
||||
// severityOrder maps severity strings to sort order (lower = more severe).
|
||||
var severityOrder = map[string]int{
|
||||
"Critical": 0,
|
||||
"High": 1,
|
||||
"Medium": 2,
|
||||
"Low": 3,
|
||||
"Negligible": 4,
|
||||
"Unknown": 5,
|
||||
}
|
||||
|
||||
func (h *VulnDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
digest := r.URL.Query().Get("digest")
|
||||
holdEndpoint := r.URL.Query().Get("holdEndpoint")
|
||||
|
||||
if digest == "" || holdEndpoint == "" {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Missing required parameters"})
|
||||
return
|
||||
}
|
||||
|
||||
holdDID := atproto.ResolveHoldDIDFromURL(holdEndpoint)
|
||||
if holdDID == "" {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Could not resolve hold identity"})
|
||||
return
|
||||
}
|
||||
|
||||
rkey := strings.TrimPrefix(digest, "sha256:")
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Step 1: Fetch the scan record to get the VulnReportBlob CID
|
||||
scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
|
||||
holdEndpoint,
|
||||
url.QueryEscape(holdDID),
|
||||
url.QueryEscape(atproto.ScanCollection),
|
||||
url.QueryEscape(rkey),
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil)
|
||||
if err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Failed to build request"})
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Hold service unreachable"})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "No scan record found"})
|
||||
return
|
||||
}
|
||||
|
||||
var envelope struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Failed to parse scan record"})
|
||||
return
|
||||
}
|
||||
|
||||
var scanRecord atproto.ScanRecord
|
||||
if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Error: "Failed to parse scan record"})
|
||||
return
|
||||
}
|
||||
|
||||
summary := vulnSummary{
|
||||
Critical: scanRecord.Critical,
|
||||
High: scanRecord.High,
|
||||
Medium: scanRecord.Medium,
|
||||
Low: scanRecord.Low,
|
||||
Total: scanRecord.Total,
|
||||
}
|
||||
|
||||
// Step 2: Fetch the vulnerability report blob
|
||||
if scanRecord.VulnReportBlob == nil || scanRecord.VulnReportBlob.Ref.String() == "" {
|
||||
h.renderDetails(w, vulnDetailsData{
|
||||
Summary: summary,
|
||||
ScannedAt: scanRecord.ScannedAt,
|
||||
Error: "No detailed vulnerability report available. Only summary counts were recorded.",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
blobCID := scanRecord.VulnReportBlob.Ref.String()
|
||||
blobURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s",
|
||||
holdEndpoint,
|
||||
url.QueryEscape(holdDID),
|
||||
url.QueryEscape(blobCID),
|
||||
)
|
||||
|
||||
blobReq, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil)
|
||||
if err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to build blob request"})
|
||||
return
|
||||
}
|
||||
|
||||
blobResp, err := http.DefaultClient.Do(blobReq)
|
||||
if err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to fetch vulnerability report"})
|
||||
return
|
||||
}
|
||||
defer blobResp.Body.Close()
|
||||
|
||||
if blobResp.StatusCode != http.StatusOK {
|
||||
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Vulnerability report not accessible"})
|
||||
return
|
||||
}
|
||||
|
||||
// Step 3: Parse the Grype JSON
|
||||
var report grypeReport
|
||||
if err := json.NewDecoder(blobResp.Body).Decode(&report); err != nil {
|
||||
h.renderDetails(w, vulnDetailsData{Summary: summary, ScannedAt: scanRecord.ScannedAt, Error: "Failed to parse vulnerability report"})
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to template data
|
||||
matches := make([]vulnMatch, 0, len(report.Matches))
|
||||
for _, m := range report.Matches {
|
||||
fixedIn := ""
|
||||
if len(m.Vulnerability.Fix.Versions) > 0 {
|
||||
fixedIn = strings.Join(m.Vulnerability.Fix.Versions, ", ")
|
||||
}
|
||||
|
||||
cveURL := ""
|
||||
if strings.HasPrefix(m.Vulnerability.ID, "CVE-") {
|
||||
cveURL = "https://nvd.nist.gov/vuln/detail/" + m.Vulnerability.ID
|
||||
} else if strings.HasPrefix(m.Vulnerability.ID, "GHSA-") {
|
||||
cveURL = "https://github.com/advisories/" + m.Vulnerability.ID
|
||||
}
|
||||
|
||||
matches = append(matches, vulnMatch{
|
||||
CVEID: m.Vulnerability.ID,
|
||||
CVEURL: cveURL,
|
||||
Severity: m.Vulnerability.Severity,
|
||||
Package: m.Artifact.Name,
|
||||
Version: m.Artifact.Version,
|
||||
FixedIn: fixedIn,
|
||||
Type: m.Artifact.Type,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort by severity (critical first)
|
||||
sort.Slice(matches, func(i, j int) bool {
|
||||
oi := severityOrder[matches[i].Severity]
|
||||
oj := severityOrder[matches[j].Severity]
|
||||
if oi != oj {
|
||||
return oi < oj
|
||||
}
|
||||
return matches[i].CVEID < matches[j].CVEID
|
||||
})
|
||||
|
||||
h.renderDetails(w, vulnDetailsData{
|
||||
Matches: matches,
|
||||
Summary: summary,
|
||||
ScannedAt: scanRecord.ScannedAt,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *VulnDetailsHandler) renderDetails(w http.ResponseWriter, data vulnDetailsData) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
if err := h.Templates.ExecuteTemplate(w, "vuln-details", data); err != nil {
|
||||
slog.Warn("Failed to render vuln details", "error", err)
|
||||
}
|
||||
}
|
||||
336
pkg/appview/handlers/vuln_details_test.go
Normal file
336
pkg/appview/handlers/vuln_details_test.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"atcr.io/pkg/appview"
|
||||
"atcr.io/pkg/appview/handlers"
|
||||
lexutil "github.com/bluesky-social/indigo/lex/util"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
// mockGrypeReport returns a minimal Grype JSON report
|
||||
func mockGrypeReport() string {
|
||||
report := map[string]any{
|
||||
"matches": []map[string]any{
|
||||
{
|
||||
"vulnerability": map[string]any{
|
||||
"id": "CVE-2024-1234",
|
||||
"severity": "Critical",
|
||||
"fix": map[string]any{"versions": []string{"1.2.4"}, "state": "fixed"},
|
||||
},
|
||||
"artifact": map[string]any{
|
||||
"name": "libssl",
|
||||
"version": "1.1.1",
|
||||
"type": "deb",
|
||||
},
|
||||
},
|
||||
{
|
||||
"vulnerability": map[string]any{
|
||||
"id": "CVE-2024-5678",
|
||||
"severity": "Low",
|
||||
"fix": map[string]any{"versions": []string{}, "state": "not-fixed"},
|
||||
},
|
||||
"artifact": map[string]any{
|
||||
"name": "zlib",
|
||||
"version": "1.2.11",
|
||||
"type": "deb",
|
||||
},
|
||||
},
|
||||
{
|
||||
"vulnerability": map[string]any{
|
||||
"id": "GHSA-abcd-efgh-ijkl",
|
||||
"severity": "High",
|
||||
"fix": map[string]any{"versions": []string{"2.0.0"}, "state": "fixed"},
|
||||
},
|
||||
"artifact": map[string]any{
|
||||
"name": "express",
|
||||
"version": "4.17.1",
|
||||
"type": "npm",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
b, _ := json.Marshal(report)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// testCID creates a valid CIDv1 from arbitrary data (for test fixtures)
|
||||
func testCID(data string) cid.Cid {
|
||||
hash := sha256.Sum256([]byte(data))
|
||||
mh, _ := multihash.Encode(hash[:], multihash.SHA2_256)
|
||||
return cid.NewCidV1(0x55, mh) // raw codec
|
||||
}
|
||||
|
||||
// mockScanRecordWithBlob returns a getRecord envelope with a VulnReportBlob reference.
|
||||
// Uses a real CID so LexBlob JSON unmarshaling works correctly.
|
||||
func mockScanRecordWithBlob(critical, high, medium, low, total int64) string {
|
||||
blobCID := testCID("test-vuln-report")
|
||||
|
||||
// Build the record using the actual LexBlob type for correct JSON format
|
||||
blob := &lexutil.LexBlob{
|
||||
Ref: lexutil.LexLink(blobCID),
|
||||
MimeType: "application/vnd.atcr.vulnerabilities+json",
|
||||
Size: 12345,
|
||||
}
|
||||
|
||||
// Marshal blob separately to get the canonical JSON format
|
||||
blobJSON, _ := json.Marshal(blob)
|
||||
|
||||
// Build the full record as a map, inserting the pre-marshaled blob
|
||||
record := map[string]json.RawMessage{
|
||||
"$type": jsonStr("io.atcr.hold.scan"),
|
||||
"manifest": jsonStr("at://did:plc:test/io.atcr.manifest/abc123"),
|
||||
"repository": jsonStr("myapp"),
|
||||
"userDid": jsonStr("did:plc:test"),
|
||||
"vulnReportBlob": blobJSON,
|
||||
"critical": jsonInt(critical),
|
||||
"high": jsonInt(high),
|
||||
"medium": jsonInt(medium),
|
||||
"low": jsonInt(low),
|
||||
"total": jsonInt(total),
|
||||
"scannerVersion": jsonStr("atcr-scanner-v1.0.0"),
|
||||
"scannedAt": jsonStr("2025-01-15T10:30:00Z"),
|
||||
}
|
||||
recordJSON, _ := json.Marshal(record)
|
||||
|
||||
envelope := map[string]any{
|
||||
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
|
||||
"cid": "bafyreiabc123",
|
||||
"value": json.RawMessage(recordJSON),
|
||||
}
|
||||
b, _ := json.Marshal(envelope)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func jsonStr(s string) json.RawMessage {
|
||||
b, _ := json.Marshal(s)
|
||||
return b
|
||||
}
|
||||
|
||||
func jsonInt(n int64) json.RawMessage {
|
||||
b, _ := json.Marshal(n)
|
||||
return b
|
||||
}
|
||||
|
||||
// mockScanRecordWithoutBlob returns a getRecord envelope without VulnReportBlob
|
||||
func mockScanRecordWithoutBlob(critical, high, medium, low, total int64) string {
|
||||
record := map[string]any{
|
||||
"$type": "io.atcr.hold.scan",
|
||||
"manifest": "at://did:plc:test/io.atcr.manifest/abc123",
|
||||
"repository": "myapp",
|
||||
"userDid": "did:plc:test",
|
||||
"critical": critical,
|
||||
"high": high,
|
||||
"medium": medium,
|
||||
"low": low,
|
||||
"total": total,
|
||||
"scannerVersion": "atcr-scanner-v1.0.0",
|
||||
"scannedAt": "2025-01-15T10:30:00Z",
|
||||
}
|
||||
envelope := map[string]any{
|
||||
"uri": "at://did:web:hold.example.com/io.atcr.hold.scan/abc123",
|
||||
"cid": "bafyreiabc123",
|
||||
"value": record,
|
||||
}
|
||||
b, _ := json.Marshal(envelope)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func setupVulnDetailsHandler(t *testing.T) *handlers.VulnDetailsHandler {
|
||||
t.Helper()
|
||||
templates, err := appview.Templates(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load templates: %v", err)
|
||||
}
|
||||
return &handlers.VulnDetailsHandler{
|
||||
BaseUIHandler: handlers.BaseUIHandler{
|
||||
Templates: templates,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnDetails_FullReport(t *testing.T) {
|
||||
grypeJSON := mockGrypeReport()
|
||||
|
||||
// Mock hold that serves both getRecord and getBlob
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
if strings.Contains(path, "getRecord") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecordWithBlob(1, 1, 0, 1, 3)))
|
||||
} else if strings.Contains(path, "getBlob") {
|
||||
// Serve the Grype JSON directly (no redirect in tests)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(grypeJSON))
|
||||
} else {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupVulnDetailsHandler(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Errorf("Expected status %d, got %d", http.StatusOK, rr.Code)
|
||||
}
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
// Should contain CVE IDs
|
||||
if !strings.Contains(body, "CVE-2024-1234") {
|
||||
t.Error("Expected body to contain CVE-2024-1234")
|
||||
}
|
||||
if !strings.Contains(body, "GHSA-abcd-efgh-ijkl") {
|
||||
t.Error("Expected body to contain GHSA-abcd-efgh-ijkl")
|
||||
}
|
||||
|
||||
// Should contain package names
|
||||
if !strings.Contains(body, "libssl") {
|
||||
t.Error("Expected body to contain package name 'libssl'")
|
||||
}
|
||||
if !strings.Contains(body, "express") {
|
||||
t.Error("Expected body to contain package name 'express'")
|
||||
}
|
||||
|
||||
// Should contain NVD link for CVE
|
||||
if !strings.Contains(body, "nvd.nist.gov") {
|
||||
t.Error("Expected body to contain NVD link")
|
||||
}
|
||||
// Should contain GitHub advisory link for GHSA
|
||||
if !strings.Contains(body, "github.com/advisories") {
|
||||
t.Error("Expected body to contain GitHub advisory link")
|
||||
}
|
||||
|
||||
// Should contain fix version
|
||||
if !strings.Contains(body, "1.2.4") {
|
||||
t.Error("Expected body to contain fix version '1.2.4'")
|
||||
}
|
||||
|
||||
// Should contain "No fix" for unfixed vuln
|
||||
if !strings.Contains(body, "No fix") {
|
||||
t.Error("Expected body to contain 'No fix' for unfixed vulnerability")
|
||||
}
|
||||
|
||||
// Should contain a table
|
||||
if !strings.Contains(body, "<table") {
|
||||
t.Error("Expected body to contain a table element")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnDetails_NoVulnReportBlob(t *testing.T) {
|
||||
// Mock hold returns scan record WITHOUT VulnReportBlob
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecordWithoutBlob(2, 5, 10, 3, 20)))
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupVulnDetailsHandler(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
// Should show summary counts
|
||||
if !strings.Contains(body, "2 Critical") {
|
||||
t.Error("Expected body to contain '2 Critical' summary")
|
||||
}
|
||||
|
||||
// Should indicate no detailed report
|
||||
if !strings.Contains(body, "No detailed vulnerability report") {
|
||||
t.Error("Expected body to indicate no detailed report available")
|
||||
}
|
||||
|
||||
// Should NOT contain a table
|
||||
if strings.Contains(body, "<table") {
|
||||
t.Error("Should not render a table when no VulnReportBlob")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnDetails_NotFound(t *testing.T) {
|
||||
// Mock hold returns 404
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupVulnDetailsHandler(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
// Should show an error message (modal still needs content)
|
||||
if !strings.Contains(body, "No scan record found") {
|
||||
t.Error("Expected body to contain error message for 404")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnDetails_SortsBySeverity(t *testing.T) {
|
||||
grypeJSON := mockGrypeReport() // Has Critical, Low, High in that order
|
||||
|
||||
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
if strings.Contains(path, "getRecord") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(mockScanRecordWithBlob(1, 1, 0, 1, 3)))
|
||||
} else if strings.Contains(path, "getBlob") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(grypeJSON))
|
||||
}
|
||||
}))
|
||||
defer hold.Close()
|
||||
|
||||
handler := setupVulnDetailsHandler(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/vuln-details?digest=sha256:abc123&holdEndpoint="+hold.URL, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
// Critical should appear before High, which should appear before Low
|
||||
critIdx := strings.Index(body, "CVE-2024-1234") // Critical
|
||||
highIdx := strings.Index(body, "GHSA-abcd-efgh") // High
|
||||
lowIdx := strings.Index(body, "CVE-2024-5678") // Low
|
||||
|
||||
if critIdx == -1 || highIdx == -1 || lowIdx == -1 {
|
||||
t.Fatal("Expected all three CVEs to be present in body")
|
||||
}
|
||||
|
||||
if critIdx > highIdx {
|
||||
t.Error("Critical CVE should appear before High CVE")
|
||||
}
|
||||
if highIdx > lowIdx {
|
||||
t.Error("High CVE should appear before Low CVE")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnDetails_MissingParams(t *testing.T) {
|
||||
handler := setupVulnDetailsHandler(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/vuln-details", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
|
||||
if !strings.Contains(body, "Missing required parameters") {
|
||||
t.Error("Expected error message for missing parameters")
|
||||
}
|
||||
}
|
||||
4
pkg/appview/public/js/bundle.min.js
vendored
4
pkg/appview/public/js/bundle.min.js
vendored
File diff suppressed because one or more lines are too long
@@ -122,6 +122,10 @@ func RegisterUIRoutes(router chi.Router, deps UIDependencies) {
|
||||
// Manifest health check API endpoint (HTMX polling)
|
||||
router.Get("/api/manifest-health", (&uihandlers.ManifestHealthHandler{BaseUIHandler: base}).ServeHTTP)
|
||||
|
||||
// Vulnerability scan result API endpoints (HTMX lazy loading + modal content)
|
||||
router.Get("/api/scan-result", (&uihandlers.ScanResultHandler{BaseUIHandler: base}).ServeHTTP)
|
||||
router.Get("/api/vuln-details", (&uihandlers.VulnDetailsHandler{BaseUIHandler: base}).ServeHTTP)
|
||||
|
||||
router.Get("/u/{handle}", middleware.OptionalAuth(deps.SessionStore, deps.Database)(
|
||||
&uihandlers.UserPageHandler{BaseUIHandler: base},
|
||||
).ServeHTTP)
|
||||
|
||||
@@ -363,6 +363,24 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// Vulnerability details modal
|
||||
async function openVulnDetails(digest, holdEndpoint) {
|
||||
const modal = document.getElementById('vuln-detail-modal');
|
||||
const body = document.getElementById('vuln-modal-body');
|
||||
if (!modal || !body) return;
|
||||
|
||||
// Show modal with loading spinner
|
||||
body.innerHTML = '<div class="flex justify-center py-8"><span class="loading loading-spinner loading-lg"></span></div>';
|
||||
modal.showModal();
|
||||
|
||||
try {
|
||||
const resp = await fetch(`/api/vuln-details?digest=${encodeURIComponent(digest)}&holdEndpoint=${encodeURIComponent(holdEndpoint)}`);
|
||||
body.innerHTML = await resp.text();
|
||||
} catch {
|
||||
body.innerHTML = '<p class="text-error">Failed to load vulnerability details</p>';
|
||||
}
|
||||
}
|
||||
|
||||
// Login page recent accounts helper (works alongside actor-typeahead web component)
|
||||
class RecentAccountsHelper {
|
||||
constructor(inputElement) {
|
||||
@@ -692,3 +710,4 @@ window.copyToClipboard = copyToClipboard;
|
||||
window.toggleOfflineManifests = toggleOfflineManifests;
|
||||
window.deleteManifest = deleteManifest;
|
||||
window.closeManifestDeleteModal = closeManifestDeleteModal;
|
||||
window.openVulnDetails = openVulnDetails;
|
||||
|
||||
@@ -208,6 +208,13 @@
|
||||
{{ else if not .Reachable }}
|
||||
<span class="badge badge-sm badge-warning">{{ icon "alert-triangle" "size-3" }} Offline</span>
|
||||
{{ end }}
|
||||
{{/* Vulnerability scan badge (lazy-loaded from hold) */}}
|
||||
{{ if and (not .IsManifestList) .Manifest.HoldEndpoint }}
|
||||
<span hx-get="/api/scan-result?digest={{ .Manifest.Digest | urlquery }}&holdEndpoint={{ .Manifest.HoldEndpoint | urlquery }}"
|
||||
hx-trigger="load delay:1s"
|
||||
hx-swap="outerHTML">
|
||||
</span>
|
||||
{{ end }}
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<code class="font-mono text-xs text-base-content/60 truncate max-w-40" title="{{ .Manifest.Digest }}">{{ .Manifest.Digest }}</code>
|
||||
@@ -283,6 +290,20 @@
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<!-- Vulnerability Details Modal -->
|
||||
<dialog id="vuln-detail-modal" class="modal">
|
||||
<div class="modal-box max-w-4xl">
|
||||
<h3 class="text-lg font-bold">Vulnerability Scan Results</h3>
|
||||
<div id="vuln-modal-body" class="py-4">
|
||||
<span class="loading loading-spinner loading-md"></span>
|
||||
</div>
|
||||
<div class="modal-action">
|
||||
<form method="dialog"><button class="btn">Close</button></form>
|
||||
</div>
|
||||
</div>
|
||||
<form method="dialog" class="modal-backdrop"><button>close</button></form>
|
||||
</dialog>
|
||||
|
||||
{{ template "footer" . }}
|
||||
</body>
|
||||
</html>
|
||||
|
||||
24
pkg/appview/templates/partials/vuln-badge.html
Normal file
24
pkg/appview/templates/partials/vuln-badge.html
Normal file
@@ -0,0 +1,24 @@
|
||||
{{ define "vuln-badge" }}
|
||||
{{ if .Error }}
|
||||
{{/* Silently hide on error / no scan record — scan badges are non-critical */}}
|
||||
{{ else if eq .Total 0 }}
|
||||
<span class="badge badge-sm badge-success" title="No vulnerabilities found (scanned {{ .ScannedAt }})">{{ icon "shield-check" "size-3" }} Clean</span>
|
||||
{{ else }}
|
||||
<button class="flex items-center gap-1 cursor-pointer hover:opacity-80 transition-opacity"
|
||||
onclick="openVulnDetails('{{ .Digest }}', '{{ .HoldEndpoint }}')"
|
||||
title="Click for vulnerability details (scanned {{ .ScannedAt }})">
|
||||
{{ if gt .Critical 0 }}
|
||||
<span class="badge badge-sm badge-error">C:{{ .Critical }}</span>
|
||||
{{ end }}
|
||||
{{ if gt .High 0 }}
|
||||
<span class="badge badge-sm badge-warning">H:{{ .High }}</span>
|
||||
{{ end }}
|
||||
{{ if gt .Medium 0 }}
|
||||
<span class="badge badge-sm badge-soft badge-warning">M:{{ .Medium }}</span>
|
||||
{{ end }}
|
||||
{{ if gt .Low 0 }}
|
||||
<span class="badge badge-sm badge-info">L:{{ .Low }}</span>
|
||||
{{ end }}
|
||||
</button>
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
87
pkg/appview/templates/partials/vuln-details.html
Normal file
87
pkg/appview/templates/partials/vuln-details.html
Normal file
@@ -0,0 +1,87 @@
|
||||
{{ define "vuln-details" }}
|
||||
{{ if .Error }}
|
||||
{{ if .Summary.Total }}
|
||||
<!-- Summary available but no detailed report -->
|
||||
<div class="space-y-4">
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{{ if gt .Summary.Critical 0 }}<span class="badge badge-error">{{ .Summary.Critical }} Critical</span>{{ end }}
|
||||
{{ if gt .Summary.High 0 }}<span class="badge badge-warning">{{ .Summary.High }} High</span>{{ end }}
|
||||
{{ if gt .Summary.Medium 0 }}<span class="badge badge-soft badge-warning">{{ .Summary.Medium }} Medium</span>{{ end }}
|
||||
{{ if gt .Summary.Low 0 }}<span class="badge badge-info">{{ .Summary.Low }} Low</span>{{ end }}
|
||||
</div>
|
||||
<p class="text-base-content/60 text-sm">{{ .Error }}</p>
|
||||
{{ if .ScannedAt }}<p class="text-base-content/40 text-xs">Scanned: {{ .ScannedAt }}</p>{{ end }}
|
||||
</div>
|
||||
{{ else }}
|
||||
<p class="text-base-content/60">{{ .Error }}</p>
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
<div class="space-y-4">
|
||||
<!-- Summary badges -->
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<span class="font-semibold text-sm">{{ .Summary.Total }} vulnerabilities found</span>
|
||||
{{ if gt .Summary.Critical 0 }}<span class="badge badge-error">{{ .Summary.Critical }} Critical</span>{{ end }}
|
||||
{{ if gt .Summary.High 0 }}<span class="badge badge-warning">{{ .Summary.High }} High</span>{{ end }}
|
||||
{{ if gt .Summary.Medium 0 }}<span class="badge badge-soft badge-warning">{{ .Summary.Medium }} Medium</span>{{ end }}
|
||||
{{ if gt .Summary.Low 0 }}<span class="badge badge-info">{{ .Summary.Low }} Low</span>{{ end }}
|
||||
</div>
|
||||
|
||||
{{ if .ScannedAt }}<p class="text-base-content/40 text-xs">Scanned: {{ .ScannedAt }}</p>{{ end }}
|
||||
|
||||
{{ if .Matches }}
|
||||
<!-- CVE table -->
|
||||
<div class="overflow-x-auto max-h-96">
|
||||
<table class="table table-sm table-pin-rows">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>CVE</th>
|
||||
<th>Severity</th>
|
||||
<th>Package</th>
|
||||
<th>Installed</th>
|
||||
<th>Fixed In</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ range .Matches }}
|
||||
<tr>
|
||||
<td class="font-mono text-xs">
|
||||
{{ if .CVEURL }}
|
||||
<a href="{{ .CVEURL }}" target="_blank" rel="noopener noreferrer" class="link link-primary">{{ .CVEID }}</a>
|
||||
{{ else }}
|
||||
{{ .CVEID }}
|
||||
{{ end }}
|
||||
</td>
|
||||
<td>
|
||||
{{ if eq .Severity "Critical" }}
|
||||
<span class="badge badge-sm badge-error">Critical</span>
|
||||
{{ else if eq .Severity "High" }}
|
||||
<span class="badge badge-sm badge-warning">High</span>
|
||||
{{ else if eq .Severity "Medium" }}
|
||||
<span class="badge badge-sm badge-soft badge-warning">Medium</span>
|
||||
{{ else if eq .Severity "Low" }}
|
||||
<span class="badge badge-sm badge-info">Low</span>
|
||||
{{ else }}
|
||||
<span class="badge badge-sm badge-ghost">{{ .Severity }}</span>
|
||||
{{ end }}
|
||||
</td>
|
||||
<td>
|
||||
<span class="font-mono text-xs">{{ .Package }}</span>
|
||||
{{ if .Type }}<span class="text-base-content/40 text-xs">({{ .Type }})</span>{{ end }}
|
||||
</td>
|
||||
<td class="font-mono text-xs">{{ .Version }}</td>
|
||||
<td class="font-mono text-xs">
|
||||
{{ if .FixedIn }}
|
||||
<span class="text-success">{{ .FixedIn }}</span>
|
||||
{{ else }}
|
||||
<span class="text-base-content/40">No fix</span>
|
||||
{{ end }}
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{ end }}
|
||||
</div>
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
@@ -1851,7 +1851,7 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error {
|
||||
|
||||
cw := cbg.NewCborWriter(w)
|
||||
|
||||
if _, err := cw.Write([]byte{172}); err != nil {
|
||||
if _, err := cw.Write([]byte{173}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2118,6 +2118,22 @@ func (t *ScanRecord) MarshalCBOR(w io.Writer) error {
|
||||
if _, err := cw.WriteString(string(t.ScannerVersion)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.VulnReportBlob (util.LexBlob) (struct)
|
||||
if len("vulnReportBlob") > 8192 {
|
||||
return xerrors.Errorf("Value in field \"vulnReportBlob\" was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vulnReportBlob"))); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := cw.WriteString(string("vulnReportBlob")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := t.VulnReportBlob.MarshalCBOR(cw); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2378,6 +2394,26 @@ func (t *ScanRecord) UnmarshalCBOR(r io.Reader) (err error) {
|
||||
|
||||
t.ScannerVersion = string(sval)
|
||||
}
|
||||
// t.VulnReportBlob (util.LexBlob) (struct)
|
||||
case "vulnReportBlob":
|
||||
|
||||
{
|
||||
|
||||
b, err := cr.ReadByte()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b != cbg.CborNull[0] {
|
||||
if err := cr.UnreadByte(); err != nil {
|
||||
return err
|
||||
}
|
||||
t.VulnReportBlob = new(util.LexBlob)
|
||||
if err := t.VulnReportBlob.UnmarshalCBOR(cr); err != nil {
|
||||
return xerrors.Errorf("unmarshaling t.VulnReportBlob pointer: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
default:
|
||||
// Field doesn't exist on this type, so ignore it
|
||||
|
||||
@@ -801,30 +801,33 @@ func CrewRecordKey(memberDID string) string {
|
||||
// RKey is deterministic: based on manifest digest (one scan per manifest)
|
||||
type ScanRecord struct {
|
||||
Type string `json:"$type" cborgen:"$type"`
|
||||
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the scanned manifest (e.g., "at://did:plc:xyz/io.atcr.manifest/abc123...")
|
||||
Repository string `json:"repository" cborgen:"repository"` // Repository name (e.g., "myapp")
|
||||
UserDID string `json:"userDid" cborgen:"userDid"` // DID of the image owner
|
||||
SbomBlob *lexutil.LexBlob `json:"sbomBlob,omitempty" cborgen:"sbomBlob"` // SBOM blob uploaded to hold's PDS blob storage
|
||||
Critical int64 `json:"critical" cborgen:"critical"` // Count of critical vulnerabilities
|
||||
High int64 `json:"high" cborgen:"high"` // Count of high vulnerabilities
|
||||
Medium int64 `json:"medium" cborgen:"medium"` // Count of medium vulnerabilities
|
||||
Low int64 `json:"low" cborgen:"low"` // Count of low vulnerabilities
|
||||
Total int64 `json:"total" cborgen:"total"` // Total vulnerability count
|
||||
ScannerVersion string `json:"scannerVersion" cborgen:"scannerVersion"` // Scanner version (e.g., "atcr-scanner-v1.0.0")
|
||||
ScannedAt string `json:"scannedAt" cborgen:"scannedAt"` // RFC3339 timestamp of scan completion
|
||||
Manifest string `json:"manifest" cborgen:"manifest"` // AT-URI of the scanned manifest (e.g., "at://did:plc:xyz/io.atcr.manifest/abc123...")
|
||||
Repository string `json:"repository" cborgen:"repository"` // Repository name (e.g., "myapp")
|
||||
UserDID string `json:"userDid" cborgen:"userDid"` // DID of the image owner
|
||||
SbomBlob *lexutil.LexBlob `json:"sbomBlob,omitempty" cborgen:"sbomBlob"` // SBOM blob uploaded to hold's PDS blob storage
|
||||
VulnReportBlob *lexutil.LexBlob `json:"vulnReportBlob,omitempty" cborgen:"vulnReportBlob"` // Grype vulnerability report blob (full CVE details)
|
||||
Critical int64 `json:"critical" cborgen:"critical"` // Count of critical vulnerabilities
|
||||
High int64 `json:"high" cborgen:"high"` // Count of high vulnerabilities
|
||||
Medium int64 `json:"medium" cborgen:"medium"` // Count of medium vulnerabilities
|
||||
Low int64 `json:"low" cborgen:"low"` // Count of low vulnerabilities
|
||||
Total int64 `json:"total" cborgen:"total"` // Total vulnerability count
|
||||
ScannerVersion string `json:"scannerVersion" cborgen:"scannerVersion"` // Scanner version (e.g., "atcr-scanner-v1.0.0")
|
||||
ScannedAt string `json:"scannedAt" cborgen:"scannedAt"` // RFC3339 timestamp of scan completion
|
||||
}
|
||||
|
||||
// NewScanRecord creates a new scan record
|
||||
// manifestDigest: the manifest digest (e.g., "sha256:abc123...")
|
||||
// userDID: the DID of the image owner (used to build the manifest AT-URI)
|
||||
// sbomBlob: blob reference from uploading SBOM to PDS blob storage (nil if no SBOM)
|
||||
func NewScanRecord(manifestDigest, repository, userDID string, sbomBlob *lexutil.LexBlob, critical, high, medium, low, total int, scannerVersion string) *ScanRecord {
|
||||
// vulnReportBlob: blob reference from uploading Grype vulnerability report (nil if no report)
|
||||
func NewScanRecord(manifestDigest, repository, userDID string, sbomBlob, vulnReportBlob *lexutil.LexBlob, critical, high, medium, low, total int, scannerVersion string) *ScanRecord {
|
||||
return &ScanRecord{
|
||||
Type: ScanCollection,
|
||||
Manifest: BuildManifestURI(userDID, manifestDigest),
|
||||
Repository: repository,
|
||||
UserDID: userDID,
|
||||
SbomBlob: sbomBlob,
|
||||
VulnReportBlob: vulnReportBlob,
|
||||
Critical: int64(critical),
|
||||
High: int64(high),
|
||||
Medium: int64(medium),
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/distribution/distribution/v3/configuration"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"atcr.io/pkg/config"
|
||||
@@ -64,7 +63,7 @@ type RegistrationConfig struct {
|
||||
Region string `yaml:"region" comment:"Deployment region, auto-detected from cloud metadata or S3 config."`
|
||||
}
|
||||
|
||||
// StorageConfig holds S3 storage credentials and the internal distribution config.
|
||||
// StorageConfig holds S3 storage credentials.
|
||||
type StorageConfig struct {
|
||||
// S3-compatible access key.
|
||||
AccessKey string `yaml:"access_key" comment:"S3-compatible access key (AWS, Storj, Minio, UpCloud)."`
|
||||
@@ -83,24 +82,24 @@ type StorageConfig struct {
|
||||
|
||||
// CDN pull zone URL for presigned download URLs.
|
||||
PullZone string `yaml:"pull_zone" comment:"CDN pull zone URL for downloads. When set, presigned GET/HEAD URLs use this host instead of the S3 endpoint. Uploads and API calls still use the S3 endpoint."`
|
||||
|
||||
// Internal distribution storage config, built from the above fields.
|
||||
distStorage configuration.Storage `yaml:"-"`
|
||||
}
|
||||
|
||||
// Type returns the storage driver type name (always "s3").
|
||||
func (s StorageConfig) Type() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
// Parameters returns the distribution driver parameters.
|
||||
func (s StorageConfig) Parameters() configuration.Parameters {
|
||||
if s.distStorage != nil {
|
||||
if params, ok := s.distStorage["s3"]; ok {
|
||||
return params
|
||||
}
|
||||
// S3Params returns a params map suitable for s3.NewS3Service.
|
||||
func (s StorageConfig) S3Params() map[string]any {
|
||||
params := map[string]any{
|
||||
"accesskey": s.AccessKey,
|
||||
"secretkey": s.SecretKey,
|
||||
"region": s.Region,
|
||||
"bucket": s.Bucket,
|
||||
}
|
||||
return nil
|
||||
if s.Endpoint != "" {
|
||||
params["regionendpoint"] = s.Endpoint
|
||||
params["forcepathstyle"] = true
|
||||
}
|
||||
if s.PullZone != "" {
|
||||
params["pullzone"] = s.PullZone
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// ServerConfig defines server settings
|
||||
@@ -276,9 +275,6 @@ func LoadConfig(yamlPath string) (*Config, error) {
|
||||
// Store config path for subsystem config loading (e.g. billing)
|
||||
cfg.configPath = yamlPath
|
||||
|
||||
// Build distribution storage config from struct fields
|
||||
cfg.Storage.distStorage = buildStorageConfigFromFields(cfg.Storage)
|
||||
|
||||
// Detect region from cloud metadata or S3 config
|
||||
if meta, err := DetectCloudMetadata(context.Background()); err == nil && meta != nil {
|
||||
cfg.Registration.Region = meta.Region
|
||||
@@ -290,25 +286,3 @@ func LoadConfig(yamlPath string) (*Config, error) {
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// buildStorageConfigFromFields creates S3 storage configuration from StorageConfig fields.
|
||||
func buildStorageConfigFromFields(sc StorageConfig) configuration.Storage {
|
||||
params := make(map[string]any)
|
||||
|
||||
params["accesskey"] = sc.AccessKey
|
||||
params["secretkey"] = sc.SecretKey
|
||||
params["region"] = sc.Region
|
||||
params["bucket"] = sc.Bucket
|
||||
if sc.Endpoint != "" {
|
||||
params["regionendpoint"] = sc.Endpoint
|
||||
params["forcepathstyle"] = true
|
||||
}
|
||||
if sc.PullZone != "" {
|
||||
params["pullzone"] = sc.PullZone
|
||||
}
|
||||
|
||||
storageCfg := configuration.Storage{}
|
||||
storageCfg["s3"] = configuration.Parameters(params)
|
||||
|
||||
return storageCfg
|
||||
}
|
||||
|
||||
@@ -187,7 +187,7 @@ func TestLoadConfig_KeyPathDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
|
||||
func TestS3Params_Complete(t *testing.T) {
|
||||
sc := StorageConfig{
|
||||
AccessKey: "test-access-key",
|
||||
SecretKey: "test-secret-key",
|
||||
@@ -196,14 +196,7 @@ func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
|
||||
Endpoint: "https://s3.example.com",
|
||||
}
|
||||
|
||||
cfg := buildStorageConfigFromFields(sc)
|
||||
|
||||
s3Params, ok := cfg["s3"]
|
||||
if !ok {
|
||||
t.Fatal("Expected s3 storage config")
|
||||
}
|
||||
|
||||
params := map[string]any(s3Params)
|
||||
params := sc.S3Params()
|
||||
|
||||
if params["accesskey"] != "test-access-key" {
|
||||
t.Errorf("Expected accesskey=test-access-key, got %v", params["accesskey"])
|
||||
@@ -222,7 +215,7 @@ func TestBuildStorageConfigFromFields_S3_Complete(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStorageConfigFromFields_S3_NoEndpoint(t *testing.T) {
|
||||
func TestS3Params_NoEndpoint(t *testing.T) {
|
||||
sc := StorageConfig{
|
||||
AccessKey: "test-key",
|
||||
SecretKey: "test-secret",
|
||||
@@ -231,14 +224,7 @@ func TestBuildStorageConfigFromFields_S3_NoEndpoint(t *testing.T) {
|
||||
Endpoint: "", // No custom endpoint
|
||||
}
|
||||
|
||||
cfg := buildStorageConfigFromFields(sc)
|
||||
|
||||
s3Params, ok := cfg["s3"]
|
||||
if !ok {
|
||||
t.Fatal("Expected s3 storage config")
|
||||
}
|
||||
|
||||
params := map[string]any(s3Params)
|
||||
params := sc.S3Params()
|
||||
|
||||
// Should have default region
|
||||
if params["region"] != "us-east-1" {
|
||||
|
||||
@@ -14,8 +14,8 @@ import (
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/hold/pds"
|
||||
"atcr.io/pkg/s3"
|
||||
"github.com/bluesky-social/indigo/atproto/syntax"
|
||||
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
)
|
||||
|
||||
// maxPreviewItems caps per-category detail slices to prevent memory/HTML bloat
|
||||
@@ -64,7 +64,7 @@ type GCPreview struct {
|
||||
// GarbageCollector handles cleanup of orphaned blobs from storage
|
||||
type GarbageCollector struct {
|
||||
pds *pds.HoldPDS
|
||||
driver storagedriver.StorageDriver
|
||||
s3 *s3.S3Service
|
||||
cfg Config
|
||||
logger *slog.Logger
|
||||
|
||||
@@ -117,10 +117,10 @@ type analysisResult struct {
|
||||
}
|
||||
|
||||
// NewGarbageCollector creates a new GC instance
|
||||
func NewGarbageCollector(holdPDS *pds.HoldPDS, driver storagedriver.StorageDriver, cfg Config) *GarbageCollector {
|
||||
func NewGarbageCollector(holdPDS *pds.HoldPDS, s3svc *s3.S3Service, cfg Config) *GarbageCollector {
|
||||
return &GarbageCollector{
|
||||
pds: holdPDS,
|
||||
driver: driver,
|
||||
s3: s3svc,
|
||||
cfg: cfg,
|
||||
logger: slog.Default().With("component", "gc"),
|
||||
stopCh: make(chan struct{}),
|
||||
@@ -454,15 +454,12 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
|
||||
totalBlobs := 0
|
||||
blobsPath := "/docker/registry/v2/blobs"
|
||||
|
||||
err := gc.driver.Walk(ctx, blobsPath, func(fi storagedriver.FileInfo) error {
|
||||
if fi.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(fi.Path(), "/data") {
|
||||
err := gc.s3.WalkBlobs(ctx, blobsPath, func(key string, size int64) error {
|
||||
if !strings.HasSuffix(key, "/data") {
|
||||
return nil
|
||||
}
|
||||
|
||||
digest := extractDigestFromPath(fi.Path())
|
||||
digest := extractDigestFromPath(key)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -473,7 +470,7 @@ func (gc *GarbageCollector) scanOrphanedBlobDetails(ctx context.Context, referen
|
||||
if len(orphaned) < maxPreviewItems {
|
||||
orphaned = append(orphaned, OrphanedBlobDetail{
|
||||
Digest: digest,
|
||||
Size: fi.Size(),
|
||||
Size: size,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -677,18 +674,14 @@ func (gc *GarbageCollector) deleteOrphanedRecords(ctx context.Context, orphanedR
|
||||
func (gc *GarbageCollector) deleteOrphanedBlobs(ctx context.Context, referenced map[string]bool, result *GCResult) error {
|
||||
blobsPath := "/docker/registry/v2/blobs"
|
||||
|
||||
err := gc.driver.Walk(ctx, blobsPath, func(fi storagedriver.FileInfo) error {
|
||||
if fi.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := gc.s3.WalkBlobs(ctx, blobsPath, func(key string, size int64) error {
|
||||
// Only process data files
|
||||
if !strings.HasSuffix(fi.Path(), "/data") {
|
||||
if !strings.HasSuffix(key, "/data") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Extract digest from path
|
||||
digest := extractDigestFromPath(fi.Path())
|
||||
digest := extractDigestFromPath(key)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -700,15 +693,15 @@ func (gc *GarbageCollector) deleteOrphanedBlobs(ctx context.Context, referenced
|
||||
|
||||
result.OrphanedBlobs++
|
||||
|
||||
if err := gc.driver.Delete(ctx, fi.Path()); err != nil {
|
||||
gc.logger.Error("Failed to delete blob", "path", fi.Path(), "error", err)
|
||||
if err := gc.s3.Delete(ctx, key); err != nil {
|
||||
gc.logger.Error("Failed to delete blob", "path", key, "error", err)
|
||||
return nil // Continue with other blobs
|
||||
}
|
||||
result.BlobsDeleted++
|
||||
result.BytesReclaimed += fi.Size()
|
||||
result.BytesReclaimed += size
|
||||
gc.logger.Debug("Deleted orphaned blob",
|
||||
"digest", digest,
|
||||
"size", fi.Size())
|
||||
"size", size)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -256,7 +256,7 @@ func (h *XRPCHandler) CompleteMultipartUploadWithManager(ctx context.Context, up
|
||||
"source", sourcePath,
|
||||
"dest", destPath)
|
||||
|
||||
if _, err := h.driver.Stat(ctx, sourcePath); err != nil {
|
||||
if _, err := h.s3Service.Stat(ctx, sourcePath); err != nil {
|
||||
slog.Error("Source blob not found after multipart complete",
|
||||
"path", sourcePath,
|
||||
"error", err)
|
||||
@@ -264,9 +264,8 @@ func (h *XRPCHandler) CompleteMultipartUploadWithManager(ctx context.Context, up
|
||||
}
|
||||
slog.Debug("Source blob verified", "path", sourcePath)
|
||||
|
||||
// Move from temp to final digest location using driver
|
||||
// Driver handles path management correctly (including S3 prefix)
|
||||
if err := h.driver.Move(ctx, sourcePath, destPath); err != nil {
|
||||
// Move from temp to final digest location (S3 copy + delete)
|
||||
if err := h.s3Service.Move(ctx, sourcePath, destPath); err != nil {
|
||||
slog.Error("Failed to move blob",
|
||||
"source", sourcePath,
|
||||
"dest", destPath,
|
||||
|
||||
@@ -12,14 +12,12 @@ import (
|
||||
"atcr.io/pkg/hold/pds"
|
||||
"atcr.io/pkg/hold/quota"
|
||||
"atcr.io/pkg/s3"
|
||||
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/render"
|
||||
)
|
||||
|
||||
// XRPCHandler handles OCI-specific XRPC endpoints for multipart uploads
|
||||
type XRPCHandler struct {
|
||||
driver storagedriver.StorageDriver
|
||||
s3Service s3.S3Service
|
||||
MultipartMgr *MultipartManager // Exported for access in route handlers
|
||||
pds *pds.HoldPDS
|
||||
@@ -30,9 +28,8 @@ type XRPCHandler struct {
|
||||
}
|
||||
|
||||
// NewXRPCHandler creates a new OCI XRPC handler
|
||||
func NewXRPCHandler(holdPDS *pds.HoldPDS, s3Service s3.S3Service, driver storagedriver.StorageDriver, enableBlueskyPosts bool, httpClient pds.HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
|
||||
func NewXRPCHandler(holdPDS *pds.HoldPDS, s3Service s3.S3Service, enableBlueskyPosts bool, httpClient pds.HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
|
||||
return &XRPCHandler{
|
||||
driver: driver,
|
||||
MultipartMgr: NewMultipartManager(),
|
||||
s3Service: s3Service,
|
||||
pds: holdPDS,
|
||||
@@ -366,7 +363,7 @@ func (h *XRPCHandler) HandleNotifyManifest(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
postURI, err = h.pds.CreateManifestPost(
|
||||
ctx,
|
||||
h.driver,
|
||||
&h.s3Service,
|
||||
req.Repository,
|
||||
req.Tag,
|
||||
userHandle,
|
||||
|
||||
@@ -2,7 +2,6 @@ package oci
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -10,17 +9,12 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/auth/oauth"
|
||||
"atcr.io/pkg/hold/pds"
|
||||
"atcr.io/pkg/s3"
|
||||
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws"
|
||||
)
|
||||
|
||||
// Shared test resources for OCI package
|
||||
@@ -68,148 +62,10 @@ func (m *mockPDSClient) Do(req *http.Request) (*http.Response, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// mockStorageDriver implements storagedriver.StorageDriver for testing
|
||||
type mockStorageDriver struct {
|
||||
mu sync.RWMutex
|
||||
blobs map[string][]byte
|
||||
|
||||
// Error injection for testing error handling
|
||||
StatError error
|
||||
MoveError error
|
||||
}
|
||||
|
||||
func newMockStorageDriver() *mockStorageDriver {
|
||||
return &mockStorageDriver{
|
||||
blobs: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Name() string { return "mock" }
|
||||
|
||||
func (m *mockStorageDriver) GetContent(ctx context.Context, path string) ([]byte, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if data, ok := m.blobs[path]; ok {
|
||||
return data, nil
|
||||
}
|
||||
return nil, storagedriver.PathNotFoundError{Path: path}
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) PutContent(ctx context.Context, path string, content []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.blobs[path] = content
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
|
||||
data, err := m.GetContent(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data[offset:])), nil
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
|
||||
return &mockFileWriter{driver: m, path: path}, nil
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
// Check for injected error
|
||||
if m.StatError != nil {
|
||||
return nil, m.StatError
|
||||
}
|
||||
|
||||
if data, ok := m.blobs[path]; ok {
|
||||
return &mockFileInfo{path: path, size: int64(len(data))}, nil
|
||||
}
|
||||
return nil, storagedriver.PathNotFoundError{Path: path}
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) List(ctx context.Context, path string) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Move(ctx context.Context, sourcePath string, destPath string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Check for injected error
|
||||
if m.MoveError != nil {
|
||||
return m.MoveError
|
||||
}
|
||||
|
||||
if data, ok := m.blobs[sourcePath]; ok {
|
||||
m.blobs[destPath] = data
|
||||
delete(m.blobs, sourcePath)
|
||||
return nil
|
||||
}
|
||||
return storagedriver.PathNotFoundError{Path: sourcePath}
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Delete(ctx context.Context, path string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.blobs, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) RedirectURL(r *http.Request, path string) (string, error) {
|
||||
return "", storagedriver.ErrUnsupportedMethod{}
|
||||
}
|
||||
|
||||
func (m *mockStorageDriver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mockFileWriter implements storagedriver.FileWriter
|
||||
type mockFileWriter struct {
|
||||
driver *mockStorageDriver
|
||||
path string
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *mockFileWriter) Write(p []byte) (int, error) {
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
|
||||
func (w *mockFileWriter) Size() int64 {
|
||||
return int64(w.buf.Len())
|
||||
}
|
||||
|
||||
func (w *mockFileWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *mockFileWriter) Cancel(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *mockFileWriter) Commit(ctx context.Context) error {
|
||||
w.driver.mu.Lock()
|
||||
defer w.driver.mu.Unlock()
|
||||
w.driver.blobs[w.path] = w.buf.Bytes()
|
||||
return nil
|
||||
}
|
||||
|
||||
// mockFileInfo implements storagedriver.FileInfo
|
||||
type mockFileInfo struct {
|
||||
path string
|
||||
size int64
|
||||
}
|
||||
|
||||
func (f *mockFileInfo) Path() string { return f.path }
|
||||
func (f *mockFileInfo) Size() int64 { return f.size }
|
||||
func (f *mockFileInfo) ModTime() time.Time { return time.Time{} }
|
||||
func (f *mockFileInfo) IsDir() bool { return false }
|
||||
|
||||
// setupTestOCIHandlerWithMockS3 creates a test OCI XRPC handler with mock S3
|
||||
// This does NOT require real S3 credentials - uses MockS3Client
|
||||
// Returns the handler, mock S3 client, and mock storage driver for test manipulation
|
||||
func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client, *mockStorageDriver) {
|
||||
// Returns the handler and mock S3 client for test manipulation
|
||||
func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client) {
|
||||
t.Helper()
|
||||
|
||||
// Create temp directory for PDS database
|
||||
@@ -226,9 +82,6 @@ func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client
|
||||
PathPrefix: "test-prefix",
|
||||
}
|
||||
|
||||
// Create mock storage driver
|
||||
mockDriver := newMockStorageDriver()
|
||||
|
||||
// Create minimal PDS for DID/auth
|
||||
dbPath := ":memory:"
|
||||
keyPath := filepath.Join(tmpDir, "signing-key")
|
||||
@@ -268,9 +121,9 @@ func setupTestOCIHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client
|
||||
mockClient := &mockPDSClient{}
|
||||
|
||||
// Create OCI handler with mock S3
|
||||
handler := NewXRPCHandler(holdPDS, s3Service, mockDriver, false, mockClient, nil)
|
||||
handler := NewXRPCHandler(holdPDS, s3Service, false, mockClient, nil)
|
||||
|
||||
return handler, mockS3Client, mockDriver
|
||||
return handler, mockS3Client
|
||||
}
|
||||
|
||||
// setupTestOCIHandlerWithS3 creates a test OCI XRPC handler with S3 driver
|
||||
@@ -307,12 +160,6 @@ func setupTestOCIHandlerWithS3(t *testing.T) (*XRPCHandler, bool) {
|
||||
}
|
||||
s3Params["rootdirectory"] = storageDir
|
||||
|
||||
driver, err := factory.Create(ctx, "s3", s3Params)
|
||||
if err != nil {
|
||||
t.Logf("Failed to create S3 storage driver: %v", err)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Create S3 service
|
||||
s3Service, err := s3.NewS3Service(s3Params)
|
||||
if err != nil {
|
||||
@@ -359,7 +206,7 @@ func setupTestOCIHandlerWithS3(t *testing.T) (*XRPCHandler, bool) {
|
||||
mockClient := &mockPDSClient{}
|
||||
|
||||
// Create OCI handler with S3
|
||||
handler := NewXRPCHandler(holdPDS, *s3Service, driver, false, mockClient, nil)
|
||||
handler := NewXRPCHandler(holdPDS, *s3Service, false, mockClient, nil)
|
||||
|
||||
return handler, true
|
||||
}
|
||||
@@ -392,7 +239,7 @@ func decodeJSONResponse(t *testing.T, w *httptest.ResponseRecorder, v any) {
|
||||
// Tests for HandleInitiateUpload - Mock S3 (no credentials required)
|
||||
|
||||
func TestHandleInitiateUpload_MockS3_Success(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
"digest": "sha256:abc123",
|
||||
@@ -421,7 +268,7 @@ func TestHandleInitiateUpload_MockS3_Success(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleInitiateUpload_MockS3_MissingDigest(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{})
|
||||
addMockAuth(req)
|
||||
@@ -437,7 +284,7 @@ func TestHandleInitiateUpload_MockS3_MissingDigest(t *testing.T) {
|
||||
// Tests for full Mock S3 upload flow (no credentials required)
|
||||
|
||||
func TestFullMockS3UploadFlow(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// 1. Initiate upload
|
||||
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
@@ -506,7 +353,7 @@ func TestFullMockS3UploadFlow(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleGetPartUploadUrl_MockS3_InvalidSession(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldGetPartUploadURL, map[string]any{
|
||||
"uploadId": "invalid-upload-id",
|
||||
@@ -523,7 +370,7 @@ func TestHandleGetPartUploadUrl_MockS3_InvalidSession(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleAbortUpload_MockS3_InvalidSession(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldAbortUpload, map[string]string{
|
||||
"uploadId": "invalid-upload-id",
|
||||
@@ -764,7 +611,7 @@ func TestFullS3UploadFlow(t *testing.T) {
|
||||
// Tests for HandleCompleteUpload with Mock S3
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
|
||||
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// 1. Initiate upload with temp path
|
||||
tempDigest := "uploads/temp-test-complete"
|
||||
@@ -796,12 +643,8 @@ func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
|
||||
t.Fatalf("Expected status 200 for part URL, got %d: %s", partW.Code, partW.Body.String())
|
||||
}
|
||||
|
||||
// 3. Pre-populate mock storage driver with temp blob (simulates S3 upload completing)
|
||||
// Path format: /docker/registry/v2/uploads/temp-{id}/data
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test blob content"))
|
||||
|
||||
// 4. Complete upload with parts
|
||||
// 3. Complete upload with parts
|
||||
// (Mock S3 CompleteMultipartUpload auto-populates object for Stat/Move)
|
||||
finalDigest := "sha256:abc123def456"
|
||||
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
"uploadId": uploadID,
|
||||
@@ -833,17 +676,15 @@ func TestHandleCompleteUpload_MockS3_Success(t *testing.T) {
|
||||
t.Errorf("Expected 1 Complete call, got %d", len(mockS3Client.CompleteCalls))
|
||||
}
|
||||
|
||||
// 6. Verify blob was moved to final location
|
||||
// Final path format: /docker/registry/v2/blobs/sha256/ab/abc123def456/data
|
||||
finalBlobPath := "/docker/registry/v2/blobs/sha256/ab/abc123def456/data"
|
||||
_, err := mockDriver.Stat(t.Context(), finalBlobPath)
|
||||
if err != nil {
|
||||
t.Errorf("Expected blob at final location %s, got error: %v", finalBlobPath, err)
|
||||
// 6. Verify blob was moved to final location in mock S3
|
||||
finalS3Key := "test-prefix/docker/registry/v2/blobs/sha256/ab/abc123def456/data"
|
||||
if mockS3Client.GetObject(finalS3Key) == nil {
|
||||
t.Errorf("Expected blob at final S3 key %s", finalS3Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_InvalidSession(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
"uploadId": "non-existent-upload-id",
|
||||
@@ -863,7 +704,7 @@ func TestHandleCompleteUpload_MockS3_InvalidSession(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_MissingParams(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -891,7 +732,7 @@ func TestHandleCompleteUpload_MockS3_MissingParams(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
|
||||
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Setup upload session
|
||||
tempDigest := "uploads/temp-etag-test"
|
||||
@@ -906,10 +747,6 @@ func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
|
||||
decodeJSONResponse(t, initW, &initResp)
|
||||
uploadID := initResp["uploadId"].(string)
|
||||
|
||||
// Pre-populate mock driver
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
|
||||
|
||||
// Complete with unquoted ETags
|
||||
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
"uploadId": uploadID,
|
||||
@@ -938,7 +775,7 @@ func TestHandleCompleteUpload_MockS3_ETagNormalization(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
|
||||
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Setup upload session
|
||||
tempDigest := "uploads/temp-s3-error"
|
||||
@@ -953,10 +790,6 @@ func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
|
||||
decodeJSONResponse(t, initW, &initResp)
|
||||
uploadID := initResp["uploadId"].(string)
|
||||
|
||||
// Pre-populate mock driver
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
|
||||
|
||||
// Inject S3 error
|
||||
mockS3Client.CompleteError = fmt.Errorf("simulated S3 CompleteMultipartUpload failure")
|
||||
|
||||
@@ -983,7 +816,7 @@ func TestHandleCompleteUpload_MockS3_S3Error(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
|
||||
handler, _, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Setup upload session
|
||||
tempDigest := "uploads/temp-stat-error"
|
||||
@@ -998,12 +831,8 @@ func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
|
||||
decodeJSONResponse(t, initW, &initResp)
|
||||
uploadID := initResp["uploadId"].(string)
|
||||
|
||||
// Pre-populate mock driver (so S3 complete succeeds)
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
|
||||
|
||||
// Inject Stat error (simulates blob not found after S3 complete)
|
||||
mockDriver.StatError = fmt.Errorf("simulated stat failure")
|
||||
// Inject HeadObject error (simulates blob not found after S3 complete)
|
||||
mockS3Client.HeadObjectError = fmt.Errorf("simulated stat failure")
|
||||
|
||||
// Complete upload should fail
|
||||
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
@@ -1023,7 +852,7 @@ func TestHandleCompleteUpload_MockS3_StatError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
|
||||
handler, _, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Setup upload session
|
||||
tempDigest := "uploads/temp-move-error"
|
||||
@@ -1038,12 +867,8 @@ func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
|
||||
decodeJSONResponse(t, initW, &initResp)
|
||||
uploadID := initResp["uploadId"].(string)
|
||||
|
||||
// Pre-populate mock driver
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
|
||||
|
||||
// Inject Move error
|
||||
mockDriver.MoveError = fmt.Errorf("simulated move failure")
|
||||
// Inject CopyObject error (Move = Copy + Delete, so Copy error simulates move failure)
|
||||
mockS3Client.CopyObjectError = fmt.Errorf("simulated move failure")
|
||||
|
||||
// Complete upload should fail
|
||||
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
@@ -1068,7 +893,7 @@ func TestHandleCompleteUpload_MockS3_MoveError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
|
||||
handler, mockS3Client, mockDriver := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Setup upload session
|
||||
tempDigest := "uploads/temp-unsorted"
|
||||
@@ -1083,10 +908,6 @@ func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
|
||||
decodeJSONResponse(t, initW, &initResp)
|
||||
uploadID := initResp["uploadId"].(string)
|
||||
|
||||
// Pre-populate mock driver
|
||||
tempBlobPath := fmt.Sprintf("/docker/registry/v2/%s/data", tempDigest)
|
||||
mockDriver.PutContent(t.Context(), tempBlobPath, []byte("test"))
|
||||
|
||||
// Complete with unsorted parts (3, 1, 2) - handler should sort them
|
||||
completeReq := makeJSONRequest("POST", atproto.HoldCompleteUpload, map[string]any{
|
||||
"uploadId": uploadID,
|
||||
@@ -1117,7 +938,7 @@ func TestHandleCompleteUpload_MockS3_UnsortedParts(t *testing.T) {
|
||||
// Tests for HandleInitiateUpload edge cases
|
||||
|
||||
func TestHandleInitiateUpload_MockS3_S3Error(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// Inject S3 error
|
||||
mockS3Client.CreateMultipartError = fmt.Errorf("simulated S3 CreateMultipartUpload failure")
|
||||
@@ -1141,7 +962,7 @@ func TestHandleInitiateUpload_MockS3_S3Error(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleInitiateUpload_MockS3_WhitespaceDigest(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
"digest": " ", // Whitespace only
|
||||
@@ -1162,7 +983,7 @@ func TestHandleInitiateUpload_MockS3_WhitespaceDigest(t *testing.T) {
|
||||
// Tests for HandleGetPartUploadURL edge cases
|
||||
|
||||
func TestHandleGetPartUploadUrl_MockS3_MissingParams(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -1189,7 +1010,7 @@ func TestHandleGetPartUploadUrl_MockS3_MissingParams(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleGetPartUploadUrl_MockS3_ValidSession(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// First initiate an upload
|
||||
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
@@ -1237,7 +1058,7 @@ func TestHandleGetPartUploadUrl_MockS3_ValidSession(t *testing.T) {
|
||||
// Tests for HandleAbortUpload edge cases
|
||||
|
||||
func TestHandleAbortUpload_MockS3_MissingUploadId(t *testing.T) {
|
||||
handler, _, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
req := makeJSONRequest("POST", atproto.HoldAbortUpload, map[string]string{})
|
||||
addMockAuth(req)
|
||||
@@ -1251,7 +1072,7 @@ func TestHandleAbortUpload_MockS3_MissingUploadId(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleAbortUpload_MockS3_S3Error(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// First initiate an upload
|
||||
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
@@ -1288,7 +1109,7 @@ func TestHandleAbortUpload_MockS3_S3Error(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandleAbortUpload_MockS3_ValidSession(t *testing.T) {
|
||||
handler, mockS3Client, _ := setupTestOCIHandlerWithMockS3(t)
|
||||
handler, mockS3Client := setupTestOCIHandlerWithMockS3(t)
|
||||
|
||||
// First initiate an upload
|
||||
initReq := makeJSONRequest("POST", atproto.HoldInitiateUpload, map[string]string{
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/s3"
|
||||
bsky "github.com/bluesky-social/indigo/api/bsky"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
)
|
||||
|
||||
// CreateManifestPost creates a Bluesky post announcing a manifest upload
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
// artifactType is "container-image", "helm-chart", or "unknown"
|
||||
func (p *HoldPDS) CreateManifestPost(
|
||||
ctx context.Context,
|
||||
storageDriver driver.StorageDriver,
|
||||
s3svc *s3.S3Service,
|
||||
repository, tag, userHandle, userDID, digest string,
|
||||
totalSize int64,
|
||||
platforms []string,
|
||||
@@ -50,7 +50,7 @@ func (p *HoldPDS) CreateManifestPost(
|
||||
slog.Warn("Failed to fetch OG image, posting without embed", "error", err)
|
||||
} else {
|
||||
// Upload OG image as blob
|
||||
thumbBlob, err := uploadBlobToStorage(ctx, storageDriver, p.did, ogImageData, "image/png")
|
||||
thumbBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, ogImageData, "image/png")
|
||||
if err != nil {
|
||||
slog.Warn("Failed to upload OG image blob", "error", err)
|
||||
} else {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package pds
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
@@ -11,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/s3"
|
||||
bsky "github.com/bluesky-social/indigo/api/bsky"
|
||||
lexutil "github.com/bluesky-social/indigo/lex/util"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
@@ -68,9 +67,9 @@ func downloadImage(ctx context.Context, url string) ([]byte, string, error) {
|
||||
return data, contentType, nil
|
||||
}
|
||||
|
||||
// uploadBlobToStorage uploads a blob to the hold's storage and returns a blob reference
|
||||
// This stores the blob at the ATProto path for the hold's DID
|
||||
func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
|
||||
// uploadBlobToStorage uploads a blob to the hold's S3 storage and returns a blob reference.
|
||||
// This stores the blob at the ATProto path for the hold's DID.
|
||||
func uploadBlobToStorage(ctx context.Context, s3svc *s3.S3Service, did string, data []byte, mimeType string) (*lexutil.LexBlob, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty blob data")
|
||||
}
|
||||
@@ -90,33 +89,14 @@ func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver
|
||||
// ATProto uses CIDv1 with raw codec for blobs
|
||||
blobCID := cid.NewCidV1(0x55, mh)
|
||||
|
||||
// Store blob via distribution driver at ATProto path
|
||||
// Store blob via S3 at ATProto path
|
||||
path := atprotoBlobPath(did, blobCID.String())
|
||||
|
||||
// Write blob to storage using distribution driver
|
||||
writer, err := storageDriver.Writer(ctx, path, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
n, err := io.Copy(writer, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
writer.Cancel(ctx)
|
||||
return nil, fmt.Errorf("failed to write blob: %w", err)
|
||||
}
|
||||
|
||||
// Commit the write
|
||||
if err := writer.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("failed to commit blob: %w", err)
|
||||
}
|
||||
|
||||
if n != size {
|
||||
return nil, fmt.Errorf("size mismatch: wrote %d bytes, expected %d", n, size)
|
||||
if err := s3svc.PutBytes(ctx, path, data, mimeType); err != nil {
|
||||
return nil, fmt.Errorf("failed to put blob: %w", err)
|
||||
}
|
||||
|
||||
// Create blob reference in the format expected by bsky.ActorProfile
|
||||
// LexLink is a type alias for cid.Cid
|
||||
lexLink := lexutil.LexLink(blobCID)
|
||||
blob := &lexutil.LexBlob{
|
||||
Ref: lexLink,
|
||||
@@ -129,7 +109,7 @@ func uploadBlobToStorage(ctx context.Context, storageDriver driver.StorageDriver
|
||||
|
||||
// CreateProfileRecord creates the app.bsky.actor.profile record for the hold
|
||||
// This will FAIL if the profile record already exists.
|
||||
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, storageDriver driver.StorageDriver, displayName, description, avatarURL string) (cid.Cid, error) {
|
||||
func (p *HoldPDS) CreateProfileRecord(ctx context.Context, s3svc *s3.S3Service, displayName, description, avatarURL string) (cid.Cid, error) {
|
||||
// Create profile struct
|
||||
profile := &bsky.ActorProfile{
|
||||
DisplayName: &displayName,
|
||||
@@ -147,7 +127,7 @@ func (p *HoldPDS) CreateProfileRecord(ctx context.Context, storageDriver driver.
|
||||
slog.Debug("Uploading avatar blob",
|
||||
"size", len(imageData),
|
||||
"mimeType", mimeType)
|
||||
avatarBlob, err := uploadBlobToStorage(ctx, storageDriver, p.did, imageData, mimeType)
|
||||
avatarBlob, err := uploadBlobToStorage(ctx, s3svc, p.did, imageData, mimeType)
|
||||
if err != nil {
|
||||
return cid.Undef, fmt.Errorf("failed to upload avatar blob: %w", err)
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"time"
|
||||
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/s3"
|
||||
lexutil "github.com/bluesky-social/indigo/lex/util"
|
||||
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
@@ -28,7 +28,7 @@ type ScanBroadcaster struct {
|
||||
db *sql.DB
|
||||
holdDID string
|
||||
holdEndpoint string
|
||||
driver storagedriver.StorageDriver
|
||||
s3 *s3.S3Service
|
||||
pds *HoldPDS
|
||||
ackTimeout time.Duration
|
||||
secret string // Shared secret for scanner authentication
|
||||
@@ -80,7 +80,7 @@ type VulnerabilitySummary struct {
|
||||
|
||||
// NewScanBroadcaster creates a new scan job broadcaster
|
||||
// dbPath should point to a SQLite database file (e.g., "/path/to/pds/db.sqlite3")
|
||||
func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver storagedriver.StorageDriver, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
|
||||
func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, s3svc *s3.S3Service, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
|
||||
dsn := dbPath
|
||||
if dbPath != ":memory:" && !strings.HasPrefix(dbPath, "file:") {
|
||||
dsn = "file:" + dbPath
|
||||
@@ -99,7 +99,7 @@ func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver sto
|
||||
db: db,
|
||||
holdDID: holdDID,
|
||||
holdEndpoint: holdEndpoint,
|
||||
driver: driver,
|
||||
s3: s3svc,
|
||||
pds: holdPDS,
|
||||
ackTimeout: 5 * time.Minute,
|
||||
secret: secret,
|
||||
@@ -119,13 +119,13 @@ func NewScanBroadcaster(holdDID, holdEndpoint, secret, dbPath string, driver sto
|
||||
|
||||
// NewScanBroadcasterWithDB creates a scan job broadcaster using an existing *sql.DB connection.
|
||||
// The caller is responsible for the DB lifecycle.
|
||||
func NewScanBroadcasterWithDB(holdDID, holdEndpoint, secret string, db *sql.DB, driver storagedriver.StorageDriver, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
|
||||
func NewScanBroadcasterWithDB(holdDID, holdEndpoint, secret string, db *sql.DB, s3svc *s3.S3Service, holdPDS *HoldPDS) (*ScanBroadcaster, error) {
|
||||
sb := &ScanBroadcaster{
|
||||
subscribers: make([]*ScanSubscriber, 0),
|
||||
db: db,
|
||||
holdDID: holdDID,
|
||||
holdEndpoint: holdEndpoint,
|
||||
driver: driver,
|
||||
s3: s3svc,
|
||||
pds: holdPDS,
|
||||
ackTimeout: 5 * time.Minute,
|
||||
secret: secret,
|
||||
@@ -424,7 +424,7 @@ func (sb *ScanBroadcaster) handleResult(sub *ScanSubscriber, msg ScannerMessage)
|
||||
// Upload SBOM as a blob to the hold's PDS blob storage (like manifest blobs)
|
||||
var sbomBlob *lexutil.LexBlob
|
||||
if msg.SBOM != "" {
|
||||
blob, err := uploadBlobToStorage(ctx, sb.driver, sb.holdDID, []byte(msg.SBOM), "application/spdx+json")
|
||||
blob, err := uploadBlobToStorage(ctx, sb.s3, sb.holdDID, []byte(msg.SBOM), "application/spdx+json")
|
||||
if err != nil {
|
||||
slog.Error("Failed to upload SBOM blob to PDS storage",
|
||||
"seq", msg.Seq,
|
||||
@@ -434,11 +434,24 @@ func (sb *ScanBroadcaster) handleResult(sub *ScanSubscriber, msg ScannerMessage)
|
||||
}
|
||||
}
|
||||
|
||||
// Upload vulnerability report as a blob (full Grype JSON with CVE details)
|
||||
var vulnReportBlob *lexutil.LexBlob
|
||||
if msg.VulnReport != "" {
|
||||
blob, err := uploadBlobToStorage(ctx, sb.s3, sb.holdDID, []byte(msg.VulnReport), "application/vnd.atcr.vulnerabilities+json")
|
||||
if err != nil {
|
||||
slog.Error("Failed to upload VulnReport blob to PDS storage",
|
||||
"seq", msg.Seq,
|
||||
"error", err)
|
||||
} else {
|
||||
vulnReportBlob = blob
|
||||
}
|
||||
}
|
||||
|
||||
// Store scan result as a record in the hold's embedded PDS
|
||||
if msg.Summary != nil {
|
||||
scanRecord := atproto.NewScanRecord(
|
||||
manifestDigest, repository, userDID,
|
||||
sbomBlob,
|
||||
sbomBlob, vulnReportBlob,
|
||||
msg.Summary.Critical, msg.Summary.High, msg.Summary.Medium, msg.Summary.Low, msg.Summary.Total,
|
||||
"atcr-scanner-v1.0.0",
|
||||
)
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"atcr.io/pkg/atproto"
|
||||
"atcr.io/pkg/auth/oauth"
|
||||
holddb "atcr.io/pkg/hold/db"
|
||||
"atcr.io/pkg/s3"
|
||||
"github.com/bluesky-social/indigo/atproto/atcrypto"
|
||||
lexutil "github.com/bluesky-social/indigo/lex/util"
|
||||
"github.com/bluesky-social/indigo/models"
|
||||
"github.com/bluesky-social/indigo/repo"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
@@ -231,7 +231,7 @@ func (p *HoldPDS) GetRecordBytes(ctx context.Context, recordPath string) (cid.Ci
|
||||
}
|
||||
|
||||
// Bootstrap initializes the hold with the captain record, owner as first crew member, and profile
|
||||
func (p *HoldPDS) Bootstrap(ctx context.Context, storageDriver driver.StorageDriver, ownerDID string, public bool, allowAllCrew bool, avatarURL, region string) error {
|
||||
func (p *HoldPDS) Bootstrap(ctx context.Context, s3svc *s3.S3Service, ownerDID string, public bool, allowAllCrew bool, avatarURL, region string) error {
|
||||
if ownerDID == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -317,15 +317,15 @@ func (p *HoldPDS) Bootstrap(ctx context.Context, storageDriver driver.StorageDri
|
||||
|
||||
// Create Bluesky profile record (idempotent - check if exists first)
|
||||
// This runs even if captain exists (for existing holds being upgraded)
|
||||
// Skip if no storage driver (e.g., in tests)
|
||||
if storageDriver != nil {
|
||||
// Skip if no S3 service (e.g., in tests)
|
||||
if s3svc != nil {
|
||||
_, _, err = p.GetProfileRecord(ctx)
|
||||
if err != nil {
|
||||
// Bluesky profile doesn't exist, create it
|
||||
displayName := "Cargo Hold"
|
||||
description := "ahoy from the cargo hold"
|
||||
|
||||
_, err = p.CreateProfileRecord(ctx, storageDriver, displayName, description, avatarURL)
|
||||
_, err = p.CreateProfileRecord(ctx, s3svc, displayName, description, avatarURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create bluesky profile record: %w", err)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestStatusPost(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create handler for XRPC endpoints
|
||||
handler := NewXRPCHandler(holdPDS, s3.S3Service{}, nil, nil, &mockPDSClient{}, nil)
|
||||
handler := NewXRPCHandler(holdPDS, s3.S3Service{}, nil, &mockPDSClient{}, nil)
|
||||
|
||||
// Helper function to list posts via XRPC
|
||||
listPosts := func() ([]map[string]any, error) {
|
||||
@@ -283,7 +283,7 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
|
||||
// Create shared handler
|
||||
sharedHandler = NewXRPCHandler(sharedPDS, s3.S3Service{}, nil, nil, &mockPDSClient{}, nil)
|
||||
sharedHandler = NewXRPCHandler(sharedPDS, s3.S3Service{}, nil, &mockPDSClient{}, nil)
|
||||
|
||||
// Run tests
|
||||
code := m.Run()
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/bluesky-social/indigo/api/bsky"
|
||||
lexutil "github.com/bluesky-social/indigo/lex/util"
|
||||
"github.com/bluesky-social/indigo/repo"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/render"
|
||||
"github.com/gorilla/websocket"
|
||||
@@ -46,7 +45,6 @@ const (
|
||||
type XRPCHandler struct {
|
||||
pds *HoldPDS
|
||||
s3Service s3.S3Service
|
||||
storageDriver driver.StorageDriver
|
||||
broadcaster *EventBroadcaster
|
||||
scanBroadcaster *ScanBroadcaster // Scan job dispatcher for connected scanners
|
||||
httpClient HTTPClient // For testing - allows injecting mock HTTP client
|
||||
@@ -68,14 +66,13 @@ type PartUploadInfo struct {
|
||||
}
|
||||
|
||||
// NewXRPCHandler creates a new XRPC handler
|
||||
func NewXRPCHandler(pds *HoldPDS, s3Service s3.S3Service, storageDriver driver.StorageDriver, broadcaster *EventBroadcaster, httpClient HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
|
||||
func NewXRPCHandler(pds *HoldPDS, s3Service s3.S3Service, broadcaster *EventBroadcaster, httpClient HTTPClient, quotaMgr *quota.Manager) *XRPCHandler {
|
||||
return &XRPCHandler{
|
||||
pds: pds,
|
||||
s3Service: s3Service,
|
||||
storageDriver: storageDriver,
|
||||
broadcaster: broadcaster,
|
||||
httpClient: httpClient,
|
||||
quotaMgr: quotaMgr,
|
||||
pds: pds,
|
||||
s3Service: s3Service,
|
||||
broadcaster: broadcaster,
|
||||
httpClient: httpClient,
|
||||
quotaMgr: quotaMgr,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1052,32 +1049,11 @@ func (h *XRPCHandler) HandleUploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||
// ATProto uses CIDv1 with raw codec for blobs
|
||||
blobCID := cid.NewCidV1(0x55, mh)
|
||||
|
||||
// Store blob via distribution driver at ATProto path
|
||||
// Store blob via S3 at ATProto path
|
||||
path := atprotoBlobPath(did, blobCID.String())
|
||||
|
||||
// Write blob to storage using distribution driver
|
||||
writer, err := h.storageDriver.Writer(r.Context(), path, false)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to create writer: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Write data
|
||||
n, err := io.Copy(writer, bytes.NewReader(blobData))
|
||||
if err != nil {
|
||||
writer.Cancel(r.Context())
|
||||
http.Error(w, fmt.Sprintf("failed to write blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Commit the write
|
||||
if err := writer.Commit(r.Context()); err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to commit blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if n != size {
|
||||
http.Error(w, fmt.Sprintf("size mismatch: wrote %d bytes, expected %d", n, size), http.StatusInternalServerError)
|
||||
if err := h.s3Service.PutBytes(r.Context(), path, blobData, "application/octet-stream"); err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to put blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1259,7 +1235,7 @@ func (h *XRPCHandler) HandleListBlobs(w http.ResponseWriter, r *http.Request) {
|
||||
safeDID := strings.ReplaceAll(did, ":", "-")
|
||||
blobsPath := fmt.Sprintf("/repos/%s/blobs", safeDID)
|
||||
|
||||
entries, err := h.storageDriver.List(r.Context(), blobsPath)
|
||||
entries, err := h.s3Service.ListPrefix(r.Context(), blobsPath)
|
||||
if err != nil {
|
||||
// Path doesn't exist = no blobs, return empty list
|
||||
render.JSON(w, r, map[string]any{"cids": []string{}})
|
||||
|
||||
@@ -18,8 +18,6 @@ import (
|
||||
"atcr.io/pkg/s3"
|
||||
indigoAtproto "github.com/bluesky-social/indigo/api/atproto"
|
||||
"github.com/bluesky-social/indigo/events"
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/ipfs/go-cid"
|
||||
@@ -76,7 +74,7 @@ func setupTestXRPCHandler(t *testing.T) (*XRPCHandler, context.Context) {
|
||||
mockS3 := s3.S3Service{}
|
||||
|
||||
// Create XRPC handler with mock HTTP client
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
|
||||
|
||||
return handler, ctx
|
||||
}
|
||||
@@ -143,7 +141,7 @@ func setupTestXRPCHandlerWithIndex(t *testing.T) (*XRPCHandler, context.Context)
|
||||
mockS3 := s3.S3Service{}
|
||||
|
||||
// Create XRPC handler with mock HTTP client
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
|
||||
|
||||
return handler, ctx
|
||||
}
|
||||
@@ -753,7 +751,7 @@ func TestHandleListRecords_EmptyCollection(t *testing.T) {
|
||||
pds, ctx := setupTestPDS(t) // Don't bootstrap - no records created yet
|
||||
mockClient := &mockPDSClient{}
|
||||
mockS3 := s3.S3Service{}
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
|
||||
|
||||
// Initialize repo manually (setupTestPDS doesn't call Bootstrap, so no crew members)
|
||||
err := pds.repomgr.InitNewActor(ctx, pds.uid, "", pds.did, "", "", "")
|
||||
@@ -1231,7 +1229,7 @@ func TestHandleListRepos_EmptyRepo(t *testing.T) {
|
||||
pds, ctx := setupTestPDS(t) // Don't bootstrap
|
||||
mockClient := &mockPDSClient{}
|
||||
mockS3 := s3.S3Service{}
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
|
||||
|
||||
// setupTestPDS creates the PDS/database but doesn't initialize the repo
|
||||
// Check if implementation returns repos before initialization
|
||||
@@ -1317,7 +1315,7 @@ func TestHandleGetRepoStatus_EmptyRepo(t *testing.T) {
|
||||
pds, ctx := setupTestPDS(t) // Don't bootstrap
|
||||
mockClient := &mockPDSClient{}
|
||||
mockS3 := s3.S3Service{}
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, nil, mockClient, nil)
|
||||
handler := NewXRPCHandler(pds, mockS3, nil, mockClient, nil)
|
||||
holdDID := "did:web:hold.example.com"
|
||||
|
||||
// Initialize repo but don't add any records
|
||||
@@ -1960,27 +1958,6 @@ func TestHandleAtprotoDID(t *testing.T) {
|
||||
// Mock S3 Service for testing blob endpoints
|
||||
|
||||
// mockS3Service is a simple mock that tracks calls and returns test URLs
|
||||
type mockS3Service struct {
|
||||
// Track calls
|
||||
downloadCalls []string // Track digests requested for download
|
||||
}
|
||||
|
||||
func newMockS3Service() *mockS3Service {
|
||||
return &mockS3Service{
|
||||
downloadCalls: []string{},
|
||||
}
|
||||
}
|
||||
|
||||
// toS3Service converts the mock to an s3.S3Service
|
||||
// Returns empty s3.S3Service since we're not testing S3 presigned URLs in these tests
|
||||
func (m *mockS3Service) toS3Service() s3.S3Service {
|
||||
return s3.S3Service{
|
||||
Client: nil, // Not testing presigned URLs
|
||||
Bucket: "",
|
||||
PathPrefix: "",
|
||||
}
|
||||
}
|
||||
|
||||
// setupTestXRPCHandlerWithMockS3 creates handler with MockS3Client for testing presigned URLs
|
||||
func setupTestXRPCHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Client, context.Context) {
|
||||
t.Helper()
|
||||
@@ -2029,27 +2006,17 @@ func setupTestXRPCHandlerWithMockS3(t *testing.T) (*XRPCHandler, *s3.MockS3Clien
|
||||
PathPrefix: "test-prefix",
|
||||
}
|
||||
|
||||
// Create filesystem storage driver for tests
|
||||
storageDir := filepath.Join(tmpDir, "storage")
|
||||
params := map[string]any{
|
||||
"rootdirectory": storageDir,
|
||||
}
|
||||
driver, err := factory.Create(ctx, "filesystem", params)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create storage driver: %v", err)
|
||||
}
|
||||
|
||||
// Create mock PDS client for DPoP validation
|
||||
mockClient := &mockPDSClient{}
|
||||
|
||||
// Create XRPC handler with mock S3 client and real filesystem driver
|
||||
handler := NewXRPCHandler(pds, s3Service, driver, nil, mockClient, nil)
|
||||
// Create XRPC handler with mock S3 client
|
||||
handler := NewXRPCHandler(pds, s3Service, nil, mockClient, nil)
|
||||
|
||||
return handler, mockS3Client, ctx
|
||||
}
|
||||
|
||||
// setupTestXRPCHandlerWithBlobs creates handler with mock s3 service and real filesystem driver
|
||||
func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *mockS3Service, context.Context) {
|
||||
// setupTestXRPCHandlerWithBlobs creates handler with MockS3Client for upload/list testing
|
||||
func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *s3.MockS3Client, context.Context) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -2088,26 +2055,21 @@ func setupTestXRPCHandlerWithBlobs(t *testing.T) (*XRPCHandler, *mockS3Service,
|
||||
t.Fatalf("Failed to bootstrap PDS: %v", err)
|
||||
}
|
||||
|
||||
// Create mock s3 service that returns test URLs
|
||||
mockS3Svc := newMockS3Service()
|
||||
|
||||
// Create filesystem storage driver for tests
|
||||
storageDir := filepath.Join(tmpDir, "storage")
|
||||
params := map[string]any{
|
||||
"rootdirectory": storageDir,
|
||||
}
|
||||
driver, err := factory.Create(ctx, "filesystem", params)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create storage driver: %v", err)
|
||||
// Create MockS3Client for blob upload/list
|
||||
mockS3Client := s3.NewMockS3Client("https://mock-s3.example.com")
|
||||
s3Service := s3.S3Service{
|
||||
Client: mockS3Client,
|
||||
Bucket: "test-bucket",
|
||||
PathPrefix: "",
|
||||
}
|
||||
|
||||
// Create mock PDS client for DPoP validation
|
||||
mockClient := &mockPDSClient{}
|
||||
|
||||
// Create XRPC handler with mock s3 service and real filesystem driver
|
||||
handler := NewXRPCHandler(pds, mockS3Svc.toS3Service(), driver, nil, mockClient, nil)
|
||||
// Create XRPC handler
|
||||
handler := NewXRPCHandler(pds, s3Service, nil, mockClient, nil)
|
||||
|
||||
return handler, mockS3Svc, ctx
|
||||
return handler, mockS3Client, ctx
|
||||
}
|
||||
|
||||
// Tests for HandleUploadBlob
|
||||
@@ -2391,9 +2353,9 @@ func TestHandleGetBlob(t *testing.T) {
|
||||
t.Error("Expected Location header in 307 redirect")
|
||||
}
|
||||
|
||||
// Should be XRPC proxy URL since we don't have S3 client
|
||||
if !strings.Contains(location, "/xrpc/com.atproto.sync.getBlob") {
|
||||
t.Errorf("Expected XRPC proxy URL, got: %s", location)
|
||||
// Should be a presigned URL from the mock S3 client
|
||||
if !strings.Contains(location, "mock-s3.example.com") {
|
||||
t.Errorf("Expected presigned S3 URL, got: %s", location)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2457,9 +2419,9 @@ func TestHandleGetBlob_HeadMethod(t *testing.T) {
|
||||
t.Error("Expected Location header in 307 redirect")
|
||||
}
|
||||
|
||||
// Should be XRPC proxy URL since we don't have S3 client
|
||||
if !strings.Contains(location, "/xrpc/com.atproto.sync.getBlob") {
|
||||
t.Errorf("Expected XRPC proxy URL, got: %s", location)
|
||||
// Should be a presigned URL from the mock S3 client
|
||||
if !strings.Contains(location, "mock-s3.example.com") {
|
||||
t.Errorf("Expected presigned S3 URL, got: %s", location)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,9 +21,6 @@ import (
|
||||
"atcr.io/pkg/logging"
|
||||
"atcr.io/pkg/s3"
|
||||
|
||||
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
@@ -72,6 +69,7 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
|
||||
|
||||
// Initialize embedded PDS if database path is configured
|
||||
var xrpcHandler *pds.XRPCHandler
|
||||
var s3Service *s3.S3Service
|
||||
if cfg.Database.Path != "" {
|
||||
holdDID := pds.GenerateDIDFromURL(cfg.Server.PublicURL)
|
||||
slog.Info("Initializing embedded PDS", "did", holdDID)
|
||||
@@ -109,14 +107,14 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
|
||||
s.broadcaster = pds.NewEventBroadcaster(holdDID, 100, ":memory:")
|
||||
}
|
||||
|
||||
// Create storage driver from config (needed for bootstrap profile avatar)
|
||||
driver, err := factory.Create(ctx, cfg.Storage.Type(), cfg.Storage.Parameters())
|
||||
// Create S3 service (used for bootstrap, handlers, GC, etc.)
|
||||
s3Service, err = s3.NewS3Service(cfg.Storage.S3Params())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create storage driver: %w", err)
|
||||
return nil, fmt.Errorf("failed to create S3 service: %w", err)
|
||||
}
|
||||
|
||||
// Bootstrap PDS with captain record, hold owner as first crew member, and profile
|
||||
if err := s.PDS.Bootstrap(ctx, driver, cfg.Registration.OwnerDID, cfg.Server.Public, cfg.Registration.AllowAllCrew, cfg.Registration.ProfileAvatarURL, cfg.Registration.Region); err != nil {
|
||||
if err := s.PDS.Bootstrap(ctx, s3Service, cfg.Registration.OwnerDID, cfg.Server.Public, cfg.Registration.AllowAllCrew, cfg.Registration.ProfileAvatarURL, cfg.Registration.Region); err != nil {
|
||||
return nil, fmt.Errorf("failed to bootstrap PDS: %w", err)
|
||||
}
|
||||
|
||||
@@ -163,32 +161,21 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
|
||||
slog.Info("Quota enforcement disabled (no quota tiers configured)")
|
||||
}
|
||||
|
||||
// Create blob store adapter and XRPC handlers
|
||||
// Create XRPC handlers
|
||||
var ociHandler *oci.XRPCHandler
|
||||
if s.PDS != nil {
|
||||
ctx := context.Background()
|
||||
driver, err := factory.Create(ctx, cfg.Storage.Type(), cfg.Storage.Parameters())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create storage driver: %w", err)
|
||||
}
|
||||
|
||||
s3Service, err := s3.NewS3Service(cfg.Storage.Parameters())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create S3 service: %w", err)
|
||||
}
|
||||
|
||||
xrpcHandler = pds.NewXRPCHandler(s.PDS, *s3Service, driver, s.broadcaster, nil, s.QuotaManager)
|
||||
ociHandler = oci.NewXRPCHandler(s.PDS, *s3Service, driver, cfg.Registration.EnableBlueskyPosts, nil, s.QuotaManager)
|
||||
xrpcHandler = pds.NewXRPCHandler(s.PDS, *s3Service, s.broadcaster, nil, s.QuotaManager)
|
||||
ociHandler = oci.NewXRPCHandler(s.PDS, *s3Service, cfg.Registration.EnableBlueskyPosts, nil, s.QuotaManager)
|
||||
|
||||
// Initialize scan broadcaster if scanner secret is configured
|
||||
if cfg.Scanner.Secret != "" {
|
||||
holdDID := pds.GenerateDIDFromURL(cfg.Server.PublicURL)
|
||||
var sb *pds.ScanBroadcaster
|
||||
if s.holdDB != nil {
|
||||
sb, err = pds.NewScanBroadcasterWithDB(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, s.holdDB.DB, driver, s.PDS)
|
||||
sb, err = pds.NewScanBroadcasterWithDB(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, s.holdDB.DB, s3Service, s.PDS)
|
||||
} else {
|
||||
scanDBPath := cfg.Database.Path + "/db.sqlite3"
|
||||
sb, err = pds.NewScanBroadcaster(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, scanDBPath, driver, s.PDS)
|
||||
sb, err = pds.NewScanBroadcaster(holdDID, cfg.Server.PublicURL, cfg.Scanner.Secret, scanDBPath, s3Service, s.PDS)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize scan broadcaster: %w", err)
|
||||
@@ -200,7 +187,7 @@ func NewHoldServer(cfg *Config) (*HoldServer, error) {
|
||||
}
|
||||
|
||||
// Initialize garbage collector
|
||||
s.garbageCollector = gc.NewGarbageCollector(s.PDS, driver, cfg.GC)
|
||||
s.garbageCollector = gc.NewGarbageCollector(s.PDS, s3Service, cfg.GC)
|
||||
slog.Info("Garbage collector initialized",
|
||||
"enabled", cfg.GC.Enabled)
|
||||
}
|
||||
|
||||
178
pkg/s3/mock.go
178
pkg/s3/mock.go
@@ -1,13 +1,17 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
@@ -22,6 +26,9 @@ type MockS3Client struct {
|
||||
// If empty, a UUID is generated.
|
||||
UploadID string
|
||||
|
||||
// Objects stores in-memory blobs for PutObject/HeadObject/DeleteObject/CopyObject/ListObjectsV2.
|
||||
Objects map[string][]byte
|
||||
|
||||
// Track calls for verification in tests
|
||||
mu sync.Mutex
|
||||
CreateMultipartCalls []CreateMultipartCall
|
||||
@@ -36,6 +43,8 @@ type MockS3Client struct {
|
||||
CreateMultipartError error
|
||||
CompleteError error
|
||||
AbortError error
|
||||
HeadObjectError error
|
||||
CopyObjectError error
|
||||
}
|
||||
|
||||
// CreateMultipartCall records a CreateMultipartUpload call
|
||||
@@ -89,6 +98,7 @@ type PutObjectCall struct {
|
||||
func NewMockS3Client(testServerURL string) *MockS3Client {
|
||||
return &MockS3Client{
|
||||
TestServerURL: testServerURL,
|
||||
Objects: make(map[string][]byte),
|
||||
CreateMultipartCalls: []CreateMultipartCall{},
|
||||
CompleteCalls: []CompleteCall{},
|
||||
AbortCalls: []AbortCall{},
|
||||
@@ -144,6 +154,14 @@ func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3
|
||||
return nil, m.CompleteError
|
||||
}
|
||||
|
||||
// Store a placeholder object at the key so Stat/HeadObject works after complete
|
||||
key := aws.ToString(input.Key)
|
||||
if m.Objects != nil {
|
||||
if _, exists := m.Objects[key]; !exists {
|
||||
m.Objects[key] = []byte("completed-multipart")
|
||||
}
|
||||
}
|
||||
|
||||
// Return a mock ETag
|
||||
etag := "\"mock-etag-" + uuid.New().String() + "\""
|
||||
return &awss3.CompleteMultipartUploadOutput{
|
||||
@@ -169,6 +187,141 @@ func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
|
||||
return &awss3.AbortMultipartUploadOutput{}, nil
|
||||
}
|
||||
|
||||
// HeadObject implements S3Client
|
||||
func (m *MockS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.HeadObjectError != nil {
|
||||
return nil, m.HeadObjectError
|
||||
}
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
data, ok := m.Objects[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("NoSuchKey: object %s not found", key)
|
||||
}
|
||||
|
||||
size := int64(len(data))
|
||||
return &awss3.HeadObjectOutput{
|
||||
ContentLength: &size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PutObject implements S3Client
|
||||
func (m *MockS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
m.PutObjectCalls = append(m.PutObjectCalls, PutObjectCall{
|
||||
Bucket: aws.ToString(input.Bucket),
|
||||
Key: key,
|
||||
})
|
||||
|
||||
if input.Body != nil {
|
||||
data, err := io.ReadAll(input.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Objects[key] = data
|
||||
} else {
|
||||
m.Objects[key] = []byte{}
|
||||
}
|
||||
|
||||
return &awss3.PutObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// CopyObject implements S3Client
|
||||
func (m *MockS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.CopyObjectError != nil {
|
||||
return nil, m.CopyObjectError
|
||||
}
|
||||
|
||||
// CopySource is "bucket/key"
|
||||
copySource := aws.ToString(input.CopySource)
|
||||
// Strip bucket prefix to get key
|
||||
parts := strings.SplitN(copySource, "/", 2)
|
||||
srcKey := copySource
|
||||
if len(parts) == 2 {
|
||||
srcKey = parts[1]
|
||||
}
|
||||
|
||||
data, ok := m.Objects[srcKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("NoSuchKey: source object %s not found", srcKey)
|
||||
}
|
||||
|
||||
dstKey := aws.ToString(input.Key)
|
||||
m.Objects[dstKey] = append([]byte{}, data...)
|
||||
|
||||
return &awss3.CopyObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// DeleteObject implements S3Client
|
||||
func (m *MockS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
delete(m.Objects, key)
|
||||
|
||||
return &awss3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 implements S3Client
|
||||
func (m *MockS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
prefix := aws.ToString(input.Prefix)
|
||||
delimiter := aws.ToString(input.Delimiter)
|
||||
|
||||
var contents []s3types.Object
|
||||
commonPrefixes := map[string]bool{}
|
||||
|
||||
for key, data := range m.Objects {
|
||||
if !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if delimiter != "" {
|
||||
// Check if there's a delimiter after the prefix
|
||||
rest := strings.TrimPrefix(key, prefix)
|
||||
idx := strings.Index(rest, delimiter)
|
||||
if idx >= 0 {
|
||||
// Has delimiter — this is a common prefix, not a content object
|
||||
cp := prefix + rest[:idx+len(delimiter)]
|
||||
commonPrefixes[cp] = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
size := int64(len(data))
|
||||
k := key
|
||||
contents = append(contents, s3types.Object{
|
||||
Key: &k,
|
||||
Size: &size,
|
||||
})
|
||||
}
|
||||
|
||||
var cps []s3types.CommonPrefix
|
||||
for cp := range commonPrefixes {
|
||||
p := cp
|
||||
cps = append(cps, s3types.CommonPrefix{Prefix: &p})
|
||||
}
|
||||
|
||||
falseVal := false
|
||||
return &awss3.ListObjectsV2Output{
|
||||
Contents: contents,
|
||||
CommonPrefixes: cps,
|
||||
IsTruncated: &falseVal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PresignUploadPart implements S3Client
|
||||
// Returns a mock presigned URL for test server
|
||||
func (m *MockS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) {
|
||||
@@ -229,6 +382,31 @@ func (m *MockS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObj
|
||||
Key: aws.ToString(input.Key),
|
||||
})
|
||||
|
||||
// Also store the body if provided (for PresignPutObject used in tests that also check objects)
|
||||
if input.Body != nil {
|
||||
key := aws.ToString(input.Key)
|
||||
data, _ := io.ReadAll(input.Body)
|
||||
m.Objects[key] = data
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/put/%s", m.TestServerURL, aws.ToString(input.Key))
|
||||
return url, nil
|
||||
}
|
||||
|
||||
// SetObject is a test helper to pre-populate an object in the mock store.
|
||||
func (m *MockS3Client) SetObject(key string, data []byte) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Objects[key] = append([]byte{}, data...)
|
||||
}
|
||||
|
||||
// GetObject is a test helper to read an object from the mock store (nil if not found).
|
||||
func (m *MockS3Client) GetObject(key string) []byte {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
data, ok := m.Objects[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return bytes.Clone(data)
|
||||
}
|
||||
|
||||
233
pkg/s3/types.go
233
pkg/s3/types.go
@@ -3,9 +3,11 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -24,6 +26,13 @@ type S3Client interface {
|
||||
CompleteMultipartUpload(ctx context.Context, input *awss3.CompleteMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error)
|
||||
|
||||
// Direct object operations
|
||||
HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
|
||||
PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
|
||||
CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error)
|
||||
DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
|
||||
ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
|
||||
|
||||
// Presigned URL operations - return URL string directly
|
||||
PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error)
|
||||
PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error)
|
||||
@@ -61,31 +70,49 @@ func (r *RealS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
|
||||
return r.client.AbortMultipartUpload(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// HeadObject implements S3Client
|
||||
func (r *RealS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
|
||||
return r.client.HeadObject(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// PutObject implements S3Client
|
||||
func (r *RealS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
|
||||
return r.client.PutObject(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// CopyObject implements S3Client
|
||||
func (r *RealS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
|
||||
return r.client.CopyObject(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// DeleteObject implements S3Client
|
||||
func (r *RealS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
|
||||
return r.client.DeleteObject(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// ListObjectsV2 implements S3Client
|
||||
func (r *RealS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
|
||||
return r.client.ListObjectsV2(ctx, input, opts...)
|
||||
}
|
||||
|
||||
// PresignGetObject implements S3Client
|
||||
func (r *RealS3Client) PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error) {
|
||||
result, err := r.presign.PresignGetObject(ctx, input, func(opts *awss3.PresignOptions) {
|
||||
opts.Expires = expires
|
||||
if r.pullZone != "" {
|
||||
opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
|
||||
o.BaseEndpoint = aws.String(r.pullZone)
|
||||
})
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return result.URL, nil
|
||||
return r.applyPullZone(result.URL), nil
|
||||
}
|
||||
|
||||
// PresignHeadObject implements S3Client
|
||||
// Note: pull zone is intentionally NOT applied to HEAD requests. CDNs like
|
||||
// Bunny may convert HEAD to GET when proxying, which breaks the SigV4
|
||||
// signature. HEAD responses have no body, so CDN caching provides no benefit.
|
||||
func (r *RealS3Client) PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error) {
|
||||
result, err := r.presign.PresignHeadObject(ctx, input, func(opts *awss3.PresignOptions) {
|
||||
opts.Expires = expires
|
||||
if r.pullZone != "" {
|
||||
opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
|
||||
o.BaseEndpoint = aws.String(r.pullZone)
|
||||
})
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -193,6 +220,190 @@ func NewS3Service(params map[string]any) (*S3Service, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// s3Key converts a blob path (with leading /) to an S3 key with prefix.
|
||||
func (s *S3Service) s3Key(blobPath string) string {
|
||||
key := strings.TrimPrefix(blobPath, "/")
|
||||
if s.PathPrefix != "" {
|
||||
key = s.PathPrefix + "/" + key
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// Stat returns the size of an object at blobPath, or an error if it doesn't exist.
|
||||
func (s *S3Service) Stat(ctx context.Context, blobPath string) (int64, error) {
|
||||
key := s.s3Key(blobPath)
|
||||
out, err := s.Client.HeadObject(ctx, &awss3.HeadObjectInput{
|
||||
Bucket: &s.Bucket,
|
||||
Key: &key,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if out.ContentLength != nil {
|
||||
return *out.ContentLength, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// PutBytes uploads data to blobPath with the given content type.
|
||||
func (s *S3Service) PutBytes(ctx context.Context, blobPath string, data []byte, contentType string) error {
|
||||
key := s.s3Key(blobPath)
|
||||
_, err := s.Client.PutObject(ctx, &awss3.PutObjectInput{
|
||||
Bucket: &s.Bucket,
|
||||
Key: &key,
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: &contentType,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Move copies srcPath to dstPath then deletes srcPath.
|
||||
func (s *S3Service) Move(ctx context.Context, srcPath, dstPath string) error {
|
||||
srcKey := s.s3Key(srcPath)
|
||||
dstKey := s.s3Key(dstPath)
|
||||
copySource := s.Bucket + "/" + srcKey
|
||||
|
||||
_, err := s.Client.CopyObject(ctx, &awss3.CopyObjectInput{
|
||||
Bucket: &s.Bucket,
|
||||
Key: &dstKey,
|
||||
CopySource: ©Source,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy %s -> %s: %w", srcPath, dstPath, err)
|
||||
}
|
||||
|
||||
_, err = s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
|
||||
Bucket: &s.Bucket,
|
||||
Key: &srcKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete source %s after copy: %w", srcPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes the object at blobPath.
|
||||
func (s *S3Service) Delete(ctx context.Context, blobPath string) error {
|
||||
key := s.s3Key(blobPath)
|
||||
_, err := s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
|
||||
Bucket: &s.Bucket,
|
||||
Key: &key,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// WalkBlobs paginates ListObjectsV2 under prefix and calls fn for each object.
|
||||
// Keys passed to fn have the PathPrefix stripped (same format as BlobPath output).
|
||||
func (s *S3Service) WalkBlobs(ctx context.Context, prefix string, fn func(key string, size int64) error) error {
|
||||
s3Prefix := s.s3Key(prefix)
|
||||
if !strings.HasSuffix(s3Prefix, "/") {
|
||||
s3Prefix += "/"
|
||||
}
|
||||
|
||||
var continuationToken *string
|
||||
for {
|
||||
out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
|
||||
Bucket: &s.Bucket,
|
||||
Prefix: &s3Prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list objects under %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
for _, obj := range out.Contents {
|
||||
if obj.Key == nil {
|
||||
continue
|
||||
}
|
||||
// Strip PathPrefix to get the logical path
|
||||
key := *obj.Key
|
||||
if s.PathPrefix != "" {
|
||||
key = strings.TrimPrefix(key, s.PathPrefix+"/")
|
||||
}
|
||||
// Restore leading / for consistency with BlobPath
|
||||
key = "/" + key
|
||||
|
||||
var size int64
|
||||
if obj.Size != nil {
|
||||
size = *obj.Size
|
||||
}
|
||||
if err := fn(key, size); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if out.IsTruncated == nil || !*out.IsTruncated {
|
||||
break
|
||||
}
|
||||
continuationToken = out.NextContinuationToken
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListPrefix returns immediate children (common prefixes) under blobPath using Delimiter="/".
|
||||
func (s *S3Service) ListPrefix(ctx context.Context, blobPath string) ([]string, error) {
|
||||
s3Prefix := s.s3Key(blobPath)
|
||||
if !strings.HasSuffix(s3Prefix, "/") {
|
||||
s3Prefix += "/"
|
||||
}
|
||||
delimiter := "/"
|
||||
|
||||
var results []string
|
||||
var continuationToken *string
|
||||
for {
|
||||
out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
|
||||
Bucket: &s.Bucket,
|
||||
Prefix: &s3Prefix,
|
||||
Delimiter: &delimiter,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list prefix %s: %w", blobPath, err)
|
||||
}
|
||||
|
||||
for _, cp := range out.CommonPrefixes {
|
||||
if cp.Prefix == nil {
|
||||
continue
|
||||
}
|
||||
// Strip the PathPrefix and reconstruct logical path
|
||||
p := *cp.Prefix
|
||||
if s.PathPrefix != "" {
|
||||
p = strings.TrimPrefix(p, s.PathPrefix+"/")
|
||||
}
|
||||
p = "/" + strings.TrimSuffix(p, "/")
|
||||
results = append(results, p)
|
||||
}
|
||||
|
||||
if out.IsTruncated == nil || !*out.IsTruncated {
|
||||
break
|
||||
}
|
||||
continuationToken = out.NextContinuationToken
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// applyPullZone replaces the host in a presigned URL with the pull zone host.
|
||||
// The signature is computed against the real S3 endpoint so that the origin
|
||||
// can validate it; the CDN just proxies the request transparently.
|
||||
func (r *RealS3Client) applyPullZone(presignedURL string) string {
|
||||
if r.pullZone == "" {
|
||||
return presignedURL
|
||||
}
|
||||
parsed, err := url.Parse(presignedURL)
|
||||
if err != nil {
|
||||
return presignedURL
|
||||
}
|
||||
pz, err := url.Parse(r.pullZone)
|
||||
if err != nil {
|
||||
return presignedURL
|
||||
}
|
||||
parsed.Scheme = pz.Scheme
|
||||
parsed.Host = pz.Host
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
// BlobPath converts a digest (e.g., "sha256:abc123...") or temp path to a storage path
|
||||
// Distribution stores blobs as: /docker/registry/v2/blobs/{algorithm}/{xx}/{hash}/data
|
||||
// where xx is the first 2 characters of the hash for directory sharding
|
||||
|
||||
Reference in New Issue
Block a user