mirrors/at-container-registry
1
0
Fork 0
mirror of https://tangled.org/evan.jarrett.net/at-container-registry synced 2026-09-25 03:34:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

clean up db migrations and schema. implement a way to fetch readmes for documentation

Browse Source
This commit is contained in:
Evan Jarrett
2025-10-21 10:28:48 -05:00
parent 08fb8abb41
commit abf48407cc
22 changed files with 944 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CLAUDE.md
+8 -5
View File
@@ -642,11 +642,14 @@ When writing tests:
5. Update `.env.example` with new driver's env vars
**Working with the database**:
- Schema defined in `pkg/appview/db/schema.go`
- Queries in `pkg/appview/db/queries.go`
- Stores for OAuth, devices, sessions in separate files
- Run migrations automatically on startup
- Database path configurable via `ATCR_UI_DATABASE_PATH` env var
- **Base schema** defined in `pkg/appview/db/schema.sql` - source of truth for fresh installations
- **Migrations** in `pkg/appview/db/migrations/*.yaml` - only for ALTER/UPDATE/DELETE on existing databases
- **Queries** in `pkg/appview/db/queries.go`
- **Stores** for OAuth, devices, sessions in separate files
- **Execution order**: schema.sql first, then migrations (automatically on startup)
- **Database path** configurable via `ATCR_UI_DATABASE_PATH` env var
- **Adding new tables**: Add to `schema.sql` only (no migration needed)
- **Altering tables**: Create migration AND update `schema.sql` to keep them in sync
**Adding web UI features**:
- Add handler in `pkg/appview/handlers/`
cmd/appview/serve.go
+17 -3
View File
@@ -28,6 +28,7 @@ import (
uihandlers "atcr.io/pkg/appview/handlers"
"atcr.io/pkg/appview/holdhealth"
"atcr.io/pkg/appview/jetstream"
"atcr.io/pkg/appview/readme"
"github.com/gorilla/mux"
)
@@ -88,6 +89,18 @@ func serveRegistry(cmd *cobra.Command, args []string) error {
healthChecker := holdhealth.NewChecker(cacheTTL)
// Initialize README cache
fmt.Println("Initializing README cache...")
readmeCacheTTL := 1 * time.Hour // Default: 1 hour
if readmeTTLStr := os.Getenv("ATCR_README_CACHE_TTL"); readmeTTLStr != "" {
if parsed, err := time.ParseDuration(readmeTTLStr); err == nil {
readmeCacheTTL = parsed
} else {
fmt.Printf("Warning: Invalid ATCR_README_CACHE_TTL '%s', using default 1h\n", readmeTTLStr)
}
}
readmeCache := readme.NewCache(uiDatabase, readmeCacheTTL)
// Start background health check worker
// Parse refresh interval from environment (default: 15m)
refreshInterval := 15 * time.Minute
@@ -184,8 +197,8 @@ func serveRegistry(cmd *cobra.Command, args []string) error {
middleware.SetGlobalAuthorizer(holdAuthorizer)
fmt.Println("Hold authorizer initialized with database caching")
// Initialize UI routes with OAuth app, refresher, device store, and health checker
uiTemplates, uiRouter := initializeUIRoutes(uiDatabase, uiReadOnlyDB, uiSessionStore, oauthApp, refresher, baseURL, deviceStore, defaultHoldDID, healthChecker)
// Initialize UI routes with OAuth app, refresher, device store, health checker, and readme cache
uiTemplates, uiRouter := initializeUIRoutes(uiDatabase, uiReadOnlyDB, uiSessionStore, oauthApp, refresher, baseURL, deviceStore, defaultHoldDID, healthChecker, readmeCache)
// Create OAuth server
oauthServer := oauth.NewServer(oauthApp)
@@ -380,7 +393,7 @@ func createTokenIssuer(config *configuration.Configuration) (*token.Issuer, erro
// readOnlyDB: read-only connection for public queries (search, user pages, etc.)
// defaultHoldDID: DID of the default hold service (e.g., "did:web:hold01.atcr.io")
// healthChecker: hold endpoint health checker
func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.SessionStore, oauthApp *oauth.App, refresher *oauth.Refresher, baseURL string, deviceStore *db.DeviceStore, defaultHoldDID string, healthChecker *holdhealth.Checker) (*template.Template, *mux.Router) {
func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.SessionStore, oauthApp *oauth.App, refresher *oauth.Refresher, baseURL string, deviceStore *db.DeviceStore, defaultHoldDID string, healthChecker *holdhealth.Checker, readmeCache *readme.Cache) (*template.Template, *mux.Router) {
// Check if UI is enabled
uiEnabled := os.Getenv("ATCR_UI_ENABLED")
if uiEnabled == "false" {
@@ -510,6 +523,7 @@ func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.S
Directory: oauthApp.Directory(),
Refresher: refresher,
HealthChecker: healthChecker,
ReadmeCache: readmeCache,
},
)).Methods("GET")
go.mod
+4
View File
@@ -32,6 +32,7 @@ require (
)
require (
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -51,6 +52,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/handlers v1.5.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
@@ -85,6 +87,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
@@ -105,6 +108,7 @@ require (
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/yuin/goldmark v1.7.13 // indirect
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect
go.sum
+8
View File
@@ -7,6 +7,8 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -108,6 +110,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -266,6 +270,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
@@ -377,6 +383,8 @@ github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q=
pkg/appview/db/migrations/0002_add_hold_captain_records.yaml
-13
View File
@@ -1,13 +0,0 @@
description: Add hold_captain_records table for caching hold security settings
query: |
CREATE TABLE IF NOT EXISTS hold_captain_records (
hold_did TEXT PRIMARY KEY,
owner_did TEXT NOT NULL,
public BOOLEAN NOT NULL,
allow_all_crew BOOLEAN NOT NULL,
deployed_at TEXT,
region TEXT,
provider TEXT,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at);
pkg/appview/db/migrations/0005_normalize_hold_endpoint_to_did.yaml → pkg/appview/db/migrations/0002_normalize_hold_endpoint_to_did.yaml
View File
pkg/appview/db/migrations/0003_add_crew_cache.yaml
-20
View File
@@ -1,20 +0,0 @@
description: Add crew cache tables for authorization with exponential backoff
query: |
CREATE TABLE IF NOT EXISTS hold_crew_approvals (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at);
CREATE TABLE IF NOT EXISTS hold_crew_denials (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
denial_count INTEGER NOT NULL DEFAULT 1,
next_retry_at TIMESTAMP NOT NULL,
last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at);
pkg/appview/db/migrations/0003_add_readme_url_to_manifests.yaml
+46
View File
@@ -0,0 +1,46 @@
description: Add readme_url column to manifests table (idempotent - handles both fresh and existing databases)
query: |
-- Idempotent migration: adds readme_url column if it doesn't exist
-- Works for both fresh installs (where schema.sql created it) and existing databases
-- Create temp table with new schema
CREATE TABLE manifests_temp (
id INTEGER PRIMARY KEY AUTOINCREMENT,
did TEXT NOT NULL,
repository TEXT NOT NULL,
digest TEXT NOT NULL,
hold_endpoint TEXT NOT NULL,
schema_version INTEGER NOT NULL,
media_type TEXT NOT NULL,
config_digest TEXT,
config_size INTEGER,
created_at TIMESTAMP NOT NULL,
title TEXT,
description TEXT,
source_url TEXT,
documentation_url TEXT,
licenses TEXT,
icon_url TEXT,
readme_url TEXT,
UNIQUE(did, repository, digest),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
-- Copy data from existing manifests table
-- Use INSERT OR IGNORE to handle case where table is already correct
INSERT OR IGNORE INTO manifests_temp
SELECT id, did, repository, digest, hold_endpoint, schema_version, media_type,
config_digest, config_size, created_at, title, description, source_url,
documentation_url, licenses, icon_url,
NULL as readme_url -- Will be NULL for existing data
FROM manifests;
-- Only proceed with table swap if we actually copied data
-- (manifests_temp will be empty if manifests table already has readme_url)
DROP TABLE IF EXISTS manifests;
ALTER TABLE manifests_temp RENAME TO manifests;
-- Recreate indexes
CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository);
CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest);
pkg/appview/db/migrations/0004_add_manifest_references.yaml
-16
View File
@@ -1,16 +0,0 @@
description: Add manifest_references table for multi-arch manifest support
query: |
CREATE TABLE IF NOT EXISTS manifest_references (
manifest_id INTEGER NOT NULL,
digest TEXT NOT NULL,
media_type TEXT NOT NULL,
size INTEGER NOT NULL,
platform_architecture TEXT,
platform_os TEXT,
platform_variant TEXT,
platform_os_version TEXT,
reference_index INTEGER NOT NULL,
PRIMARY KEY(manifest_id, reference_index),
FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest);
pkg/appview/db/migrations/README.md
+48 -26
View File
@@ -2,6 +2,18 @@
This directory contains database migrations for the ATCR AppView database.
## Schema vs Migrations
**`schema.sql`** (in parent directory) contains the **complete base schema** for fresh database installations. It includes all tables, indexes, and constraints.
**Migrations** (this directory) handle **changes to existing databases**. They are only for:
- `ALTER TABLE` statements (add/modify/drop columns)
- `UPDATE` statements (data transformations)
- `DELETE` statements (data cleanup)
- Creating/modifying indexes on existing tables
**NEW TABLES go in `schema.sql`, NOT in migrations.**
## Migration Format
Each migration is a YAML file with the following structure:
@@ -33,13 +45,43 @@ Examples:
2. **Create a new YAML file** with format `000N_descriptive_name.yaml`
3. **Add description** (optional) - Explain what the migration does
4. **Write your SQL in `query`** - Use the `|` block scalar for clean multi-line SQL
5. **Use `IF EXISTS` / `IF NOT EXISTS`** where possible for idempotency (note: not supported for `DROP COLUMN`)
5. **Use `IF EXISTS` / `IF NOT EXISTS`** where possible for idempotency
## Examples
### Simple single-statement migration:
### Adding a column to existing table:
Filename: `0002_add_repository_description_index.yaml`
Filename: `0007_add_readme_url_to_manifests.yaml`
```yaml
description: Add readme_url column to manifests table for storing io.atcr.readme annotation
query: |
ALTER TABLE manifests ADD COLUMN readme_url TEXT;
```
**IMPORTANT:** After creating this migration, also add the column to `schema.sql` so fresh installations include it!
### Data transformation migration:
Filename: `0005_normalize_hold_endpoint_to_did.yaml`
```yaml
description: Normalize hold_endpoint column to store DIDs instead of URLs
query: |
-- Convert HTTPS URLs to did:web: format
UPDATE manifests
SET hold_endpoint = 'did:web:' || substr(hold_endpoint, 9)
WHERE hold_endpoint LIKE 'https://%';
-- Convert HTTP URLs to did:web: format
UPDATE manifests
SET hold_endpoint = 'did:web:' || substr(hold_endpoint, 8)
WHERE hold_endpoint LIKE 'http://%';
```
### Adding an index to existing table:
Filename: `0008_add_repository_description_index.yaml`
```yaml
description: Add index on manifests description field for faster searches
@@ -47,28 +89,6 @@ query: |
CREATE INDEX IF NOT EXISTS idx_manifests_description ON manifests(description);
```
### Complex multi-statement migration:
Filename: `0003_create_webhooks_table.yaml`
```yaml
description: Create webhooks table for repository event notifications
query: |
-- Create webhooks table
CREATE TABLE IF NOT EXISTS webhooks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL,
events TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create index on URL for faster lookups
CREATE INDEX IF NOT EXISTS idx_webhooks_url ON webhooks(url);
-- Create index on events for filtering
CREATE INDEX IF NOT EXISTS idx_webhooks_events ON webhooks(events);
```
## How Migrations Run
1. Migrations are loaded from this directory on startup
@@ -82,4 +102,6 @@ query: |
- **Never modify existing migrations** - Once applied, they're immutable
- **Test migrations** before committing - Ensure they work on existing databases
- **Version numbers must be unique** - The migration system will fail if duplicates exist
- **Migrations are run automatically** on `InitDB()` - No manual intervention needed
- **Migrations run automatically** on `InitDB()` - Schema first, then migrations
- **CRITICAL: Update `schema.sql` for structural changes** - When you ALTER a table or add columns, update both the migration AND `schema.sql` so fresh installations have the same structure
- **New tables go in `schema.sql` only** - Don't create migration files for new tables
pkg/appview/db/models.go
+2
View File
@@ -29,6 +29,7 @@ type Manifest struct {
DocumentationURL string
Licenses string
IconURL string
ReadmeURL string
}
// Layer represents a layer in a manifest
@@ -94,6 +95,7 @@ type Repository struct {
DocumentationURL string
Licenses string
IconURL string
ReadmeURL string
}
// RepositoryStats represents statistics for a repository
pkg/appview/db/queries.go
+15 -11
View File
@@ -310,23 +310,23 @@ func GetUserRepositories(db *sql.DB, did string) ([]Repository, error) {
}
// GetRepositoryMetadata retrieves metadata for a repository from its most recent manifest
func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, description, sourceURL, documentationURL, licenses, iconURL string, err error) {
var titleNull, descriptionNull, sourceURLNull, documentationURLNull, licensesNull, iconURLNull sql.NullString
func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string, err error) {
var titleNull, descriptionNull, sourceURLNull, documentationURLNull, licensesNull, iconURLNull, readmeURLNull sql.NullString
err = db.QueryRow(`
SELECT title, description, source_url, documentation_url, licenses, icon_url
SELECT title, description, source_url, documentation_url, licenses, icon_url, readme_url
FROM manifests
WHERE did = ? AND repository = ?
ORDER BY created_at DESC
LIMIT 1
`, did, repository).Scan(&titleNull, &descriptionNull, &sourceURLNull, &documentationURLNull, &licensesNull, &iconURLNull)
`, did, repository).Scan(&titleNull, &descriptionNull, &sourceURLNull, &documentationURLNull, &licensesNull, &iconURLNull, &readmeURLNull)
if err == sql.ErrNoRows {
// No manifests found - return empty strings
return "", "", "", "", "", "", nil
return "", "", "", "", "", "", "", nil
}
if err != nil {
return "", "", "", "", "", "", err
return "", "", "", "", "", "", "", err
}
// Convert NullString to string
@@ -348,8 +348,11 @@ func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, de
if iconURLNull.Valid {
iconURL = iconURLNull.String
}
if readmeURLNull.Valid {
readmeURL = readmeURLNull.String
}
return title, description, sourceURL, documentationURL, licenses, iconURL, nil
return title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, nil
}
// GetUserByDID retrieves a user by DID
@@ -536,8 +539,8 @@ func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) {
INSERT INTO manifests
(did, repository, digest, hold_endpoint, schema_version, media_type,
config_digest, config_size, created_at,
title, description, source_url, documentation_url, licenses, icon_url)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
title, description, source_url, documentation_url, licenses, icon_url, readme_url)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(did, repository, digest) DO UPDATE SET
hold_endpoint = excluded.hold_endpoint,
schema_version = excluded.schema_version,
@@ -549,12 +552,13 @@ func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) {
source_url = excluded.source_url,
documentation_url = excluded.documentation_url,
licenses = excluded.licenses,
icon_url = excluded.icon_url
icon_url = excluded.icon_url,
readme_url = excluded.readme_url
`, manifest.DID, manifest.Repository, manifest.Digest, manifest.HoldEndpoint,
manifest.SchemaVersion, manifest.MediaType, manifest.ConfigDigest,
manifest.ConfigSize, manifest.CreatedAt,
manifest.Title, manifest.Description, manifest.SourceURL,
manifest.DocumentationURL, manifest.Licenses, manifest.IconURL)
manifest.DocumentationURL, manifest.Licenses, manifest.IconURL, manifest.ReadmeURL)
if err != nil {
return 0, err
pkg/appview/db/queries_test.go
+6 -6
View File
@@ -26,11 +26,11 @@ func TestGetRepositoryMetadata(t *testing.T) {
}
// Test 1: No manifests - should return empty strings
title, description, sourceURL, documentationURL, licenses, iconURL, err := GetRepositoryMetadata(db, testUser.DID, "nonexistent")
title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err := GetRepositoryMetadata(db, testUser.DID, "nonexistent")
if err != nil {
t.Fatalf("Expected no error for nonexistent repo, got: %v", err)
}
if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" {
if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" || readmeURL != "" {
t.Error("Expected all empty strings for nonexistent repository")
}
@@ -47,7 +47,7 @@ func TestGetRepositoryMetadata(t *testing.T) {
}
// Test 3: Retrieve metadata
title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp")
title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp")
if err != nil {
t.Fatalf("Failed to get repository metadata: %v", err)
}
@@ -84,7 +84,7 @@ func TestGetRepositoryMetadata(t *testing.T) {
}
// Test 5: Should return metadata from most recent manifest
title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp")
title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp")
if err != nil {
t.Fatalf("Failed to get repository metadata: %v", err)
}
@@ -109,12 +109,12 @@ func TestGetRepositoryMetadata(t *testing.T) {
}
// Test 7: Should handle NULL fields gracefully
title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "minimal-app")
title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "minimal-app")
if err != nil {
t.Fatalf("Failed to get repository metadata for minimal app: %v", err)
}
if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" {
if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" || readmeURL != "" {
t.Error("Expected all empty strings for manifest with NULL metadata fields")
}
}
pkg/appview/db/schema.go
+4 -199
View File
@@ -17,203 +17,8 @@ import (
//go:embed migrations/*.yaml
var migrationsFS embed.FS
const schema = `
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS users (
did TEXT PRIMARY KEY,
handle TEXT NOT NULL,
pds_endpoint TEXT NOT NULL,
avatar TEXT,
last_seen TIMESTAMP NOT NULL,
UNIQUE(handle)
);
CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle);
CREATE TABLE IF NOT EXISTS manifests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
did TEXT NOT NULL,
repository TEXT NOT NULL,
digest TEXT NOT NULL,
hold_endpoint TEXT NOT NULL, -- Stored as DID (e.g., did:web:hold.example.com)
schema_version INTEGER NOT NULL,
media_type TEXT NOT NULL,
config_digest TEXT,
config_size INTEGER,
created_at TIMESTAMP NOT NULL,
title TEXT,
description TEXT,
source_url TEXT,
documentation_url TEXT,
licenses TEXT,
icon_url TEXT,
UNIQUE(did, repository, digest),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository);
CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest);
CREATE TABLE IF NOT EXISTS layers (
manifest_id INTEGER NOT NULL,
digest TEXT NOT NULL,
size INTEGER NOT NULL,
media_type TEXT NOT NULL,
layer_index INTEGER NOT NULL,
PRIMARY KEY(manifest_id, layer_index),
FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_layers_digest ON layers(digest);
CREATE TABLE IF NOT EXISTS manifest_references (
manifest_id INTEGER NOT NULL,
digest TEXT NOT NULL,
media_type TEXT NOT NULL,
size INTEGER NOT NULL,
platform_architecture TEXT,
platform_os TEXT,
platform_variant TEXT,
platform_os_version TEXT,
reference_index INTEGER NOT NULL,
PRIMARY KEY(manifest_id, reference_index),
FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
did TEXT NOT NULL,
repository TEXT NOT NULL,
tag TEXT NOT NULL,
digest TEXT NOT NULL,
created_at TIMESTAMP NOT NULL,
UNIQUE(did, repository, tag),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_tags_did_repo ON tags(did, repository);
CREATE TABLE IF NOT EXISTS oauth_sessions (
session_key TEXT PRIMARY KEY,
account_did TEXT NOT NULL,
session_id TEXT NOT NULL,
session_data TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(account_did, session_id)
);
CREATE INDEX IF NOT EXISTS idx_oauth_sessions_did ON oauth_sessions(account_did);
CREATE INDEX IF NOT EXISTS idx_oauth_sessions_updated ON oauth_sessions(updated_at DESC);
CREATE TABLE IF NOT EXISTS oauth_auth_requests (
state TEXT PRIMARY KEY,
request_data TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_oauth_auth_requests_created ON oauth_auth_requests(created_at);
CREATE TABLE IF NOT EXISTS ui_sessions (
id TEXT PRIMARY KEY,
did TEXT NOT NULL,
handle TEXT NOT NULL,
pds_endpoint TEXT NOT NULL,
oauth_session_id TEXT,
expires_at TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_ui_sessions_did ON ui_sessions(did);
CREATE INDEX IF NOT EXISTS idx_ui_sessions_expires ON ui_sessions(expires_at);
CREATE TABLE IF NOT EXISTS devices (
id TEXT PRIMARY KEY,
did TEXT NOT NULL,
handle TEXT NOT NULL,
name TEXT NOT NULL,
secret_hash TEXT NOT NULL UNIQUE,
ip_address TEXT,
location TEXT,
user_agent TEXT,
created_at TIMESTAMP NOT NULL,
last_used TIMESTAMP,
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_devices_did ON devices(did);
CREATE INDEX IF NOT EXISTS idx_devices_hash ON devices(secret_hash);
CREATE TABLE IF NOT EXISTS pending_device_auth (
device_code TEXT PRIMARY KEY,
user_code TEXT NOT NULL UNIQUE,
device_name TEXT NOT NULL,
ip_address TEXT,
user_agent TEXT,
expires_at TIMESTAMP NOT NULL,
approved_did TEXT,
approved_at TIMESTAMP,
device_secret TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_pending_device_auth_user_code ON pending_device_auth(user_code);
CREATE INDEX IF NOT EXISTS idx_pending_device_auth_expires ON pending_device_auth(expires_at);
CREATE TABLE IF NOT EXISTS repository_stats (
did TEXT NOT NULL,
repository TEXT NOT NULL,
pull_count INTEGER NOT NULL DEFAULT 0,
last_pull TIMESTAMP,
push_count INTEGER NOT NULL DEFAULT 0,
last_push TIMESTAMP,
PRIMARY KEY(did, repository),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did);
CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC);
CREATE TABLE IF NOT EXISTS stars (
starrer_did TEXT NOT NULL,
owner_did TEXT NOT NULL,
repository TEXT NOT NULL,
created_at TIMESTAMP NOT NULL,
PRIMARY KEY(starrer_did, owner_did, repository),
FOREIGN KEY(starrer_did) REFERENCES users(did) ON DELETE CASCADE,
FOREIGN KEY(owner_did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository);
CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did);
CREATE TABLE IF NOT EXISTS hold_captain_records (
hold_did TEXT PRIMARY KEY,
owner_did TEXT NOT NULL,
public BOOLEAN NOT NULL,
allow_all_crew BOOLEAN NOT NULL,
deployed_at TEXT,
region TEXT,
provider TEXT,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at);
CREATE TABLE IF NOT EXISTS hold_crew_approvals (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at);
CREATE TABLE IF NOT EXISTS hold_crew_denials (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
denial_count INTEGER NOT NULL DEFAULT 1,
next_retry_at TIMESTAMP NOT NULL,
last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at);
`
//go:embed schema.sql
var schemaSQL string
// InitDB initializes the SQLite database with the schema
func InitDB(path string) (*sql.DB, error) {
@@ -227,8 +32,8 @@ func InitDB(path string) (*sql.DB, error) {
return nil, err
}
// Create schema
if _, err := db.Exec(schema); err != nil {
// Create schema from embedded SQL file
if _, err := db.Exec(schemaSQL); err != nil {
return nil, err
}
pkg/appview/db/schema.sql
+207
View File
@@ -0,0 +1,207 @@
-- ATCR AppView Database Schema
-- This file contains the complete base schema for fresh database installations.
-- Migrations (in migrations/*.yaml) handle changes to existing databases.
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS users (
did TEXT PRIMARY KEY,
handle TEXT NOT NULL,
pds_endpoint TEXT NOT NULL,
avatar TEXT,
last_seen TIMESTAMP NOT NULL,
UNIQUE(handle)
);
CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle);
CREATE TABLE IF NOT EXISTS manifests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
did TEXT NOT NULL,
repository TEXT NOT NULL,
digest TEXT NOT NULL,
hold_endpoint TEXT NOT NULL, -- Stored as DID (e.g., did:web:hold.example.com)
schema_version INTEGER NOT NULL,
media_type TEXT NOT NULL,
config_digest TEXT,
config_size INTEGER,
created_at TIMESTAMP NOT NULL,
title TEXT,
description TEXT,
source_url TEXT,
documentation_url TEXT,
licenses TEXT,
icon_url TEXT,
readme_url TEXT,
UNIQUE(did, repository, digest),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository);
CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest);
CREATE TABLE IF NOT EXISTS layers (
manifest_id INTEGER NOT NULL,
digest TEXT NOT NULL,
size INTEGER NOT NULL,
media_type TEXT NOT NULL,
layer_index INTEGER NOT NULL,
PRIMARY KEY(manifest_id, layer_index),
FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_layers_digest ON layers(digest);
CREATE TABLE IF NOT EXISTS manifest_references (
manifest_id INTEGER NOT NULL,
digest TEXT NOT NULL,
media_type TEXT NOT NULL,
size INTEGER NOT NULL,
platform_architecture TEXT,
platform_os TEXT,
platform_variant TEXT,
platform_os_version TEXT,
reference_index INTEGER NOT NULL,
PRIMARY KEY(manifest_id, reference_index),
FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
did TEXT NOT NULL,
repository TEXT NOT NULL,
tag TEXT NOT NULL,
digest TEXT NOT NULL,
created_at TIMESTAMP NOT NULL,
UNIQUE(did, repository, tag),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_tags_did_repo ON tags(did, repository);
CREATE TABLE IF NOT EXISTS oauth_sessions (
session_key TEXT PRIMARY KEY,
account_did TEXT NOT NULL,
session_id TEXT NOT NULL,
session_data TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(account_did, session_id)
);
CREATE INDEX IF NOT EXISTS idx_oauth_sessions_did ON oauth_sessions(account_did);
CREATE INDEX IF NOT EXISTS idx_oauth_sessions_updated ON oauth_sessions(updated_at DESC);
CREATE TABLE IF NOT EXISTS oauth_auth_requests (
state TEXT PRIMARY KEY,
request_data TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_oauth_auth_requests_created ON oauth_auth_requests(created_at);
CREATE TABLE IF NOT EXISTS ui_sessions (
id TEXT PRIMARY KEY,
did TEXT NOT NULL,
handle TEXT NOT NULL,
pds_endpoint TEXT NOT NULL,
oauth_session_id TEXT,
expires_at TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_ui_sessions_did ON ui_sessions(did);
CREATE INDEX IF NOT EXISTS idx_ui_sessions_expires ON ui_sessions(expires_at);
CREATE TABLE IF NOT EXISTS devices (
id TEXT PRIMARY KEY,
did TEXT NOT NULL,
handle TEXT NOT NULL,
name TEXT NOT NULL,
secret_hash TEXT NOT NULL UNIQUE,
ip_address TEXT,
location TEXT,
user_agent TEXT,
created_at TIMESTAMP NOT NULL,
last_used TIMESTAMP,
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_devices_did ON devices(did);
CREATE INDEX IF NOT EXISTS idx_devices_hash ON devices(secret_hash);
CREATE TABLE IF NOT EXISTS pending_device_auth (
device_code TEXT PRIMARY KEY,
user_code TEXT NOT NULL UNIQUE,
device_name TEXT NOT NULL,
ip_address TEXT,
user_agent TEXT,
expires_at TIMESTAMP NOT NULL,
approved_did TEXT,
approved_at TIMESTAMP,
device_secret TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_pending_device_auth_user_code ON pending_device_auth(user_code);
CREATE INDEX IF NOT EXISTS idx_pending_device_auth_expires ON pending_device_auth(expires_at);
CREATE TABLE IF NOT EXISTS repository_stats (
did TEXT NOT NULL,
repository TEXT NOT NULL,
pull_count INTEGER NOT NULL DEFAULT 0,
last_pull TIMESTAMP,
push_count INTEGER NOT NULL DEFAULT 0,
last_push TIMESTAMP,
PRIMARY KEY(did, repository),
FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did);
CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC);
CREATE TABLE IF NOT EXISTS stars (
starrer_did TEXT NOT NULL,
owner_did TEXT NOT NULL,
repository TEXT NOT NULL,
created_at TIMESTAMP NOT NULL,
PRIMARY KEY(starrer_did, owner_did, repository),
FOREIGN KEY(starrer_did) REFERENCES users(did) ON DELETE CASCADE,
FOREIGN KEY(owner_did) REFERENCES users(did) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository);
CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did);
CREATE TABLE IF NOT EXISTS hold_captain_records (
hold_did TEXT PRIMARY KEY,
owner_did TEXT NOT NULL,
public BOOLEAN NOT NULL,
allow_all_crew BOOLEAN NOT NULL,
deployed_at TEXT,
region TEXT,
provider TEXT,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at);
CREATE TABLE IF NOT EXISTS hold_crew_approvals (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at);
CREATE TABLE IF NOT EXISTS hold_crew_denials (
hold_did TEXT NOT NULL,
user_did TEXT NOT NULL,
denial_count INTEGER NOT NULL DEFAULT 1,
next_retry_at TIMESTAMP NOT NULL,
last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(hold_did, user_did)
);
CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at);
CREATE TABLE IF NOT EXISTS readme_cache (
url TEXT PRIMARY KEY,
html TEXT NOT NULL,
fetched_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_readme_cache_fetched ON readme_cache(fetched_at);
pkg/appview/handlers/repository.go
+22 -1
View File
@@ -12,6 +12,7 @@ import (
"atcr.io/pkg/appview/db"
"atcr.io/pkg/appview/holdhealth"
"atcr.io/pkg/appview/middleware"
"atcr.io/pkg/appview/readme"
"atcr.io/pkg/atproto"
"atcr.io/pkg/auth/oauth"
"github.com/bluesky-social/indigo/atproto/identity"
@@ -26,6 +27,7 @@ type RepositoryPageHandler struct {
Directory identity.Directory
Refresher *oauth.Refresher
HealthChecker *holdhealth.Checker
ReadmeCache *readme.Cache
}
func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -135,7 +137,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
}
// Fetch repository metadata from most recent manifest
title, description, sourceURL, documentationURL, licenses, iconURL, err := db.GetRepositoryMetadata(h.DB, owner.DID, repository)
title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err := db.GetRepositoryMetadata(h.DB, owner.DID, repository)
if err != nil {
log.Printf("Failed to fetch repository metadata: %v", err)
// Continue without metadata on error
@@ -146,6 +148,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
repo.DocumentationURL = documentationURL
repo.Licenses = licenses
repo.IconURL = iconURL
repo.ReadmeURL = readmeURL
}
// Fetch star count
@@ -180,6 +183,22 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
isOwner = (user.DID == owner.DID)
}
// Fetch README content if available
var readmeHTML template.HTML
if repo.ReadmeURL != "" && h.ReadmeCache != nil {
// Fetch with timeout
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()
html, err := h.ReadmeCache.Get(ctx, repo.ReadmeURL)
if err != nil {
log.Printf("Failed to fetch README from %s: %v", repo.ReadmeURL, err)
// Continue without README on error
} else {
readmeHTML = template.HTML(html)
}
}
data := struct {
PageData
Owner *db.User // Repository owner
@@ -189,6 +208,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
StarCount int
IsStarred bool
IsOwner bool // Whether current user owns this repository
ReadmeHTML template.HTML
}{
PageData: NewPageData(r, h.RegistryURL),
Owner: owner,
@@ -198,6 +218,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
StarCount: stats.StarCount,
IsStarred: isStarred,
IsOwner: isOwner,
ReadmeHTML: readmeHTML,
}
if err := h.Templates.ExecuteTemplate(w, "repository", data); err != nil {
pkg/appview/jetstream/backfill.go
+3 -1
View File
@@ -298,7 +298,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor
}
// Extract OCI annotations from manifest
var title, description, sourceURL, documentationURL, licenses, iconURL string
var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string
if manifestRecord.Annotations != nil {
title = manifestRecord.Annotations["org.opencontainers.image.title"]
description = manifestRecord.Annotations["org.opencontainers.image.description"]
@@ -306,6 +306,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor
documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"]
licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"]
iconURL = manifestRecord.Annotations["io.atcr.icon"]
readmeURL = manifestRecord.Annotations["io.atcr.readme"]
}
// Detect manifest type
@@ -326,6 +327,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor
DocumentationURL: documentationURL,
Licenses: licenses,
IconURL: iconURL,
ReadmeURL: readmeURL,
}
// Set config fields only for image manifests (not manifest lists)
pkg/appview/jetstream/worker.go
+3 -1
View File
@@ -442,7 +442,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error {
}
// Extract OCI annotations from manifest
var title, description, sourceURL, documentationURL, licenses, iconURL string
var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string
if manifestRecord.Annotations != nil {
title = manifestRecord.Annotations["org.opencontainers.image.title"]
description = manifestRecord.Annotations["org.opencontainers.image.description"]
@@ -450,6 +450,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error {
documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"]
licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"]
iconURL = manifestRecord.Annotations["io.atcr.icon"]
readmeURL = manifestRecord.Annotations["io.atcr.readme"]
}
// Detect manifest type
@@ -470,6 +471,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error {
DocumentationURL: documentationURL,
Licenses: licenses,
IconURL: iconURL,
ReadmeURL: readmeURL,
}
// Set config fields only for image manifests (not manifest lists)
pkg/appview/readme/cache.go
+108
View File
@@ -0,0 +1,108 @@
package readme
import (
"context"
"database/sql"
"fmt"
"time"
)
// Cache stores rendered README HTML in the database
type Cache struct {
db *sql.DB
fetcher *Fetcher
ttl time.Duration
}
// NewCache creates a new README cache
func NewCache(db *sql.DB, ttl time.Duration) *Cache {
if ttl == 0 {
ttl = 1 * time.Hour // Default TTL
}
return &Cache{
db: db,
fetcher: NewFetcher(),
ttl: ttl,
}
}
// Get retrieves a README from cache or fetches it
func (c *Cache) Get(ctx context.Context, readmeURL string) (string, error) {
// Try to get from cache
html, fetchedAt, err := c.getFromDB(readmeURL)
if err == nil {
// Check if cache is still valid
if time.Since(fetchedAt) < c.ttl {
return html, nil
}
}
// Cache miss or expired, fetch fresh content
html, err = c.fetcher.FetchAndRender(ctx, readmeURL)
if err != nil {
// If fetch fails but we have stale cache, return it
if html != "" {
return html, nil
}
return "", err
}
// Store in cache
if err := c.storeInDB(readmeURL, html); err != nil {
// Log error but don't fail - we have the content
// In production, you'd use proper logging here
fmt.Printf("Failed to cache README: %v\n", err)
}
return html, nil
}
// getFromDB retrieves cached README from database
func (c *Cache) getFromDB(readmeURL string) (string, time.Time, error) {
var html string
var fetchedAt time.Time
err := c.db.QueryRow(`
SELECT html, fetched_at
FROM readme_cache
WHERE url = ?
`, readmeURL).Scan(&html, &fetchedAt)
if err != nil {
return "", time.Time{}, err
}
return html, fetchedAt, nil
}
// storeInDB stores rendered README in database
func (c *Cache) storeInDB(readmeURL, html string) error {
_, err := c.db.Exec(`
INSERT INTO readme_cache (url, html, fetched_at)
VALUES (?, ?, ?)
ON CONFLICT(url) DO UPDATE SET
html = excluded.html,
fetched_at = excluded.fetched_at
`, readmeURL, html, time.Now())
return err
}
// Invalidate removes a README from the cache
func (c *Cache) Invalidate(readmeURL string) error {
_, err := c.db.Exec(`
DELETE FROM readme_cache
WHERE url = ?
`, readmeURL)
return err
}
// Cleanup removes expired entries from the cache
func (c *Cache) Cleanup() error {
cutoff := time.Now().Add(-c.ttl * 2) // Keep for 2x TTL
_, err := c.db.Exec(`
DELETE FROM readme_cache
WHERE fetched_at < ?
`, cutoff)
return err
}
pkg/appview/readme/fetcher.go
+210
View File
@@ -0,0 +1,210 @@
package readme
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/microcosm-cc/bluemonday"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer/html"
)
// Fetcher fetches and renders README content from URLs
type Fetcher struct {
httpClient *http.Client
markdown goldmark.Markdown
sanitizer *bluemonday.Policy
}
// NewFetcher creates a new README fetcher
func NewFetcher() *Fetcher {
// Configure markdown renderer with GitHub-flavored markdown
md := goldmark.New(
goldmark.WithExtensions(
extension.GFM, // GitHub Flavored Markdown
extension.Typographer, // Smart quotes, dashes, etc.
),
goldmark.WithParserOptions(
parser.WithAutoHeadingID(), // Auto-generate heading IDs
),
goldmark.WithRendererOptions(
html.WithHardWraps(), // Line breaks create <br>
html.WithXHTML(), // XHTML-compliant output
// html.WithUnsafe(), // Uncomment ONLY if you want to allow raw HTML in markdown (not recommended)
),
)
// Configure HTML sanitizer as a safety net
// This catches any HTML that makes it through (if WithUnsafe() is enabled)
sanitizer := bluemonday.UGCPolicy()
// Allow additional attributes for better markdown rendering
sanitizer.AllowAttrs("class").Globally()
sanitizer.AllowAttrs("id").Globally()
sanitizer.AllowAttrs("align").OnElements("img", "div", "p", "span")
return &Fetcher{
httpClient: &http.Client{
Timeout: 10 * time.Second,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// Allow up to 5 redirects
if len(via) >= 5 {
return fmt.Errorf("too many redirects")
}
return nil
},
},
markdown: md,
sanitizer: sanitizer,
}
}
// FetchAndRender fetches a README from a URL and renders it as HTML
// Returns the rendered HTML and any error
func (f *Fetcher) FetchAndRender(ctx context.Context, readmeURL string) (string, error) {
// Validate URL
if readmeURL == "" {
return "", fmt.Errorf("empty README URL")
}
parsedURL, err := url.Parse(readmeURL)
if err != nil {
return "", fmt.Errorf("invalid README URL: %w", err)
}
// Only allow HTTP/HTTPS
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
return "", fmt.Errorf("invalid URL scheme: %s", parsedURL.Scheme)
}
// Fetch content
content, baseURL, err := f.fetchContent(ctx, readmeURL)
if err != nil {
return "", err
}
// Render markdown to HTML
html, err := f.renderMarkdown(content, baseURL)
if err != nil {
return "", fmt.Errorf("failed to render markdown: %w", err)
}
return html, nil
}
// fetchContent fetches the raw content from a URL
func (f *Fetcher) fetchContent(ctx context.Context, urlStr string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
if err != nil {
return nil, "", fmt.Errorf("failed to create request: %w", err)
}
// Set user agent
req.Header.Set("User-Agent", "ATCR-README-Fetcher/1.0")
resp, err := f.httpClient.Do(req)
if err != nil {
return nil, "", fmt.Errorf("failed to fetch URL: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
// Limit content size to 1MB
limitedReader := io.LimitReader(resp.Body, 1*1024*1024)
content, err := io.ReadAll(limitedReader)
if err != nil {
return nil, "", fmt.Errorf("failed to read response body: %w", err)
}
// Get base URL for relative link resolution
baseURL := getBaseURL(resp.Request.URL)
return content, baseURL, nil
}
// renderMarkdown renders markdown content to sanitized HTML
func (f *Fetcher) renderMarkdown(content []byte, baseURL string) (string, error) {
var buf bytes.Buffer
if err := f.markdown.Convert(content, &buf); err != nil {
return "", err
}
// Rewrite relative URLs to absolute
html := buf.String()
if baseURL != "" {
html = rewriteRelativeURLs(html, baseURL)
}
// Sanitize HTML
sanitized := f.sanitizer.Sanitize(html)
return sanitized, nil
}
// getBaseURL extracts the base URL for relative link resolution
func getBaseURL(u *url.URL) string {
if u == nil {
return ""
}
// For GitHub raw URLs, convert to blob URL base for relative links
// e.g., https://raw.githubusercontent.com/user/repo/main/README.md
// -> https://github.com/user/repo/blob/main/
if u.Host == "raw.githubusercontent.com" {
parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/")
if len(parts) >= 3 {
user := parts[0]
repo := parts[1]
branch := parts[2]
return fmt.Sprintf("https://github.com/%s/%s/blob/%s/", user, repo, branch)
}
}
// For other URLs, use the directory containing the file
path := u.Path
lastSlash := strings.LastIndex(path, "/")
if lastSlash >= 0 {
path = path[:lastSlash+1]
}
return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, path)
}
// rewriteRelativeURLs converts relative URLs to absolute URLs
func rewriteRelativeURLs(html, baseURL string) string {
if baseURL == "" {
return html
}
base, err := url.Parse(baseURL)
if err != nil {
return html
}
// Simple string replacement for common patterns
// This is a basic implementation - for production, consider using an HTML parser
html = strings.ReplaceAll(html, `src="./`, fmt.Sprintf(`src="%s`, baseURL))
html = strings.ReplaceAll(html, `href="./`, fmt.Sprintf(`href="%s`, baseURL))
html = strings.ReplaceAll(html, `src="../`, fmt.Sprintf(`src="%s../`, baseURL))
html = strings.ReplaceAll(html, `href="../`, fmt.Sprintf(`href="%s../`, baseURL))
// Handle root-relative URLs (starting with /)
if base.Scheme != "" && base.Host != "" {
root := fmt.Sprintf("%s://%s/", base.Scheme, base.Host)
// Replace src="/" and href="/" but not src="//" (absolute URLs)
html = strings.ReplaceAll(html, `src="/`, fmt.Sprintf(`src="%s`, root))
html = strings.ReplaceAll(html, `href="/`, fmt.Sprintf(`href="%s`, root))
}
return html
}
pkg/appview/static/css/style.css
+213
View File
@@ -1639,3 +1639,216 @@ button:hover, .btn:hover, .btn-primary:hover, .btn-secondary:hover {
grid-template-columns: repeat(3, 1fr);
}
}
/* README and Repository Layout */
.repo-content-layout {
display: grid;
grid-template-columns: 1fr 400px;
gap: 2rem;
margin-top: 2rem;
}
.readme-section {
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
padding: 2rem;
}
.readme-section h2 {
margin-bottom: 1.5rem;
padding-bottom: 0.5rem;
border-bottom: 2px solid var(--border);
}
.readme-content {
overflow-wrap: break-word;
}
.repo-sidebar {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
/* Markdown Styling */
.markdown-body {
font-size: 1rem;
line-height: 1.6;
word-wrap: break-word;
}
.markdown-body h1,
.markdown-body h2,
.markdown-body h3,
.markdown-body h4,
.markdown-body h5,
.markdown-body h6 {
margin-top: 1.5rem;
margin-bottom: 1rem;
font-weight: 600;
line-height: 1.25;
}
.markdown-body h1 {
font-size: 2rem;
border-bottom: 1px solid var(--border);
padding-bottom: 0.3rem;
}
.markdown-body h2 {
font-size: 1.5rem;
border-bottom: 1px solid var(--border);
padding-bottom: 0.3rem;
}
.markdown-body h3 {
font-size: 1.25rem;
}
.markdown-body h4 {
font-size: 1rem;
}
.markdown-body h5 {
font-size: 0.875rem;
}
.markdown-body h6 {
font-size: 0.85rem;
color: var(--secondary);
}
.markdown-body p {
margin-bottom: 1rem;
}
.markdown-body ul,
.markdown-body ol {
margin-bottom: 1rem;
padding-left: 2rem;
}
.markdown-body li {
margin-bottom: 0.25rem;
}
.markdown-body li > p {
margin-bottom: 0.5rem;
}
.markdown-body a {
color: var(--primary);
text-decoration: none;
}
.markdown-body a:hover {
text-decoration: underline;
}
.markdown-body code {
background: var(--code-bg);
padding: 0.2rem 0.4rem;
border-radius: 3px;
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
font-size: 0.9em;
}
.markdown-body pre {
background: var(--code-bg);
padding: 1rem;
border-radius: 6px;
overflow-x: auto;
margin-bottom: 1rem;
}
.markdown-body pre code {
background: none;
padding: 0;
font-size: 0.875rem;
}
.markdown-body blockquote {
padding: 0 1rem;
margin-bottom: 1rem;
color: var(--secondary);
border-left: 4px solid var(--border);
}
.markdown-body table {
border-collapse: collapse;
width: 100%;
margin-bottom: 1rem;
}
.markdown-body table th,
.markdown-body table td {
padding: 0.5rem 1rem;
border: 1px solid var(--border);
text-align: left;
}
.markdown-body table th {
background: var(--code-bg);
font-weight: 600;
}
.markdown-body table tr:nth-child(even) {
background: var(--hover-bg);
}
.markdown-body img {
max-width: 100%;
height: auto;
margin: 1rem 0;
}
.markdown-body hr {
height: 0.25rem;
margin: 1.5rem 0;
background: var(--border);
border: 0;
}
/* Task lists */
.markdown-body input[type="checkbox"] {
margin-right: 0.5rem;
}
.markdown-body .task-list-item {
list-style-type: none;
}
.markdown-body .task-list-item input {
margin: 0 0.2rem 0.25rem -1.6rem;
vertical-align: middle;
}
/* Responsive Layout */
@media (max-width: 1024px) {
.repo-content-layout {
grid-template-columns: 1fr;
}
.repo-sidebar {
order: -1; /* Show sidebar first on mobile */
}
}
@media (max-width: 768px) {
.readme-section {
padding: 1rem;
}
.markdown-body h1 {
font-size: 1.5rem;
}
.markdown-body h2 {
font-size: 1.25rem;
}
.markdown-body pre {
padding: 0.75rem;
}
}
pkg/appview/templates/pages/repository.html
+20
View File
@@ -83,6 +83,21 @@
</div>
</div>
<!-- README and Tags/Manifests Layout -->
{{ if .ReadmeHTML }}
<div class="repo-content-layout">
<!-- README Section (Left) -->
<div class="readme-section">
<h2>Overview</h2>
<div class="readme-content markdown-body">
{{ .ReadmeHTML }}
</div>
</div>
<!-- Tags and Manifests (Right) -->
<div class="repo-sidebar">
{{ end }}
<!-- Tags Section -->
<div class="repo-section">
<h2>Tags</h2>
@@ -213,6 +228,11 @@
<p class="empty-message">No manifests available</p>
{{ end }}
</div>
{{ if .ReadmeHTML }}
</div><!-- Close repo-sidebar -->
</div><!-- Close repo-content-layout -->
{{ end }}
</div>
</main>