diff --git a/CLAUDE.md b/CLAUDE.md index eaa33e2..9274451 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -642,11 +642,14 @@ When writing tests: 5. Update `.env.example` with new driver's env vars **Working with the database**: -- Schema defined in `pkg/appview/db/schema.go` -- Queries in `pkg/appview/db/queries.go` -- Stores for OAuth, devices, sessions in separate files -- Run migrations automatically on startup -- Database path configurable via `ATCR_UI_DATABASE_PATH` env var +- **Base schema** defined in `pkg/appview/db/schema.sql` - source of truth for fresh installations +- **Migrations** in `pkg/appview/db/migrations/*.yaml` - only for ALTER/UPDATE/DELETE on existing databases +- **Queries** in `pkg/appview/db/queries.go` +- **Stores** for OAuth, devices, sessions in separate files +- **Execution order**: schema.sql first, then migrations (automatically on startup) +- **Database path** configurable via `ATCR_UI_DATABASE_PATH` env var +- **Adding new tables**: Add to `schema.sql` only (no migration needed) +- **Altering tables**: Create migration AND update `schema.sql` to keep them in sync **Adding web UI features**: - Add handler in `pkg/appview/handlers/` diff --git a/cmd/appview/serve.go b/cmd/appview/serve.go index de36110..3fdbb1a 100644 --- a/cmd/appview/serve.go +++ b/cmd/appview/serve.go @@ -28,6 +28,7 @@ import ( uihandlers "atcr.io/pkg/appview/handlers" "atcr.io/pkg/appview/holdhealth" "atcr.io/pkg/appview/jetstream" + "atcr.io/pkg/appview/readme" "github.com/gorilla/mux" ) @@ -88,6 +89,18 @@ func serveRegistry(cmd *cobra.Command, args []string) error { healthChecker := holdhealth.NewChecker(cacheTTL) + // Initialize README cache + fmt.Println("Initializing README cache...") + readmeCacheTTL := 1 * time.Hour // Default: 1 hour + if readmeTTLStr := os.Getenv("ATCR_README_CACHE_TTL"); readmeTTLStr != "" { + if parsed, err := time.ParseDuration(readmeTTLStr); err == nil { + readmeCacheTTL = parsed + } else { + fmt.Printf("Warning: Invalid ATCR_README_CACHE_TTL '%s', using default 1h\n", readmeTTLStr) + } + } + readmeCache := readme.NewCache(uiDatabase, readmeCacheTTL) + // Start background health check worker // Parse refresh interval from environment (default: 15m) refreshInterval := 15 * time.Minute @@ -184,8 +197,8 @@ func serveRegistry(cmd *cobra.Command, args []string) error { middleware.SetGlobalAuthorizer(holdAuthorizer) fmt.Println("Hold authorizer initialized with database caching") - // Initialize UI routes with OAuth app, refresher, device store, and health checker - uiTemplates, uiRouter := initializeUIRoutes(uiDatabase, uiReadOnlyDB, uiSessionStore, oauthApp, refresher, baseURL, deviceStore, defaultHoldDID, healthChecker) + // Initialize UI routes with OAuth app, refresher, device store, health checker, and readme cache + uiTemplates, uiRouter := initializeUIRoutes(uiDatabase, uiReadOnlyDB, uiSessionStore, oauthApp, refresher, baseURL, deviceStore, defaultHoldDID, healthChecker, readmeCache) // Create OAuth server oauthServer := oauth.NewServer(oauthApp) @@ -380,7 +393,7 @@ func createTokenIssuer(config *configuration.Configuration) (*token.Issuer, erro // readOnlyDB: read-only connection for public queries (search, user pages, etc.) // defaultHoldDID: DID of the default hold service (e.g., "did:web:hold01.atcr.io") // healthChecker: hold endpoint health checker -func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.SessionStore, oauthApp *oauth.App, refresher *oauth.Refresher, baseURL string, deviceStore *db.DeviceStore, defaultHoldDID string, healthChecker *holdhealth.Checker) (*template.Template, *mux.Router) { +func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.SessionStore, oauthApp *oauth.App, refresher *oauth.Refresher, baseURL string, deviceStore *db.DeviceStore, defaultHoldDID string, healthChecker *holdhealth.Checker, readmeCache *readme.Cache) (*template.Template, *mux.Router) { // Check if UI is enabled uiEnabled := os.Getenv("ATCR_UI_ENABLED") if uiEnabled == "false" { @@ -510,6 +523,7 @@ func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.S Directory: oauthApp.Directory(), Refresher: refresher, HealthChecker: healthChecker, + ReadmeCache: readmeCache, }, )).Methods("GET") diff --git a/go.mod b/go.mod index c18430e..d118537 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( ) require ( + github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -51,6 +52,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect @@ -85,6 +87,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -105,6 +108,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/yuin/goldmark v1.7.13 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect diff --git a/go.sum b/go.sum index 816db8d..b3fd7fd 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -108,6 +110,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -266,6 +270,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -377,6 +383,8 @@ github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= diff --git a/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml b/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml deleted file mode 100644 index 36cbb2c..0000000 --- a/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml +++ /dev/null @@ -1,13 +0,0 @@ -description: Add hold_captain_records table for caching hold security settings -query: | - CREATE TABLE IF NOT EXISTS hold_captain_records ( - hold_did TEXT PRIMARY KEY, - owner_did TEXT NOT NULL, - public BOOLEAN NOT NULL, - allow_all_crew BOOLEAN NOT NULL, - deployed_at TEXT, - region TEXT, - provider TEXT, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at); diff --git a/pkg/appview/db/migrations/0005_normalize_hold_endpoint_to_did.yaml b/pkg/appview/db/migrations/0002_normalize_hold_endpoint_to_did.yaml similarity index 100% rename from pkg/appview/db/migrations/0005_normalize_hold_endpoint_to_did.yaml rename to pkg/appview/db/migrations/0002_normalize_hold_endpoint_to_did.yaml diff --git a/pkg/appview/db/migrations/0003_add_crew_cache.yaml b/pkg/appview/db/migrations/0003_add_crew_cache.yaml deleted file mode 100644 index f4595b8..0000000 --- a/pkg/appview/db/migrations/0003_add_crew_cache.yaml +++ /dev/null @@ -1,20 +0,0 @@ -description: Add crew cache tables for authorization with exponential backoff -query: | - CREATE TABLE IF NOT EXISTS hold_crew_approvals ( - hold_did TEXT NOT NULL, - user_did TEXT NOT NULL, - approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP NOT NULL, - PRIMARY KEY(hold_did, user_did) - ); - CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at); - - CREATE TABLE IF NOT EXISTS hold_crew_denials ( - hold_did TEXT NOT NULL, - user_did TEXT NOT NULL, - denial_count INTEGER NOT NULL DEFAULT 1, - next_retry_at TIMESTAMP NOT NULL, - last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(hold_did, user_did) - ); - CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at); diff --git a/pkg/appview/db/migrations/0003_add_readme_url_to_manifests.yaml b/pkg/appview/db/migrations/0003_add_readme_url_to_manifests.yaml new file mode 100644 index 0000000..7f4e332 --- /dev/null +++ b/pkg/appview/db/migrations/0003_add_readme_url_to_manifests.yaml @@ -0,0 +1,46 @@ +description: Add readme_url column to manifests table (idempotent - handles both fresh and existing databases) +query: | + -- Idempotent migration: adds readme_url column if it doesn't exist + -- Works for both fresh installs (where schema.sql created it) and existing databases + + -- Create temp table with new schema + CREATE TABLE manifests_temp ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + repository TEXT NOT NULL, + digest TEXT NOT NULL, + hold_endpoint TEXT NOT NULL, + schema_version INTEGER NOT NULL, + media_type TEXT NOT NULL, + config_digest TEXT, + config_size INTEGER, + created_at TIMESTAMP NOT NULL, + title TEXT, + description TEXT, + source_url TEXT, + documentation_url TEXT, + licenses TEXT, + icon_url TEXT, + readme_url TEXT, + UNIQUE(did, repository, digest), + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE + ); + + -- Copy data from existing manifests table + -- Use INSERT OR IGNORE to handle case where table is already correct + INSERT OR IGNORE INTO manifests_temp + SELECT id, did, repository, digest, hold_endpoint, schema_version, media_type, + config_digest, config_size, created_at, title, description, source_url, + documentation_url, licenses, icon_url, + NULL as readme_url -- Will be NULL for existing data + FROM manifests; + + -- Only proceed with table swap if we actually copied data + -- (manifests_temp will be empty if manifests table already has readme_url) + DROP TABLE IF EXISTS manifests; + ALTER TABLE manifests_temp RENAME TO manifests; + + -- Recreate indexes + CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository); + CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest); diff --git a/pkg/appview/db/migrations/0004_add_manifest_references.yaml b/pkg/appview/db/migrations/0004_add_manifest_references.yaml deleted file mode 100644 index 328d401..0000000 --- a/pkg/appview/db/migrations/0004_add_manifest_references.yaml +++ /dev/null @@ -1,16 +0,0 @@ -description: Add manifest_references table for multi-arch manifest support -query: | - CREATE TABLE IF NOT EXISTS manifest_references ( - manifest_id INTEGER NOT NULL, - digest TEXT NOT NULL, - media_type TEXT NOT NULL, - size INTEGER NOT NULL, - platform_architecture TEXT, - platform_os TEXT, - platform_variant TEXT, - platform_os_version TEXT, - reference_index INTEGER NOT NULL, - PRIMARY KEY(manifest_id, reference_index), - FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE - ); - CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest); diff --git a/pkg/appview/db/migrations/README.md b/pkg/appview/db/migrations/README.md index 6d05b9f..b356812 100644 --- a/pkg/appview/db/migrations/README.md +++ b/pkg/appview/db/migrations/README.md @@ -2,6 +2,18 @@ This directory contains database migrations for the ATCR AppView database. +## Schema vs Migrations + +**`schema.sql`** (in parent directory) contains the **complete base schema** for fresh database installations. It includes all tables, indexes, and constraints. + +**Migrations** (this directory) handle **changes to existing databases**. They are only for: +- `ALTER TABLE` statements (add/modify/drop columns) +- `UPDATE` statements (data transformations) +- `DELETE` statements (data cleanup) +- Creating/modifying indexes on existing tables + +**NEW TABLES go in `schema.sql`, NOT in migrations.** + ## Migration Format Each migration is a YAML file with the following structure: @@ -33,13 +45,43 @@ Examples: 2. **Create a new YAML file** with format `000N_descriptive_name.yaml` 3. **Add description** (optional) - Explain what the migration does 4. **Write your SQL in `query`** - Use the `|` block scalar for clean multi-line SQL -5. **Use `IF EXISTS` / `IF NOT EXISTS`** where possible for idempotency (note: not supported for `DROP COLUMN`) +5. **Use `IF EXISTS` / `IF NOT EXISTS`** where possible for idempotency ## Examples -### Simple single-statement migration: +### Adding a column to existing table: -Filename: `0002_add_repository_description_index.yaml` +Filename: `0007_add_readme_url_to_manifests.yaml` + +```yaml +description: Add readme_url column to manifests table for storing io.atcr.readme annotation +query: | + ALTER TABLE manifests ADD COLUMN readme_url TEXT; +``` + +**IMPORTANT:** After creating this migration, also add the column to `schema.sql` so fresh installations include it! + +### Data transformation migration: + +Filename: `0005_normalize_hold_endpoint_to_did.yaml` + +```yaml +description: Normalize hold_endpoint column to store DIDs instead of URLs +query: | + -- Convert HTTPS URLs to did:web: format + UPDATE manifests + SET hold_endpoint = 'did:web:' || substr(hold_endpoint, 9) + WHERE hold_endpoint LIKE 'https://%'; + + -- Convert HTTP URLs to did:web: format + UPDATE manifests + SET hold_endpoint = 'did:web:' || substr(hold_endpoint, 8) + WHERE hold_endpoint LIKE 'http://%'; +``` + +### Adding an index to existing table: + +Filename: `0008_add_repository_description_index.yaml` ```yaml description: Add index on manifests description field for faster searches @@ -47,28 +89,6 @@ query: | CREATE INDEX IF NOT EXISTS idx_manifests_description ON manifests(description); ``` -### Complex multi-statement migration: - -Filename: `0003_create_webhooks_table.yaml` - -```yaml -description: Create webhooks table for repository event notifications -query: | - -- Create webhooks table - CREATE TABLE IF NOT EXISTS webhooks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT NOT NULL, - events TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - - -- Create index on URL for faster lookups - CREATE INDEX IF NOT EXISTS idx_webhooks_url ON webhooks(url); - - -- Create index on events for filtering - CREATE INDEX IF NOT EXISTS idx_webhooks_events ON webhooks(events); -``` - ## How Migrations Run 1. Migrations are loaded from this directory on startup @@ -82,4 +102,6 @@ query: | - **Never modify existing migrations** - Once applied, they're immutable - **Test migrations** before committing - Ensure they work on existing databases - **Version numbers must be unique** - The migration system will fail if duplicates exist -- **Migrations are run automatically** on `InitDB()` - No manual intervention needed +- **Migrations run automatically** on `InitDB()` - Schema first, then migrations +- **CRITICAL: Update `schema.sql` for structural changes** - When you ALTER a table or add columns, update both the migration AND `schema.sql` so fresh installations have the same structure +- **New tables go in `schema.sql` only** - Don't create migration files for new tables diff --git a/pkg/appview/db/models.go b/pkg/appview/db/models.go index ba1e878..b81489c 100644 --- a/pkg/appview/db/models.go +++ b/pkg/appview/db/models.go @@ -29,6 +29,7 @@ type Manifest struct { DocumentationURL string Licenses string IconURL string + ReadmeURL string } // Layer represents a layer in a manifest @@ -94,6 +95,7 @@ type Repository struct { DocumentationURL string Licenses string IconURL string + ReadmeURL string } // RepositoryStats represents statistics for a repository diff --git a/pkg/appview/db/queries.go b/pkg/appview/db/queries.go index 175dbf7..9cd7206 100644 --- a/pkg/appview/db/queries.go +++ b/pkg/appview/db/queries.go @@ -310,23 +310,23 @@ func GetUserRepositories(db *sql.DB, did string) ([]Repository, error) { } // GetRepositoryMetadata retrieves metadata for a repository from its most recent manifest -func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, description, sourceURL, documentationURL, licenses, iconURL string, err error) { - var titleNull, descriptionNull, sourceURLNull, documentationURLNull, licensesNull, iconURLNull sql.NullString +func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string, err error) { + var titleNull, descriptionNull, sourceURLNull, documentationURLNull, licensesNull, iconURLNull, readmeURLNull sql.NullString err = db.QueryRow(` - SELECT title, description, source_url, documentation_url, licenses, icon_url + SELECT title, description, source_url, documentation_url, licenses, icon_url, readme_url FROM manifests WHERE did = ? AND repository = ? ORDER BY created_at DESC LIMIT 1 - `, did, repository).Scan(&titleNull, &descriptionNull, &sourceURLNull, &documentationURLNull, &licensesNull, &iconURLNull) + `, did, repository).Scan(&titleNull, &descriptionNull, &sourceURLNull, &documentationURLNull, &licensesNull, &iconURLNull, &readmeURLNull) if err == sql.ErrNoRows { // No manifests found - return empty strings - return "", "", "", "", "", "", nil + return "", "", "", "", "", "", "", nil } if err != nil { - return "", "", "", "", "", "", err + return "", "", "", "", "", "", "", err } // Convert NullString to string @@ -348,8 +348,11 @@ func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, de if iconURLNull.Valid { iconURL = iconURLNull.String } + if readmeURLNull.Valid { + readmeURL = readmeURLNull.String + } - return title, description, sourceURL, documentationURL, licenses, iconURL, nil + return title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, nil } // GetUserByDID retrieves a user by DID @@ -536,8 +539,8 @@ func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) { INSERT INTO manifests (did, repository, digest, hold_endpoint, schema_version, media_type, config_digest, config_size, created_at, - title, description, source_url, documentation_url, licenses, icon_url) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + title, description, source_url, documentation_url, licenses, icon_url, readme_url) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(did, repository, digest) DO UPDATE SET hold_endpoint = excluded.hold_endpoint, schema_version = excluded.schema_version, @@ -549,12 +552,13 @@ func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) { source_url = excluded.source_url, documentation_url = excluded.documentation_url, licenses = excluded.licenses, - icon_url = excluded.icon_url + icon_url = excluded.icon_url, + readme_url = excluded.readme_url `, manifest.DID, manifest.Repository, manifest.Digest, manifest.HoldEndpoint, manifest.SchemaVersion, manifest.MediaType, manifest.ConfigDigest, manifest.ConfigSize, manifest.CreatedAt, manifest.Title, manifest.Description, manifest.SourceURL, - manifest.DocumentationURL, manifest.Licenses, manifest.IconURL) + manifest.DocumentationURL, manifest.Licenses, manifest.IconURL, manifest.ReadmeURL) if err != nil { return 0, err diff --git a/pkg/appview/db/queries_test.go b/pkg/appview/db/queries_test.go index 14a6793..0ad76cf 100644 --- a/pkg/appview/db/queries_test.go +++ b/pkg/appview/db/queries_test.go @@ -26,11 +26,11 @@ func TestGetRepositoryMetadata(t *testing.T) { } // Test 1: No manifests - should return empty strings - title, description, sourceURL, documentationURL, licenses, iconURL, err := GetRepositoryMetadata(db, testUser.DID, "nonexistent") + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err := GetRepositoryMetadata(db, testUser.DID, "nonexistent") if err != nil { t.Fatalf("Expected no error for nonexistent repo, got: %v", err) } - if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" { + if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" || readmeURL != "" { t.Error("Expected all empty strings for nonexistent repository") } @@ -47,7 +47,7 @@ func TestGetRepositoryMetadata(t *testing.T) { } // Test 3: Retrieve metadata - title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp") + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp") if err != nil { t.Fatalf("Failed to get repository metadata: %v", err) } @@ -84,7 +84,7 @@ func TestGetRepositoryMetadata(t *testing.T) { } // Test 5: Should return metadata from most recent manifest - title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp") + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "myapp") if err != nil { t.Fatalf("Failed to get repository metadata: %v", err) } @@ -109,12 +109,12 @@ func TestGetRepositoryMetadata(t *testing.T) { } // Test 7: Should handle NULL fields gracefully - title, description, sourceURL, documentationURL, licenses, iconURL, err = GetRepositoryMetadata(db, testUser.DID, "minimal-app") + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err = GetRepositoryMetadata(db, testUser.DID, "minimal-app") if err != nil { t.Fatalf("Failed to get repository metadata for minimal app: %v", err) } - if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" { + if title != "" || description != "" || sourceURL != "" || documentationURL != "" || licenses != "" || iconURL != "" || readmeURL != "" { t.Error("Expected all empty strings for manifest with NULL metadata fields") } } diff --git a/pkg/appview/db/schema.go b/pkg/appview/db/schema.go index 6704ca1..0f01904 100644 --- a/pkg/appview/db/schema.go +++ b/pkg/appview/db/schema.go @@ -17,203 +17,8 @@ import ( //go:embed migrations/*.yaml var migrationsFS embed.FS -const schema = ` -CREATE TABLE IF NOT EXISTS schema_migrations ( - version INTEGER PRIMARY KEY, - applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE IF NOT EXISTS users ( - did TEXT PRIMARY KEY, - handle TEXT NOT NULL, - pds_endpoint TEXT NOT NULL, - avatar TEXT, - last_seen TIMESTAMP NOT NULL, - UNIQUE(handle) -); -CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle); - -CREATE TABLE IF NOT EXISTS manifests ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - did TEXT NOT NULL, - repository TEXT NOT NULL, - digest TEXT NOT NULL, - hold_endpoint TEXT NOT NULL, -- Stored as DID (e.g., did:web:hold.example.com) - schema_version INTEGER NOT NULL, - media_type TEXT NOT NULL, - config_digest TEXT, - config_size INTEGER, - created_at TIMESTAMP NOT NULL, - title TEXT, - description TEXT, - source_url TEXT, - documentation_url TEXT, - licenses TEXT, - icon_url TEXT, - UNIQUE(did, repository, digest), - FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository); -CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC); -CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest); - -CREATE TABLE IF NOT EXISTS layers ( - manifest_id INTEGER NOT NULL, - digest TEXT NOT NULL, - size INTEGER NOT NULL, - media_type TEXT NOT NULL, - layer_index INTEGER NOT NULL, - PRIMARY KEY(manifest_id, layer_index), - FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_layers_digest ON layers(digest); - -CREATE TABLE IF NOT EXISTS manifest_references ( - manifest_id INTEGER NOT NULL, - digest TEXT NOT NULL, - media_type TEXT NOT NULL, - size INTEGER NOT NULL, - platform_architecture TEXT, - platform_os TEXT, - platform_variant TEXT, - platform_os_version TEXT, - reference_index INTEGER NOT NULL, - PRIMARY KEY(manifest_id, reference_index), - FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest); - -CREATE TABLE IF NOT EXISTS tags ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - did TEXT NOT NULL, - repository TEXT NOT NULL, - tag TEXT NOT NULL, - digest TEXT NOT NULL, - created_at TIMESTAMP NOT NULL, - UNIQUE(did, repository, tag), - FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_tags_did_repo ON tags(did, repository); - -CREATE TABLE IF NOT EXISTS oauth_sessions ( - session_key TEXT PRIMARY KEY, - account_did TEXT NOT NULL, - session_id TEXT NOT NULL, - session_data TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(account_did, session_id) -); -CREATE INDEX IF NOT EXISTS idx_oauth_sessions_did ON oauth_sessions(account_did); -CREATE INDEX IF NOT EXISTS idx_oauth_sessions_updated ON oauth_sessions(updated_at DESC); - -CREATE TABLE IF NOT EXISTS oauth_auth_requests ( - state TEXT PRIMARY KEY, - request_data TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_oauth_auth_requests_created ON oauth_auth_requests(created_at); - -CREATE TABLE IF NOT EXISTS ui_sessions ( - id TEXT PRIMARY KEY, - did TEXT NOT NULL, - handle TEXT NOT NULL, - pds_endpoint TEXT NOT NULL, - oauth_session_id TEXT, - expires_at TIMESTAMP NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_ui_sessions_did ON ui_sessions(did); -CREATE INDEX IF NOT EXISTS idx_ui_sessions_expires ON ui_sessions(expires_at); - -CREATE TABLE IF NOT EXISTS devices ( - id TEXT PRIMARY KEY, - did TEXT NOT NULL, - handle TEXT NOT NULL, - name TEXT NOT NULL, - secret_hash TEXT NOT NULL UNIQUE, - ip_address TEXT, - location TEXT, - user_agent TEXT, - created_at TIMESTAMP NOT NULL, - last_used TIMESTAMP, - FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_devices_did ON devices(did); -CREATE INDEX IF NOT EXISTS idx_devices_hash ON devices(secret_hash); - -CREATE TABLE IF NOT EXISTS pending_device_auth ( - device_code TEXT PRIMARY KEY, - user_code TEXT NOT NULL UNIQUE, - device_name TEXT NOT NULL, - ip_address TEXT, - user_agent TEXT, - expires_at TIMESTAMP NOT NULL, - approved_did TEXT, - approved_at TIMESTAMP, - device_secret TEXT, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_pending_device_auth_user_code ON pending_device_auth(user_code); -CREATE INDEX IF NOT EXISTS idx_pending_device_auth_expires ON pending_device_auth(expires_at); - -CREATE TABLE IF NOT EXISTS repository_stats ( - did TEXT NOT NULL, - repository TEXT NOT NULL, - pull_count INTEGER NOT NULL DEFAULT 0, - last_pull TIMESTAMP, - push_count INTEGER NOT NULL DEFAULT 0, - last_push TIMESTAMP, - PRIMARY KEY(did, repository), - FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did); -CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC); - -CREATE TABLE IF NOT EXISTS stars ( - starrer_did TEXT NOT NULL, - owner_did TEXT NOT NULL, - repository TEXT NOT NULL, - created_at TIMESTAMP NOT NULL, - PRIMARY KEY(starrer_did, owner_did, repository), - FOREIGN KEY(starrer_did) REFERENCES users(did) ON DELETE CASCADE, - FOREIGN KEY(owner_did) REFERENCES users(did) ON DELETE CASCADE -); -CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository); -CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did); - -CREATE TABLE IF NOT EXISTS hold_captain_records ( - hold_did TEXT PRIMARY KEY, - owner_did TEXT NOT NULL, - public BOOLEAN NOT NULL, - allow_all_crew BOOLEAN NOT NULL, - deployed_at TEXT, - region TEXT, - provider TEXT, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at); - -CREATE TABLE IF NOT EXISTS hold_crew_approvals ( - hold_did TEXT NOT NULL, - user_did TEXT NOT NULL, - approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP NOT NULL, - PRIMARY KEY(hold_did, user_did) -); -CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at); - -CREATE TABLE IF NOT EXISTS hold_crew_denials ( - hold_did TEXT NOT NULL, - user_did TEXT NOT NULL, - denial_count INTEGER NOT NULL DEFAULT 1, - next_retry_at TIMESTAMP NOT NULL, - last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(hold_did, user_did) -); -CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at); -` +//go:embed schema.sql +var schemaSQL string // InitDB initializes the SQLite database with the schema func InitDB(path string) (*sql.DB, error) { @@ -227,8 +32,8 @@ func InitDB(path string) (*sql.DB, error) { return nil, err } - // Create schema - if _, err := db.Exec(schema); err != nil { + // Create schema from embedded SQL file + if _, err := db.Exec(schemaSQL); err != nil { return nil, err } diff --git a/pkg/appview/db/schema.sql b/pkg/appview/db/schema.sql new file mode 100644 index 0000000..bd009c7 --- /dev/null +++ b/pkg/appview/db/schema.sql @@ -0,0 +1,207 @@ +-- ATCR AppView Database Schema +-- This file contains the complete base schema for fresh database installations. +-- Migrations (in migrations/*.yaml) handle changes to existing databases. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS users ( + did TEXT PRIMARY KEY, + handle TEXT NOT NULL, + pds_endpoint TEXT NOT NULL, + avatar TEXT, + last_seen TIMESTAMP NOT NULL, + UNIQUE(handle) +); +CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle); + +CREATE TABLE IF NOT EXISTS manifests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + repository TEXT NOT NULL, + digest TEXT NOT NULL, + hold_endpoint TEXT NOT NULL, -- Stored as DID (e.g., did:web:hold.example.com) + schema_version INTEGER NOT NULL, + media_type TEXT NOT NULL, + config_digest TEXT, + config_size INTEGER, + created_at TIMESTAMP NOT NULL, + title TEXT, + description TEXT, + source_url TEXT, + documentation_url TEXT, + licenses TEXT, + icon_url TEXT, + readme_url TEXT, + UNIQUE(did, repository, digest), + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository); +CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest); + +CREATE TABLE IF NOT EXISTS layers ( + manifest_id INTEGER NOT NULL, + digest TEXT NOT NULL, + size INTEGER NOT NULL, + media_type TEXT NOT NULL, + layer_index INTEGER NOT NULL, + PRIMARY KEY(manifest_id, layer_index), + FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_layers_digest ON layers(digest); + +CREATE TABLE IF NOT EXISTS manifest_references ( + manifest_id INTEGER NOT NULL, + digest TEXT NOT NULL, + media_type TEXT NOT NULL, + size INTEGER NOT NULL, + platform_architecture TEXT, + platform_os TEXT, + platform_variant TEXT, + platform_os_version TEXT, + reference_index INTEGER NOT NULL, + PRIMARY KEY(manifest_id, reference_index), + FOREIGN KEY(manifest_id) REFERENCES manifests(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_manifest_references_digest ON manifest_references(digest); + +CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + repository TEXT NOT NULL, + tag TEXT NOT NULL, + digest TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + UNIQUE(did, repository, tag), + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_tags_did_repo ON tags(did, repository); + +CREATE TABLE IF NOT EXISTS oauth_sessions ( + session_key TEXT PRIMARY KEY, + account_did TEXT NOT NULL, + session_id TEXT NOT NULL, + session_data TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(account_did, session_id) +); +CREATE INDEX IF NOT EXISTS idx_oauth_sessions_did ON oauth_sessions(account_did); +CREATE INDEX IF NOT EXISTS idx_oauth_sessions_updated ON oauth_sessions(updated_at DESC); + +CREATE TABLE IF NOT EXISTS oauth_auth_requests ( + state TEXT PRIMARY KEY, + request_data TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_oauth_auth_requests_created ON oauth_auth_requests(created_at); + +CREATE TABLE IF NOT EXISTS ui_sessions ( + id TEXT PRIMARY KEY, + did TEXT NOT NULL, + handle TEXT NOT NULL, + pds_endpoint TEXT NOT NULL, + oauth_session_id TEXT, + expires_at TIMESTAMP NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_ui_sessions_did ON ui_sessions(did); +CREATE INDEX IF NOT EXISTS idx_ui_sessions_expires ON ui_sessions(expires_at); + +CREATE TABLE IF NOT EXISTS devices ( + id TEXT PRIMARY KEY, + did TEXT NOT NULL, + handle TEXT NOT NULL, + name TEXT NOT NULL, + secret_hash TEXT NOT NULL UNIQUE, + ip_address TEXT, + location TEXT, + user_agent TEXT, + created_at TIMESTAMP NOT NULL, + last_used TIMESTAMP, + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_devices_did ON devices(did); +CREATE INDEX IF NOT EXISTS idx_devices_hash ON devices(secret_hash); + +CREATE TABLE IF NOT EXISTS pending_device_auth ( + device_code TEXT PRIMARY KEY, + user_code TEXT NOT NULL UNIQUE, + device_name TEXT NOT NULL, + ip_address TEXT, + user_agent TEXT, + expires_at TIMESTAMP NOT NULL, + approved_did TEXT, + approved_at TIMESTAMP, + device_secret TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_pending_device_auth_user_code ON pending_device_auth(user_code); +CREATE INDEX IF NOT EXISTS idx_pending_device_auth_expires ON pending_device_auth(expires_at); + +CREATE TABLE IF NOT EXISTS repository_stats ( + did TEXT NOT NULL, + repository TEXT NOT NULL, + pull_count INTEGER NOT NULL DEFAULT 0, + last_pull TIMESTAMP, + push_count INTEGER NOT NULL DEFAULT 0, + last_push TIMESTAMP, + PRIMARY KEY(did, repository), + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did); +CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC); + +CREATE TABLE IF NOT EXISTS stars ( + starrer_did TEXT NOT NULL, + owner_did TEXT NOT NULL, + repository TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY(starrer_did, owner_did, repository), + FOREIGN KEY(starrer_did) REFERENCES users(did) ON DELETE CASCADE, + FOREIGN KEY(owner_did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository); +CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did); + +CREATE TABLE IF NOT EXISTS hold_captain_records ( + hold_did TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + public BOOLEAN NOT NULL, + allow_all_crew BOOLEAN NOT NULL, + deployed_at TEXT, + region TEXT, + provider TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at); + +CREATE TABLE IF NOT EXISTS hold_crew_approvals ( + hold_did TEXT NOT NULL, + user_did TEXT NOT NULL, + approved_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMP NOT NULL, + PRIMARY KEY(hold_did, user_did) +); +CREATE INDEX IF NOT EXISTS idx_crew_approvals_expires ON hold_crew_approvals(expires_at); + +CREATE TABLE IF NOT EXISTS hold_crew_denials ( + hold_did TEXT NOT NULL, + user_did TEXT NOT NULL, + denial_count INTEGER NOT NULL DEFAULT 1, + next_retry_at TIMESTAMP NOT NULL, + last_denied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(hold_did, user_did) +); +CREATE INDEX IF NOT EXISTS idx_crew_denials_retry ON hold_crew_denials(next_retry_at); + +CREATE TABLE IF NOT EXISTS readme_cache ( + url TEXT PRIMARY KEY, + html TEXT NOT NULL, + fetched_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_readme_cache_fetched ON readme_cache(fetched_at); diff --git a/pkg/appview/handlers/repository.go b/pkg/appview/handlers/repository.go index 168b6e5..a9b6bb4 100644 --- a/pkg/appview/handlers/repository.go +++ b/pkg/appview/handlers/repository.go @@ -12,6 +12,7 @@ import ( "atcr.io/pkg/appview/db" "atcr.io/pkg/appview/holdhealth" "atcr.io/pkg/appview/middleware" + "atcr.io/pkg/appview/readme" "atcr.io/pkg/atproto" "atcr.io/pkg/auth/oauth" "github.com/bluesky-social/indigo/atproto/identity" @@ -26,6 +27,7 @@ type RepositoryPageHandler struct { Directory identity.Directory Refresher *oauth.Refresher HealthChecker *holdhealth.Checker + ReadmeCache *readme.Cache } func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -135,7 +137,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request } // Fetch repository metadata from most recent manifest - title, description, sourceURL, documentationURL, licenses, iconURL, err := db.GetRepositoryMetadata(h.DB, owner.DID, repository) + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, err := db.GetRepositoryMetadata(h.DB, owner.DID, repository) if err != nil { log.Printf("Failed to fetch repository metadata: %v", err) // Continue without metadata on error @@ -146,6 +148,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request repo.DocumentationURL = documentationURL repo.Licenses = licenses repo.IconURL = iconURL + repo.ReadmeURL = readmeURL } // Fetch star count @@ -180,6 +183,22 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request isOwner = (user.DID == owner.DID) } + // Fetch README content if available + var readmeHTML template.HTML + if repo.ReadmeURL != "" && h.ReadmeCache != nil { + // Fetch with timeout + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + html, err := h.ReadmeCache.Get(ctx, repo.ReadmeURL) + if err != nil { + log.Printf("Failed to fetch README from %s: %v", repo.ReadmeURL, err) + // Continue without README on error + } else { + readmeHTML = template.HTML(html) + } + } + data := struct { PageData Owner *db.User // Repository owner @@ -189,6 +208,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request StarCount int IsStarred bool IsOwner bool // Whether current user owns this repository + ReadmeHTML template.HTML }{ PageData: NewPageData(r, h.RegistryURL), Owner: owner, @@ -198,6 +218,7 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request StarCount: stats.StarCount, IsStarred: isStarred, IsOwner: isOwner, + ReadmeHTML: readmeHTML, } if err := h.Templates.ExecuteTemplate(w, "repository", data); err != nil { diff --git a/pkg/appview/jetstream/backfill.go b/pkg/appview/jetstream/backfill.go index fe71e17..b997ed5 100644 --- a/pkg/appview/jetstream/backfill.go +++ b/pkg/appview/jetstream/backfill.go @@ -298,7 +298,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor } // Extract OCI annotations from manifest - var title, description, sourceURL, documentationURL, licenses, iconURL string + var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string if manifestRecord.Annotations != nil { title = manifestRecord.Annotations["org.opencontainers.image.title"] description = manifestRecord.Annotations["org.opencontainers.image.description"] @@ -306,6 +306,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] iconURL = manifestRecord.Annotations["io.atcr.icon"] + readmeURL = manifestRecord.Annotations["io.atcr.readme"] } // Detect manifest type @@ -326,6 +327,7 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor DocumentationURL: documentationURL, Licenses: licenses, IconURL: iconURL, + ReadmeURL: readmeURL, } // Set config fields only for image manifests (not manifest lists) diff --git a/pkg/appview/jetstream/worker.go b/pkg/appview/jetstream/worker.go index 923be67..26c503c 100644 --- a/pkg/appview/jetstream/worker.go +++ b/pkg/appview/jetstream/worker.go @@ -442,7 +442,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error { } // Extract OCI annotations from manifest - var title, description, sourceURL, documentationURL, licenses, iconURL string + var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string if manifestRecord.Annotations != nil { title = manifestRecord.Annotations["org.opencontainers.image.title"] description = manifestRecord.Annotations["org.opencontainers.image.description"] @@ -450,6 +450,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error { documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] iconURL = manifestRecord.Annotations["io.atcr.icon"] + readmeURL = manifestRecord.Annotations["io.atcr.readme"] } // Detect manifest type @@ -470,6 +471,7 @@ func (w *Worker) processManifest(commit *CommitEvent) error { DocumentationURL: documentationURL, Licenses: licenses, IconURL: iconURL, + ReadmeURL: readmeURL, } // Set config fields only for image manifests (not manifest lists) diff --git a/pkg/appview/readme/cache.go b/pkg/appview/readme/cache.go new file mode 100644 index 0000000..bb12753 --- /dev/null +++ b/pkg/appview/readme/cache.go @@ -0,0 +1,108 @@ +package readme + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// Cache stores rendered README HTML in the database +type Cache struct { + db *sql.DB + fetcher *Fetcher + ttl time.Duration +} + +// NewCache creates a new README cache +func NewCache(db *sql.DB, ttl time.Duration) *Cache { + if ttl == 0 { + ttl = 1 * time.Hour // Default TTL + } + return &Cache{ + db: db, + fetcher: NewFetcher(), + ttl: ttl, + } +} + +// Get retrieves a README from cache or fetches it +func (c *Cache) Get(ctx context.Context, readmeURL string) (string, error) { + // Try to get from cache + html, fetchedAt, err := c.getFromDB(readmeURL) + if err == nil { + // Check if cache is still valid + if time.Since(fetchedAt) < c.ttl { + return html, nil + } + } + + // Cache miss or expired, fetch fresh content + html, err = c.fetcher.FetchAndRender(ctx, readmeURL) + if err != nil { + // If fetch fails but we have stale cache, return it + if html != "" { + return html, nil + } + return "", err + } + + // Store in cache + if err := c.storeInDB(readmeURL, html); err != nil { + // Log error but don't fail - we have the content + // In production, you'd use proper logging here + fmt.Printf("Failed to cache README: %v\n", err) + } + + return html, nil +} + +// getFromDB retrieves cached README from database +func (c *Cache) getFromDB(readmeURL string) (string, time.Time, error) { + var html string + var fetchedAt time.Time + + err := c.db.QueryRow(` + SELECT html, fetched_at + FROM readme_cache + WHERE url = ? + `, readmeURL).Scan(&html, &fetchedAt) + + if err != nil { + return "", time.Time{}, err + } + + return html, fetchedAt, nil +} + +// storeInDB stores rendered README in database +func (c *Cache) storeInDB(readmeURL, html string) error { + _, err := c.db.Exec(` + INSERT INTO readme_cache (url, html, fetched_at) + VALUES (?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + html = excluded.html, + fetched_at = excluded.fetched_at + `, readmeURL, html, time.Now()) + + return err +} + +// Invalidate removes a README from the cache +func (c *Cache) Invalidate(readmeURL string) error { + _, err := c.db.Exec(` + DELETE FROM readme_cache + WHERE url = ? + `, readmeURL) + return err +} + +// Cleanup removes expired entries from the cache +func (c *Cache) Cleanup() error { + cutoff := time.Now().Add(-c.ttl * 2) // Keep for 2x TTL + _, err := c.db.Exec(` + DELETE FROM readme_cache + WHERE fetched_at < ? + `, cutoff) + return err +} diff --git a/pkg/appview/readme/fetcher.go b/pkg/appview/readme/fetcher.go new file mode 100644 index 0000000..a0b48de --- /dev/null +++ b/pkg/appview/readme/fetcher.go @@ -0,0 +1,210 @@ +package readme + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/microcosm-cc/bluemonday" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer/html" +) + +// Fetcher fetches and renders README content from URLs +type Fetcher struct { + httpClient *http.Client + markdown goldmark.Markdown + sanitizer *bluemonday.Policy +} + +// NewFetcher creates a new README fetcher +func NewFetcher() *Fetcher { + // Configure markdown renderer with GitHub-flavored markdown + md := goldmark.New( + goldmark.WithExtensions( + extension.GFM, // GitHub Flavored Markdown + extension.Typographer, // Smart quotes, dashes, etc. + ), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), // Auto-generate heading IDs + ), + goldmark.WithRendererOptions( + html.WithHardWraps(), // Line breaks create
+ html.WithXHTML(), // XHTML-compliant output + // html.WithUnsafe(), // Uncomment ONLY if you want to allow raw HTML in markdown (not recommended) + ), + ) + + // Configure HTML sanitizer as a safety net + // This catches any HTML that makes it through (if WithUnsafe() is enabled) + sanitizer := bluemonday.UGCPolicy() + // Allow additional attributes for better markdown rendering + sanitizer.AllowAttrs("class").Globally() + sanitizer.AllowAttrs("id").Globally() + sanitizer.AllowAttrs("align").OnElements("img", "div", "p", "span") + + return &Fetcher{ + httpClient: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // Allow up to 5 redirects + if len(via) >= 5 { + return fmt.Errorf("too many redirects") + } + return nil + }, + }, + markdown: md, + sanitizer: sanitizer, + } +} + +// FetchAndRender fetches a README from a URL and renders it as HTML +// Returns the rendered HTML and any error +func (f *Fetcher) FetchAndRender(ctx context.Context, readmeURL string) (string, error) { + // Validate URL + if readmeURL == "" { + return "", fmt.Errorf("empty README URL") + } + + parsedURL, err := url.Parse(readmeURL) + if err != nil { + return "", fmt.Errorf("invalid README URL: %w", err) + } + + // Only allow HTTP/HTTPS + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return "", fmt.Errorf("invalid URL scheme: %s", parsedURL.Scheme) + } + + // Fetch content + content, baseURL, err := f.fetchContent(ctx, readmeURL) + if err != nil { + return "", err + } + + // Render markdown to HTML + html, err := f.renderMarkdown(content, baseURL) + if err != nil { + return "", fmt.Errorf("failed to render markdown: %w", err) + } + + return html, nil +} + +// fetchContent fetches the raw content from a URL +func (f *Fetcher) fetchContent(ctx context.Context, urlStr string) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, "", fmt.Errorf("failed to create request: %w", err) + } + + // Set user agent + req.Header.Set("User-Agent", "ATCR-README-Fetcher/1.0") + + resp, err := f.httpClient.Do(req) + if err != nil { + return nil, "", fmt.Errorf("failed to fetch URL: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + // Limit content size to 1MB + limitedReader := io.LimitReader(resp.Body, 1*1024*1024) + content, err := io.ReadAll(limitedReader) + if err != nil { + return nil, "", fmt.Errorf("failed to read response body: %w", err) + } + + // Get base URL for relative link resolution + baseURL := getBaseURL(resp.Request.URL) + + return content, baseURL, nil +} + +// renderMarkdown renders markdown content to sanitized HTML +func (f *Fetcher) renderMarkdown(content []byte, baseURL string) (string, error) { + var buf bytes.Buffer + + if err := f.markdown.Convert(content, &buf); err != nil { + return "", err + } + + // Rewrite relative URLs to absolute + html := buf.String() + if baseURL != "" { + html = rewriteRelativeURLs(html, baseURL) + } + + // Sanitize HTML + sanitized := f.sanitizer.Sanitize(html) + + return sanitized, nil +} + +// getBaseURL extracts the base URL for relative link resolution +func getBaseURL(u *url.URL) string { + if u == nil { + return "" + } + + // For GitHub raw URLs, convert to blob URL base for relative links + // e.g., https://raw.githubusercontent.com/user/repo/main/README.md + // -> https://github.com/user/repo/blob/main/ + if u.Host == "raw.githubusercontent.com" { + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + if len(parts) >= 3 { + user := parts[0] + repo := parts[1] + branch := parts[2] + return fmt.Sprintf("https://github.com/%s/%s/blob/%s/", user, repo, branch) + } + } + + // For other URLs, use the directory containing the file + path := u.Path + lastSlash := strings.LastIndex(path, "/") + if lastSlash >= 0 { + path = path[:lastSlash+1] + } + return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, path) +} + +// rewriteRelativeURLs converts relative URLs to absolute URLs +func rewriteRelativeURLs(html, baseURL string) string { + if baseURL == "" { + return html + } + + base, err := url.Parse(baseURL) + if err != nil { + return html + } + + // Simple string replacement for common patterns + // This is a basic implementation - for production, consider using an HTML parser + html = strings.ReplaceAll(html, `src="./`, fmt.Sprintf(`src="%s`, baseURL)) + html = strings.ReplaceAll(html, `href="./`, fmt.Sprintf(`href="%s`, baseURL)) + html = strings.ReplaceAll(html, `src="../`, fmt.Sprintf(`src="%s../`, baseURL)) + html = strings.ReplaceAll(html, `href="../`, fmt.Sprintf(`href="%s../`, baseURL)) + + // Handle root-relative URLs (starting with /) + if base.Scheme != "" && base.Host != "" { + root := fmt.Sprintf("%s://%s/", base.Scheme, base.Host) + // Replace src="/" and href="/" but not src="//" (absolute URLs) + html = strings.ReplaceAll(html, `src="/`, fmt.Sprintf(`src="%s`, root)) + html = strings.ReplaceAll(html, `href="/`, fmt.Sprintf(`href="%s`, root)) + } + + return html +} diff --git a/pkg/appview/static/css/style.css b/pkg/appview/static/css/style.css index d8462b3..c09fc3f 100644 --- a/pkg/appview/static/css/style.css +++ b/pkg/appview/static/css/style.css @@ -1639,3 +1639,216 @@ button:hover, .btn:hover, .btn-primary:hover, .btn-secondary:hover { grid-template-columns: repeat(3, 1fr); } } + +/* README and Repository Layout */ +.repo-content-layout { + display: grid; + grid-template-columns: 1fr 400px; + gap: 2rem; + margin-top: 2rem; +} + +.readme-section { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 2rem; +} + +.readme-section h2 { + margin-bottom: 1.5rem; + padding-bottom: 0.5rem; + border-bottom: 2px solid var(--border); +} + +.readme-content { + overflow-wrap: break-word; +} + +.repo-sidebar { + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +/* Markdown Styling */ +.markdown-body { + font-size: 1rem; + line-height: 1.6; + word-wrap: break-word; +} + +.markdown-body h1, +.markdown-body h2, +.markdown-body h3, +.markdown-body h4, +.markdown-body h5, +.markdown-body h6 { + margin-top: 1.5rem; + margin-bottom: 1rem; + font-weight: 600; + line-height: 1.25; +} + +.markdown-body h1 { + font-size: 2rem; + border-bottom: 1px solid var(--border); + padding-bottom: 0.3rem; +} + +.markdown-body h2 { + font-size: 1.5rem; + border-bottom: 1px solid var(--border); + padding-bottom: 0.3rem; +} + +.markdown-body h3 { + font-size: 1.25rem; +} + +.markdown-body h4 { + font-size: 1rem; +} + +.markdown-body h5 { + font-size: 0.875rem; +} + +.markdown-body h6 { + font-size: 0.85rem; + color: var(--secondary); +} + +.markdown-body p { + margin-bottom: 1rem; +} + +.markdown-body ul, +.markdown-body ol { + margin-bottom: 1rem; + padding-left: 2rem; +} + +.markdown-body li { + margin-bottom: 0.25rem; +} + +.markdown-body li > p { + margin-bottom: 0.5rem; +} + +.markdown-body a { + color: var(--primary); + text-decoration: none; +} + +.markdown-body a:hover { + text-decoration: underline; +} + +.markdown-body code { + background: var(--code-bg); + padding: 0.2rem 0.4rem; + border-radius: 3px; + font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; + font-size: 0.9em; +} + +.markdown-body pre { + background: var(--code-bg); + padding: 1rem; + border-radius: 6px; + overflow-x: auto; + margin-bottom: 1rem; +} + +.markdown-body pre code { + background: none; + padding: 0; + font-size: 0.875rem; +} + +.markdown-body blockquote { + padding: 0 1rem; + margin-bottom: 1rem; + color: var(--secondary); + border-left: 4px solid var(--border); +} + +.markdown-body table { + border-collapse: collapse; + width: 100%; + margin-bottom: 1rem; +} + +.markdown-body table th, +.markdown-body table td { + padding: 0.5rem 1rem; + border: 1px solid var(--border); + text-align: left; +} + +.markdown-body table th { + background: var(--code-bg); + font-weight: 600; +} + +.markdown-body table tr:nth-child(even) { + background: var(--hover-bg); +} + +.markdown-body img { + max-width: 100%; + height: auto; + margin: 1rem 0; +} + +.markdown-body hr { + height: 0.25rem; + margin: 1.5rem 0; + background: var(--border); + border: 0; +} + +/* Task lists */ +.markdown-body input[type="checkbox"] { + margin-right: 0.5rem; +} + +.markdown-body .task-list-item { + list-style-type: none; +} + +.markdown-body .task-list-item input { + margin: 0 0.2rem 0.25rem -1.6rem; + vertical-align: middle; +} + +/* Responsive Layout */ +@media (max-width: 1024px) { + .repo-content-layout { + grid-template-columns: 1fr; + } + + .repo-sidebar { + order: -1; /* Show sidebar first on mobile */ + } +} + +@media (max-width: 768px) { + .readme-section { + padding: 1rem; + } + + .markdown-body h1 { + font-size: 1.5rem; + } + + .markdown-body h2 { + font-size: 1.25rem; + } + + .markdown-body pre { + padding: 0.75rem; + } +} diff --git a/pkg/appview/templates/pages/repository.html b/pkg/appview/templates/pages/repository.html index ab06ff2..5f9bcc1 100644 --- a/pkg/appview/templates/pages/repository.html +++ b/pkg/appview/templates/pages/repository.html @@ -83,6 +83,21 @@ + + {{ if .ReadmeHTML }} +
+ +
+

Overview

+
+ {{ .ReadmeHTML }} +
+
+ + +
+ {{ end }} +

Tags

@@ -213,6 +228,11 @@

No manifests available

{{ end }}
+ + {{ if .ReadmeHTML }} +
+
+ {{ end }}