From 454298645ce8928bc50a31c1c3adcde04cc99fae Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Wed, 8 Oct 2025 12:31:51 -0500 Subject: [PATCH] implement stars and pull tracking rename registry -> appview --- CLAUDE.md | 8 +- Dockerfile | 12 +- README.md | 30 +-- SPEC.md | 6 +- cmd/{registry => appview}/main.go | 0 cmd/{registry => appview}/serve.go | 37 ++++ config/config.yml | 4 +- docker-compose.yml | 8 +- docs/API_KEY_MIGRATION.md | 8 +- docs/APPVIEW-UI-IMPLEMENTATION.md | 18 +- docs/TESTING.md | 12 +- lexicons/io/atcr/manifest.json | 4 + lexicons/io/atcr/sailor/star.json | 44 ++++ pkg/appview/db/models.go | 12 +- pkg/appview/db/queries.go | 157 +++++++++++++- pkg/appview/db/schema.go | 51 +++-- pkg/appview/handlers/api.go | 226 ++++++++++++++++++++ pkg/appview/interfaces.go | 50 ----- pkg/appview/jetstream/backfill.go | 54 +++-- pkg/appview/jetstream/worker.go | 56 ++++- pkg/appview/templates/components/modal.html | 3 - pkg/atproto/lexicon.go | 89 ++++---- pkg/atproto/manifest_store.go | 25 ++- pkg/middleware/registry.go | 16 +- pkg/storage/proxy_blob_store.go | 19 +- pkg/storage/routing_repository.go | 13 +- test-registry.sh | 10 +- 27 files changed, 751 insertions(+), 221 deletions(-) rename cmd/{registry => appview}/main.go (100%) rename cmd/{registry => appview}/serve.go (93%) create mode 100644 lexicons/io/atcr/sailor/star.json create mode 100644 pkg/appview/handlers/api.go delete mode 100644 pkg/appview/interfaces.go diff --git a/CLAUDE.md b/CLAUDE.md index 13ab2af..016e36a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ ATCR (ATProto Container Registry) is an OCI-compliant container registry that us ```bash # Build all binaries # create go builds in the bin/ directory -go build -o bin/atcr-registry ./cmd/registry +go build -o bin/atcr-appview ./cmd/appview go build -o bin/atcr-hold ./cmd/hold go build -o bin/docker-credential-atcr ./cmd/credential-helper @@ -25,7 +25,7 @@ go test -race ./... go mod tidy # Build Docker images -docker build -t atcr.io/registry:latest . +docker build -t atcr.io/appview:latest . docker build -f Dockerfile.hold -t atcr.io/hold:latest . # Or use docker-compose @@ -34,7 +34,7 @@ docker-compose up -d # Run locally (AppView) export ATPROTO_DID=did:plc:your-did export ATPROTO_ACCESS_TOKEN=your-token -./atcr-registry serve config/config.yml +./atcr-appview serve config/config.yml # Run hold service (configure via env vars - see .env.example) export HOLD_PUBLIC_URL=http://127.0.0.1:8080 @@ -57,7 +57,7 @@ ATCR uses **distribution/distribution** as a library and extends it through midd ### Three-Component Architecture -1. **AppView** (`cmd/registry`) - OCI Distribution API server +1. **AppView** (`cmd/appview`) - OCI Distribution API server - Resolves identities (handle/DID → PDS endpoint) - Routes manifests to user's PDS - Routes blobs to storage endpoint (default or BYOS) diff --git a/Dockerfile b/Dockerfile index f7808f3..c3d3a07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN go mod download COPY . . # Build the binary with CGO enabled for SQLite support -RUN CGO_ENABLED=1 GOOS=linux go build -a -o atcr-registry ./cmd/registry +RUN CGO_ENABLED=1 GOOS=linux go build -a -o atcr-appview ./cmd/appview # Runtime stage FROM alpine:latest @@ -29,7 +29,7 @@ RUN apk --no-cache add ca-certificates sqlite-libs sqlite WORKDIR /app # Copy binary from builder -COPY --from=builder /build/atcr-registry . +COPY --from=builder /build/atcr-appview . # Copy default configuration COPY config/config.yml /etc/atcr/config.yml @@ -44,15 +44,15 @@ EXPOSE 5000 5001 ENV ATCR_CONFIG=/etc/atcr/config.yml # OCI image annotations -LABEL org.opencontainers.image.title="ATCR Registry" \ +LABEL org.opencontainers.image.title="ATCR AppView" \ org.opencontainers.image.description="ATProto Container Registry - OCI-compliant registry using AT Protocol for manifest storage" \ org.opencontainers.image.authors="ATCR Contributors" \ org.opencontainers.image.source="https://github.com/example/atcr" \ org.opencontainers.image.documentation="https://atcr.io/docs" \ org.opencontainers.image.licenses="MIT" \ org.opencontainers.image.version="0.1.0" \ - io.atcr.icon="https://atcr.io/images/registry-icon.png" + io.atcr.icon="https://atcr.io/images/appview-icon.png" -# Run the registry -ENTRYPOINT ["/app/atcr-registry"] +# Run the AppView +ENTRYPOINT ["/app/atcr-appview"] CMD ["serve", "/etc/atcr/config.yml"] diff --git a/README.md b/README.md index c3c5054..e9b57a6 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ ATCR is an OCI-compliant container registry that integrates with the AT Protocol ```bash # Build all binaries locally -go build -o atcr-registry ./cmd/registry +go build -o atcr-appview ./cmd/appview go build -o atcr-hold ./cmd/hold go build -o docker-credential-atcr ./cmd/credential-helper # Build Docker images -docker build -t atcr.io/registry:latest . +docker build -t atcr.io/appview:latest . docker build -f Dockerfile.hold -t atcr.io/hold:latest . ``` @@ -56,7 +56,7 @@ sudo mkdir -p /var/lib/atcr/{blobs,hold,auth} sudo chown -R $USER:$USER /var/lib/atcr # 2. Build binaries -go build -o atcr-registry ./cmd/registry +go build -o atcr-appview ./cmd/appview go build -o atcr-hold ./cmd/hold # 3. Configure environment @@ -66,7 +66,7 @@ export $(cat .env | xargs) # 4. Start services # Terminal 1: -./atcr-registry serve config/config.yml +./atcr-appview serve config/config.yml # Terminal 2 (will prompt for OAuth): ./atcr-hold config/hold.yml # Follow OAuth URL in logs to authorize @@ -94,9 +94,9 @@ cp .env.example .env export $(cat .env | xargs) ``` -**AppView (Registry):** +**AppView:** ```bash -./atcr-registry serve config/config.yml +./atcr-appview serve config/config.yml ``` **Hold (Storage Service):** @@ -115,17 +115,17 @@ docker-compose up -d **Or run containers separately:** -**AppView (Registry):** +**AppView:** ```bash docker run -d \ - --name atcr-registry \ + --name atcr-appview \ -p 5000:5000 \ -e ATPROTO_DID=did:plc:your-did \ -e ATPROTO_ACCESS_TOKEN=your-access-token \ -e AWS_ACCESS_KEY_ID=your-aws-key \ -e AWS_SECRET_ACCESS_KEY=your-aws-secret \ -v $(pwd)/config/config.yml:/etc/atcr/config.yml \ - atcr.io/registry:latest + atcr.io/appview:latest ``` **Hold (Storage Service):** @@ -145,20 +145,20 @@ docker run -d \ apiVersion: apps/v1 kind: Deployment metadata: - name: atcr-registry + name: atcr-appview spec: replicas: 3 selector: matchLabels: - app: atcr-registry + app: atcr-appview template: metadata: labels: - app: atcr-registry + app: atcr-appview spec: containers: - - name: registry - image: atcr.io/registry:latest + - name: appview + image: atcr.io/appview:latest ports: - containerPort: 5000 env: @@ -215,7 +215,7 @@ docker pull atcr.io/alice/myapp:latest ``` atcr.io/ -├── cmd/registry/ # Main entrypoint +├── cmd/appview/ # AppView entrypoint ├── pkg/ │ ├── atproto/ # ATProto client and manifest store │ ├── storage/ # S3 blob store and routing diff --git a/SPEC.md b/SPEC.md index 9884f3c..b24eeb9 100644 --- a/SPEC.md +++ b/SPEC.md @@ -35,7 +35,7 @@ ATProto Container Registry (atcr.io) Implementation Plan 1. Initialize Go module with github.com/distribution/distribution/v3 and github.com/bluesky-social/indigo 2. Create basic project structure - 3. Set up cmd/registry/main.go that imports distribution and registers middleware + 3. Set up cmd/appview/main.go that imports distribution and registers middleware Phase 2: Core ATProto Integration @@ -78,7 +78,7 @@ ATProto Container Registry (atcr.io) Implementation Plan Phase 6: Configuration & Deployment 13. Create registry configuration (config/config.yml) - 14. Create Dockerfile for building atcr-registry binary + 14. Create Dockerfile for building atcr-appview binary 16. Write README.md with usage instructions Phase 7: Documentation @@ -193,7 +193,7 @@ Perfect. To match Docker Hub/ghcr.io/gcr.io, here's what we need: atproto: # Used by auth service to validate credentials pds_endpoint: https://bsky.social - client_id: atcr-registry + client_id: atcr-appview oauth_redirect: http://localhost:8888/callback ATProto OAuth Implementation Plan diff --git a/cmd/registry/main.go b/cmd/appview/main.go similarity index 100% rename from cmd/registry/main.go rename to cmd/appview/main.go diff --git a/cmd/registry/serve.go b/cmd/appview/serve.go similarity index 93% rename from cmd/registry/serve.go rename to cmd/appview/serve.go index 48a1f50..51bc487 100644 --- a/cmd/registry/serve.go +++ b/cmd/appview/serve.go @@ -107,6 +107,10 @@ func serveRegistry(cmd *cobra.Command, args []string) error { // 6. Set global refresher for middleware middleware.SetGlobalRefresher(refresher) + // 6.5. Set global database for pull/push metrics tracking + metricsDB := db.NewMetricsDB(uiDatabase) + middleware.SetGlobalDatabase(metricsDB) + // 7. Initialize UI routes with OAuth app, refresher, and device store uiTemplates, uiRouter := initializeUIRoutes(uiDatabase, uiSessionStore, oauthApp, refresher, baseURL, deviceStore) @@ -422,6 +426,39 @@ func initializeUIRoutes(database *sql.DB, sessionStore *db.SessionStore, oauthAp }, )).Methods("GET") + // API route for repository stats (public) + router.Handle("/api/stats/{handle}/{repository}", appmiddleware.OptionalAuth(sessionStore, database)( + &uihandlers.GetStatsHandler{ + DB: database, + Directory: oauthApp.Directory(), + }, + )).Methods("GET") + + // API routes for stars (require authentication) + router.Handle("/api/stars/{handle}/{repository}", appmiddleware.RequireAuth(sessionStore, database)( + &uihandlers.StarRepositoryHandler{ + DB: database, + Directory: oauthApp.Directory(), + Refresher: refresher, + }, + )).Methods("POST") + + router.Handle("/api/stars/{handle}/{repository}", appmiddleware.RequireAuth(sessionStore, database)( + &uihandlers.UnstarRepositoryHandler{ + DB: database, + Directory: oauthApp.Directory(), + Refresher: refresher, + }, + )).Methods("DELETE") + + router.Handle("/api/stars/{handle}/{repository}", appmiddleware.OptionalAuth(sessionStore, database)( + &uihandlers.CheckStarHandler{ + DB: database, + Directory: oauthApp.Directory(), + Refresher: refresher, + }, + )).Methods("GET") + router.Handle("/u/{handle}", appmiddleware.OptionalAuth(sessionStore, database)( &uihandlers.UserPageHandler{ DB: database, diff --git a/config/config.yml b/config/config.yml index 64a77c5..49505ad 100644 --- a/config/config.yml +++ b/config/config.yml @@ -3,12 +3,12 @@ log: level: info formatter: text fields: - service: atcr-registry + service: atcr-appview # Storage is handled by external services: # - Manifests/Tags -> ATProto PDS (user's personal data server) # - Blobs/Layers -> Hold service (default or BYOS) -# The AppView (registry) should be stateless with no local storage +# The AppView should be stateless with no local storage # # NOTE: The storage section below is required for distribution config validation # but is NOT actually used - all blob operations are routed through hold service diff --git a/docker-compose.yml b/docker-compose.yml index fff5af5..9b46cc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ services: - atcr-registry: + atcr-appview: build: context: . dockerfile: Dockerfile - image: atcr-registry:latest - container_name: atcr-registry + image: atcr-appview:latest + container_name: atcr-appview ports: - "5000:5000" environment: @@ -22,7 +22,7 @@ services: networks: atcr-network: ipv4_address: 172.28.0.2 - # The registry should be stateless - all storage is external: + # The AppView should be stateless - all storage is external: # - Manifests/Tags -> ATProto PDS # - Blobs/Layers -> Hold service # - OAuth tokens -> Persistent volume (atcr-tokens) diff --git a/docs/API_KEY_MIGRATION.md b/docs/API_KEY_MIGRATION.md index c9ab97c..883cbff 100644 --- a/docs/API_KEY_MIGRATION.md +++ b/docs/API_KEY_MIGRATION.md @@ -326,7 +326,7 @@ The `/auth/exchange` endpoint was only used for exchanging session tokens for re - `pkg/auth/exchange/handler.go` **Files to update:** -- `cmd/registry/serve.go` - Remove exchange handler registration +- `cmd/appview/serve.go` - Remove exchange handler registration ### Phase 3: Update UI @@ -489,7 +489,7 @@ loadKeys(); ``` -#### 3.2 Register API Key Routes (`cmd/registry/serve.go`) +#### 3.2 Register API Key Routes (`cmd/appview/serve.go`) ```go // In initializeUI() function, add: @@ -678,7 +678,7 @@ func NewServer(app *App) *Server { } ``` -#### 5.4 Update Registry Initialization (`cmd/registry/serve.go`) +#### 5.4 Update Registry Initialization (`cmd/appview/serve.go`) ```go // REMOVE session manager creation: @@ -800,7 +800,7 @@ if issuer != nil { - `pkg/appview/handlers/settings.go` - Add API key management UI - `pkg/appview/templates/settings.html` - Add API key section - `cmd/credential-helper/main.go` - Simplify to use API keys -- `cmd/registry/serve.go` - Initialize API key store, remove session manager +- `cmd/appview/serve.go` - Initialize API key store, remove session manager ### Deleted Files - `pkg/auth/session/handler.go` - Session token system diff --git a/docs/APPVIEW-UI-IMPLEMENTATION.md b/docs/APPVIEW-UI-IMPLEMENTATION.md index 4ecb60d..f84659f 100644 --- a/docs/APPVIEW-UI-IMPLEMENTATION.md +++ b/docs/APPVIEW-UI-IMPLEMENTATION.md @@ -14,7 +14,7 @@ This document provides step-by-step implementation details for building the ATCR ## Project Structure ``` -cmd/registry/ +cmd/appview/ ├── main.go # Add AppView routes here pkg/appview/ @@ -1249,7 +1249,7 @@ func GetUser(r *http.Request) *db.User { ## Step 8: Main Integration -**cmd/registry/main.go (additions):** +**cmd/appview/main.go (additions):** ```go package main @@ -1747,19 +1747,19 @@ tmpl = template.Must(tmpl.ParseGlob("web/templates/**/*.html")) ### Development ```bash # Run migrations -go run cmd/registry/main.go migrate +go run cmd/appview/main.go migrate # Start server -go run cmd/registry/main.go serve +go run cmd/appview/main.go serve ``` ### Production ```bash # Build binary -go build -o bin/atcr-registry ./cmd/registry +go build -o bin/atcr-appview ./cmd/appview # Run with config -./bin/atcr-registry serve config/production.yml +./bin/atcr-appview serve config/production.yml ``` ### Environment Variables @@ -1786,7 +1786,7 @@ UI_SESSION_DURATION=24h ### Single Binary Deployment - All templates and static files embedded with `//go:embed` - No need to ship separate `web/` directory -- Single `atcr-registry` binary contains everything +- Single `atcr-appview` binary contains everything - Easy deployment: just copy one file ### Package Structure @@ -1807,12 +1807,12 @@ var staticFS embed.FS **Build:** ```bash -go build -o bin/atcr-registry ./cmd/registry +go build -o bin/atcr-appview ./cmd/appview ``` **Deploy:** ```bash -scp bin/atcr-registry server:/usr/local/bin/ +scp bin/atcr-appview server:/usr/local/bin/ # Done! No webpack, no node_modules, no separate assets folder ``` diff --git a/docs/TESTING.md b/docs/TESTING.md index d79ae17..ee4ffa1 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -24,7 +24,7 @@ sudo chown -R $USER:$USER /var/lib/atcr ### 2. Build Binaries ```bash -go build -o atcr-registry ./cmd/registry +go build -o atcr-appview ./cmd/appview go build -o atcr-hold ./cmd/hold go build -o docker-credential-atcr ./cmd/credential-helper ``` @@ -62,9 +62,9 @@ export $(cat .env | xargs) ### 4. Start Services -**Terminal 1 - Registry:** +**Terminal 1 - AppView:** ```bash -./atcr-registry serve config/config.yml +./atcr-appview serve config/config.yml ``` **Terminal 2 - Hold:** @@ -74,9 +74,9 @@ export $(cat .env | xargs) ### 5. Start Services and OAuth Registration -**Terminal 1 - Registry:** +**Terminal 1 - AppView:** ```bash -./atcr-registry serve config/config.yml +./atcr-appview serve config/config.yml ``` **Terminal 2 - Hold (OAuth registration):** @@ -295,7 +295,7 @@ ls -lh /var/lib/atcr/blobs/docker/registry/v2/blobs/sha256/ kill $(cat .atcr-pids) # Or manually -pkill atcr-registry +pkill atcr-appview pkill atcr-hold ``` diff --git a/lexicons/io/atcr/manifest.json b/lexicons/io/atcr/manifest.json index 78f9b6b..c0e50ef 100644 --- a/lexicons/io/atcr/manifest.json +++ b/lexicons/io/atcr/manifest.json @@ -58,6 +58,10 @@ "ref": "#blobReference", "description": "Optional reference to another manifest (for attestations, signatures)" }, + "manifestBlob": { + "type": "blob", + "description": "The full OCI manifest stored as a blob in ATProto." + }, "createdAt": { "type": "string", "format": "datetime", diff --git a/lexicons/io/atcr/sailor/star.json b/lexicons/io/atcr/sailor/star.json new file mode 100644 index 0000000..1f85e3f --- /dev/null +++ b/lexicons/io/atcr/sailor/star.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "io.atcr.sailor.star", + "defs": { + "main": { + "type": "record", + "description": "A star (like) on a container image repository. Stored in the starrer's PDS, similar to Bluesky likes.", + "key": "any", + "record": { + "type": "object", + "required": ["subject", "createdAt"], + "properties": { + "subject": { + "type": "ref", + "ref": "#subject", + "description": "The repository being starred" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "Star creation timestamp" + } + } + } + }, + "subject": { + "type": "object", + "description": "Reference to a repository owned by a user", + "required": ["did", "repository"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository owner" + }, + "repository": { + "type": "string", + "description": "Repository name (e.g., 'myapp')", + "maxLength": 255 + } + } + } + } +} diff --git a/pkg/appview/db/models.go b/pkg/appview/db/models.go index 42e566c..1b9e062 100644 --- a/pkg/appview/db/models.go +++ b/pkg/appview/db/models.go @@ -22,7 +22,6 @@ type Manifest struct { MediaType string ConfigDigest string ConfigSize int64 - RawManifest string // JSON CreatedAt time.Time Title string Description string @@ -77,3 +76,14 @@ type Repository struct { Licenses string IconURL string } + +// RepositoryStats represents statistics for a repository +type RepositoryStats struct { + DID string + Repository string + StarCount int + PullCount int + LastPull *time.Time + PushCount int + LastPush *time.Time +} diff --git a/pkg/appview/db/queries.go b/pkg/appview/db/queries.go index 705c10b..3fb5a32 100644 --- a/pkg/appview/db/queries.go +++ b/pkg/appview/db/queries.go @@ -135,7 +135,7 @@ func GetUserRepositories(db *sql.DB, did string) ([]Repository, error) { // Get manifests for this repo manifestRows, err := db.Query(` SELECT id, digest, hold_endpoint, schema_version, media_type, - config_digest, config_size, raw_manifest, created_at, + config_digest, config_size, created_at, title, description, source_url, documentation_url, licenses, icon_url FROM manifests WHERE did = ? AND repository = ? @@ -155,7 +155,7 @@ func GetUserRepositories(db *sql.DB, did string) ([]Repository, error) { var title, description, sourceURL, documentationURL, licenses, iconURL sql.NullString if err := manifestRows.Scan(&m.ID, &m.Digest, &m.HoldEndpoint, &m.SchemaVersion, - &m.MediaType, &m.ConfigDigest, &m.ConfigSize, &m.RawManifest, &m.CreatedAt, + &m.MediaType, &m.ConfigDigest, &m.ConfigSize, &m.CreatedAt, &title, &description, &sourceURL, &documentationURL, &licenses, &iconURL); err != nil { manifestRows.Close() return nil, err @@ -384,12 +384,12 @@ func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) { result, err := db.Exec(` INSERT OR IGNORE INTO manifests (did, repository, digest, hold_endpoint, schema_version, media_type, - config_digest, config_size, raw_manifest, created_at, + config_digest, config_size, created_at, title, description, source_url, documentation_url, licenses, icon_url) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, manifest.DID, manifest.Repository, manifest.Digest, manifest.HoldEndpoint, manifest.SchemaVersion, manifest.MediaType, manifest.ConfigDigest, - manifest.ConfigSize, manifest.RawManifest, manifest.CreatedAt, + manifest.ConfigSize, manifest.CreatedAt, manifest.Title, manifest.Description, manifest.SourceURL, manifest.DocumentationURL, manifest.Licenses, manifest.IconURL) @@ -446,13 +446,13 @@ func GetManifest(db *sql.DB, digest string) (*Manifest, error) { err := db.QueryRow(` SELECT id, did, repository, digest, hold_endpoint, schema_version, - media_type, config_digest, config_size, raw_manifest, created_at, + media_type, config_digest, config_size, created_at, title, description, source_url, documentation_url, licenses, icon_url FROM manifests WHERE digest = ? `, digest).Scan(&m.ID, &m.DID, &m.Repository, &m.Digest, &m.HoldEndpoint, &m.SchemaVersion, &m.MediaType, &m.ConfigDigest, &m.ConfigSize, - &m.RawManifest, &m.CreatedAt, + &m.CreatedAt, &title, &description, &sourceURL, &documentationURL, &licenses, &iconURL) if err != nil { @@ -698,7 +698,7 @@ func GetRepository(db *sql.DB, did, repository string) (*Repository, error) { // Get manifests for this repo manifestRows, err := db.Query(` SELECT id, digest, hold_endpoint, schema_version, media_type, - config_digest, config_size, raw_manifest, created_at, + config_digest, config_size, created_at, title, description, source_url, documentation_url, licenses, icon_url FROM manifests WHERE did = ? AND repository = ? @@ -718,7 +718,7 @@ func GetRepository(db *sql.DB, did, repository string) (*Repository, error) { var title, description, sourceURL, documentationURL, licenses, iconURL sql.NullString if err := manifestRows.Scan(&m.ID, &m.Digest, &m.HoldEndpoint, &m.SchemaVersion, - &m.MediaType, &m.ConfigDigest, &m.ConfigSize, &m.RawManifest, &m.CreatedAt, + &m.MediaType, &m.ConfigDigest, &m.ConfigSize, &m.CreatedAt, &title, &description, &sourceURL, &documentationURL, &licenses, &iconURL); err != nil { manifestRows.Close() return nil, err @@ -761,3 +761,142 @@ func GetRepository(db *sql.DB, did, repository string) (*Repository, error) { return &r, nil } + +// GetRepositoryStats fetches stats for a repository +func GetRepositoryStats(db *sql.DB, did, repository string) (*RepositoryStats, error) { + var stats RepositoryStats + var lastPullStr, lastPushStr sql.NullString + + err := db.QueryRow(` + SELECT did, repository, star_count, pull_count, last_pull, push_count, last_push + FROM repository_stats + WHERE did = ? AND repository = ? + `, did, repository).Scan(&stats.DID, &stats.Repository, &stats.StarCount, &stats.PullCount, &lastPullStr, &stats.PushCount, &lastPushStr) + + if err == sql.ErrNoRows { + // Return zero stats if no record exists yet + return &RepositoryStats{ + DID: did, + Repository: repository, + StarCount: 0, + PullCount: 0, + PushCount: 0, + }, nil + } + if err != nil { + return nil, err + } + + // Parse timestamps + if lastPullStr.Valid { + t, err := parseTimestamp(lastPullStr.String) + if err == nil { + stats.LastPull = &t + } + } + if lastPushStr.Valid { + t, err := parseTimestamp(lastPushStr.String) + if err == nil { + stats.LastPush = &t + } + } + + return &stats, nil +} + +// UpsertRepositoryStats inserts or updates repository stats +func UpsertRepositoryStats(db *sql.DB, stats *RepositoryStats) error { + _, err := db.Exec(` + INSERT INTO repository_stats (did, repository, star_count, pull_count, last_pull, push_count, last_push) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(did, repository) DO UPDATE SET + star_count = excluded.star_count, + pull_count = excluded.pull_count, + last_pull = excluded.last_pull, + push_count = excluded.push_count, + last_push = excluded.last_push + `, stats.DID, stats.Repository, stats.StarCount, stats.PullCount, stats.LastPull, stats.PushCount, stats.LastPush) + return err +} + +// IncrementStarCount increments the star count for a repository +func IncrementStarCount(db *sql.DB, did, repository string) error { + _, err := db.Exec(` + INSERT INTO repository_stats (did, repository, star_count) + VALUES (?, ?, 1) + ON CONFLICT(did, repository) DO UPDATE SET + star_count = star_count + 1 + `, did, repository) + return err +} + +// DecrementStarCount decrements the star count for a repository +func DecrementStarCount(db *sql.DB, did, repository string) error { + _, err := db.Exec(` + UPDATE repository_stats + SET star_count = MAX(0, star_count - 1) + WHERE did = ? AND repository = ? + `, did, repository) + return err +} + +// IncrementPullCount increments the pull count for a repository +func IncrementPullCount(db *sql.DB, did, repository string) error { + _, err := db.Exec(` + INSERT INTO repository_stats (did, repository, pull_count, last_pull) + VALUES (?, ?, 1, datetime('now')) + ON CONFLICT(did, repository) DO UPDATE SET + pull_count = pull_count + 1, + last_pull = datetime('now') + `, did, repository) + return err +} + +// IncrementPushCount increments the push count for a repository +func IncrementPushCount(db *sql.DB, did, repository string) error { + _, err := db.Exec(` + INSERT INTO repository_stats (did, repository, push_count, last_push) + VALUES (?, ?, 1, datetime('now')) + ON CONFLICT(did, repository) DO UPDATE SET + push_count = push_count + 1, + last_push = datetime('now') + `, did, repository) + return err +} + +// parseTimestamp parses a timestamp string with multiple format attempts +func parseTimestamp(s string) (time.Time, error) { + formats := []string{ + time.RFC3339Nano, + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999", + time.RFC3339, + "2006-01-02 15:04:05", + } + for _, format := range formats { + if t, err := time.Parse(format, s); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("unable to parse timestamp: %s", s) +} + +// MetricsDB wraps a sql.DB and implements the metrics interface for middleware +type MetricsDB struct { + db *sql.DB +} + +// NewMetricsDB creates a new metrics database wrapper +func NewMetricsDB(db *sql.DB) *MetricsDB { + return &MetricsDB{db: db} +} + +// IncrementPullCount increments the pull count for a repository +func (m *MetricsDB) IncrementPullCount(did, repository string) error { + return IncrementPullCount(m.db, did, repository) +} + +// IncrementPushCount increments the push count for a repository +func (m *MetricsDB) IncrementPushCount(did, repository string) error { + return IncrementPushCount(m.db, did, repository) +} diff --git a/pkg/appview/db/schema.go b/pkg/appview/db/schema.go index d024f1f..0b17031 100644 --- a/pkg/appview/db/schema.go +++ b/pkg/appview/db/schema.go @@ -2,7 +2,6 @@ package db import ( "database/sql" - "strings" _ "github.com/mattn/go-sqlite3" ) @@ -28,7 +27,6 @@ CREATE TABLE IF NOT EXISTS manifests ( media_type TEXT NOT NULL, config_digest TEXT, config_size INTEGER, - raw_manifest TEXT NOT NULL, created_at TIMESTAMP NOT NULL, title TEXT, description TEXT, @@ -142,6 +140,21 @@ CREATE TABLE IF NOT EXISTS pending_device_auth ( ); CREATE INDEX IF NOT EXISTS idx_pending_device_auth_user_code ON pending_device_auth(user_code); CREATE INDEX IF NOT EXISTS idx_pending_device_auth_expires ON pending_device_auth(expires_at); + +CREATE TABLE IF NOT EXISTS repository_stats ( + did TEXT NOT NULL, + repository TEXT NOT NULL, + star_count INTEGER NOT NULL DEFAULT 0, + pull_count INTEGER NOT NULL DEFAULT 0, + last_pull TIMESTAMP, + push_count INTEGER NOT NULL DEFAULT 0, + last_push TIMESTAMP, + PRIMARY KEY(did, repository), + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did); +CREATE INDEX IF NOT EXISTS idx_repository_stats_star_count ON repository_stats(star_count DESC); +CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC); ` // InitDB initializes the SQLite database with the schema @@ -161,28 +174,22 @@ func InitDB(path string) (*sql.DB, error) { return nil, err } - // Migration: Add avatar column if it doesn't exist - _, err = db.Exec(`ALTER TABLE users ADD COLUMN avatar TEXT`) - // Ignore error if column already exists - if err != nil && !strings.Contains(err.Error(), "duplicate column") { - // Log but don't fail - column might already exist + // Migration: Drop raw_manifest column if it exists + // Check if column exists first + var columnExists bool + err = db.QueryRow(` + SELECT COUNT(*) > 0 + FROM pragma_table_info('manifests') + WHERE name = 'raw_manifest' + `).Scan(&columnExists) + if err != nil { + return nil, err } - // Migration: Add OCI annotation columns to manifests table - annotationColumns := []string{ - "title TEXT", - "description TEXT", - "source_url TEXT", - "documentation_url TEXT", - "licenses TEXT", - "icon_url TEXT", - } - for _, col := range annotationColumns { - colName := strings.Split(col, " ")[0] - _, err = db.Exec(`ALTER TABLE manifests ADD COLUMN ` + col) - if err != nil && !strings.Contains(err.Error(), "duplicate column") { - // Log but continue - column might already exist - println("Warning: Failed to add column", colName, "to manifests:", err.Error()) + if columnExists { + // Drop the column (requires SQLite 3.35.0+) + if _, err := db.Exec(`ALTER TABLE manifests DROP COLUMN raw_manifest`); err != nil { + return nil, err } } diff --git a/pkg/appview/handlers/api.go b/pkg/appview/handlers/api.go new file mode 100644 index 0000000..fe9ee65 --- /dev/null +++ b/pkg/appview/handlers/api.go @@ -0,0 +1,226 @@ +package handlers + +import ( + "context" + "database/sql" + "encoding/json" + "net/http" + + "atcr.io/pkg/appview/db" + "atcr.io/pkg/appview/middleware" + "atcr.io/pkg/atproto" + "atcr.io/pkg/auth/oauth" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/mux" +) + +// StarRepositoryHandler handles starring a repository +type StarRepositoryHandler struct { + DB *sql.DB + Directory identity.Directory + Refresher *oauth.Refresher +} + +func (h *StarRepositoryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Get authenticated user from middleware + user := middleware.GetUser(r) + if user == nil { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + // Extract parameters + vars := mux.Vars(r) + handle := vars["handle"] + repository := vars["repository"] + + // Resolve owner's handle to DID + ownerDID, err := resolveIdentityToDID(r.Context(), h.Directory, handle) + if err != nil { + http.Error(w, "Failed to resolve handle", http.StatusBadRequest) + return + } + + // Get OAuth session for the authenticated user + session, err := h.Refresher.GetSession(r.Context(), user.DID) + if err != nil { + http.Error(w, "Failed to get OAuth session", http.StatusUnauthorized) + return + } + + // Get user's PDS client (use indigo's API client which handles DPoP automatically) + apiClient := session.APIClient() + pdsClient := atproto.NewClientWithIndigoClient(user.PDSEndpoint, user.DID, apiClient) + + // Create star record + starRecord := atproto.NewStarRecord(ownerDID, repository) + rkey := atproto.StarRecordKey(ownerDID, repository) + + // Write star record to user's PDS + _, err = pdsClient.PutRecord(r.Context(), atproto.StarCollection, rkey, starRecord) + if err != nil { + http.Error(w, "Failed to create star", http.StatusInternalServerError) + return + } + + // Return success + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(map[string]bool{"starred": true}) +} + +// UnstarRepositoryHandler handles unstarring a repository +type UnstarRepositoryHandler struct { + DB *sql.DB + Directory identity.Directory + Refresher *oauth.Refresher +} + +func (h *UnstarRepositoryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Get authenticated user from middleware + user := middleware.GetUser(r) + if user == nil { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + // Extract parameters + vars := mux.Vars(r) + handle := vars["handle"] + repository := vars["repository"] + + // Resolve owner's handle to DID + ownerDID, err := resolveIdentityToDID(r.Context(), h.Directory, handle) + if err != nil { + http.Error(w, "Failed to resolve handle", http.StatusBadRequest) + return + } + + // Get OAuth session for the authenticated user + session, err := h.Refresher.GetSession(r.Context(), user.DID) + if err != nil { + http.Error(w, "Failed to get OAuth session", http.StatusUnauthorized) + return + } + + // Get user's PDS client (use indigo's API client which handles DPoP automatically) + apiClient := session.APIClient() + pdsClient := atproto.NewClientWithIndigoClient(user.PDSEndpoint, user.DID, apiClient) + + // Delete star record from user's PDS + rkey := atproto.StarRecordKey(ownerDID, repository) + err = pdsClient.DeleteRecord(r.Context(), atproto.StarCollection, rkey) + if err != nil { + // If record doesn't exist, still return success (idempotent) + if err.Error() != "record not found" { + http.Error(w, "Failed to delete star", http.StatusInternalServerError) + return + } + } + + // Return success + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]bool{"starred": false}) +} + +// CheckStarHandler checks if current user has starred a repository +type CheckStarHandler struct { + DB *sql.DB + Directory identity.Directory + Refresher *oauth.Refresher +} + +func (h *CheckStarHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Get authenticated user from middleware + user := middleware.GetUser(r) + if user == nil { + // Not authenticated - return not starred + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]bool{"starred": false}) + return + } + + // Extract parameters + vars := mux.Vars(r) + handle := vars["handle"] + repository := vars["repository"] + + // Resolve owner's handle to DID + ownerDID, err := resolveIdentityToDID(r.Context(), h.Directory, handle) + if err != nil { + http.Error(w, "Failed to resolve handle", http.StatusBadRequest) + return + } + + // Get OAuth session for the authenticated user + session, err := h.Refresher.GetSession(r.Context(), user.DID) + if err != nil { + // No OAuth session - return not starred + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]bool{"starred": false}) + return + } + + // Get user's PDS client (use indigo's API client which handles DPoP automatically) + apiClient := session.APIClient() + pdsClient := atproto.NewClientWithIndigoClient(user.PDSEndpoint, user.DID, apiClient) + + // Check if star record exists + rkey := atproto.StarRecordKey(ownerDID, repository) + _, err = pdsClient.GetRecord(r.Context(), atproto.StarCollection, rkey) + + starred := err == nil + + // Return result + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]bool{"starred": starred}) +} + +// GetStatsHandler returns repository statistics +type GetStatsHandler struct { + DB *sql.DB + Directory identity.Directory +} + +func (h *GetStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Extract parameters + vars := mux.Vars(r) + handle := vars["handle"] + repository := vars["repository"] + + // Resolve owner's handle to DID + ownerDID, err := resolveIdentityToDID(r.Context(), h.Directory, handle) + if err != nil { + http.Error(w, "Failed to resolve handle", http.StatusBadRequest) + return + } + + // Get repository stats from database + stats, err := db.GetRepositoryStats(h.DB, ownerDID, repository) + if err != nil { + http.Error(w, "Failed to fetch stats", http.StatusInternalServerError) + return + } + + // Return stats as JSON + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(stats) +} + +// resolveIdentityToDID is a helper function that resolves a handle or DID to a DID +func resolveIdentityToDID(ctx context.Context, directory identity.Directory, identityStr string) (string, error) { + // Parse as AT identifier (handle or DID) + atID, err := syntax.ParseAtIdentifier(identityStr) + if err != nil { + return "", err + } + + // Resolve to DID via directory + ident, err := directory.Lookup(ctx, *atID) + if err != nil { + return "", err + } + + return ident.DID.String(), nil +} diff --git a/pkg/appview/interfaces.go b/pkg/appview/interfaces.go deleted file mode 100644 index 3fb6a80..0000000 --- a/pkg/appview/interfaces.go +++ /dev/null @@ -1,50 +0,0 @@ -package appview - -import "time" - -// SessionStore interface for UI session management -// Implemented by both session.Store (file-based) and db.SessionStore (SQLite-based) -type SessionStore interface { - Create(did, handle, pdsEndpoint string, duration time.Duration) (string, error) - CreateWithOAuth(did, handle, pdsEndpoint, oauthSessionID string, duration time.Duration) (string, error) - Get(id string) (Session, bool) - Delete(id string) - Cleanup() -} - -// Session represents a user session -// Compatible with both file-based and SQLite implementations -type Session interface { - GetID() string - GetDID() string - GetHandle() string - GetPDSEndpoint() string - GetOAuthSessionID() string -} - -// DeviceStore interface for device authorization management -// Implemented by both device.Store (file-based) and db.DeviceStore (SQLite-based) -type DeviceStore interface { - CreatePendingAuth(deviceName, ip, userAgent string) (PendingAuth, error) - GetPendingByUserCode(userCode string) (PendingAuth, bool) - GetPendingByDeviceCode(deviceCode string) (PendingAuth, bool) - ApprovePending(userCode, did, handle string) (deviceSecret string, err error) - ValidateDeviceSecret(secret string) (Device, error) - ListDevices(did string) []Device - RevokeDevice(did, deviceID string) error - CleanupExpired() -} - -// PendingAuth interface for pending device authorizations -type PendingAuth interface { - GetDeviceCode() string - GetUserCode() string - GetDeviceName() string -} - -// Device interface for authorized devices -type Device interface { - GetID() string - GetDID() string - GetHandle() string -} diff --git a/pkg/appview/jetstream/backfill.go b/pkg/appview/jetstream/backfill.go index a6795d3..e88dc8d 100644 --- a/pkg/appview/jetstream/backfill.go +++ b/pkg/appview/jetstream/backfill.go @@ -52,6 +52,7 @@ func (b *BackfillWorker) Start(ctx context.Context) error { collections := []string{ atproto.ManifestCollection, // io.atcr.manifest atproto.TagCollection, // io.atcr.tag + atproto.StarCollection, // io.atcr.sailor.star } for _, collection := range collections { @@ -243,6 +244,8 @@ func (b *BackfillWorker) processRecord(ctx context.Context, did, collection stri return b.processManifestRecord(did, record) case atproto.TagCollection: return b.processTagRecord(did, record) + case atproto.StarCollection: + return b.processStarRecord(did, record) default: return fmt.Errorf("unsupported collection: %s", collection) } @@ -255,24 +258,34 @@ func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Recor return fmt.Errorf("failed to unmarshal manifest: %w", err) } - // Serialize full manifest as JSON for storage - manifestJSON, err := json.Marshal(manifestRecord) - if err != nil { - return fmt.Errorf("failed to marshal manifest: %w", err) + // Extract OCI annotations from manifest + var title, description, sourceURL, documentationURL, licenses, iconURL string + if manifestRecord.Annotations != nil { + title = manifestRecord.Annotations["org.opencontainers.image.title"] + description = manifestRecord.Annotations["org.opencontainers.image.description"] + sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] + documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] + licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] + iconURL = manifestRecord.Annotations["io.atcr.icon"] } // Insert manifest manifestID, err := db.InsertManifest(b.db, &db.Manifest{ - DID: did, - Repository: manifestRecord.Repository, - Digest: manifestRecord.Digest, - MediaType: manifestRecord.MediaType, - SchemaVersion: manifestRecord.SchemaVersion, - ConfigDigest: manifestRecord.Config.Digest, - ConfigSize: manifestRecord.Config.Size, - RawManifest: string(manifestJSON), - HoldEndpoint: manifestRecord.HoldEndpoint, - CreatedAt: manifestRecord.CreatedAt, + DID: did, + Repository: manifestRecord.Repository, + Digest: manifestRecord.Digest, + MediaType: manifestRecord.MediaType, + SchemaVersion: manifestRecord.SchemaVersion, + ConfigDigest: manifestRecord.Config.Digest, + ConfigSize: manifestRecord.Config.Size, + HoldEndpoint: manifestRecord.HoldEndpoint, + CreatedAt: manifestRecord.CreatedAt, + Title: title, + Description: description, + SourceURL: sourceURL, + DocumentationURL: documentationURL, + Licenses: licenses, + IconURL: iconURL, }) if err != nil { // Skip if already exists @@ -316,6 +329,19 @@ func (b *BackfillWorker) processTagRecord(did string, record *atproto.Record) er }) } +// processStarRecord processes a star record +func (b *BackfillWorker) processStarRecord(did string, record *atproto.Record) error { + var starRecord atproto.StarRecord + if err := json.Unmarshal(record.Value, &starRecord); err != nil { + return fmt.Errorf("failed to unmarshal star: %w", err) + } + + // Increment star count for the repository being starred + // The DID here is the starrer (user who starred) + // The subject contains the owner DID and repository + return db.IncrementStarCount(b.db, starRecord.Subject.DID, starRecord.Subject.Repository) +} + // ensureUser resolves and upserts a user by DID func (b *BackfillWorker) ensureUser(ctx context.Context, did string) error { // Check if user already exists diff --git a/pkg/appview/jetstream/worker.go b/pkg/appview/jetstream/worker.go index 448971b..f613c2b 100644 --- a/pkg/appview/jetstream/worker.go +++ b/pkg/appview/jetstream/worker.go @@ -53,6 +53,7 @@ func NewWorker(database *sql.DB, jetstreamURL string, startCursor int64) *Worker wantedCollections: []string{ atproto.ManifestCollection, // io.atcr.manifest atproto.TagCollection, // io.atcr.tag + atproto.StarCollection, // io.atcr.sailor.star }, userCache: &UserCache{ cache: make(map[string]*db.User), @@ -199,6 +200,10 @@ func (w *Worker) processMessage(message []byte) error { fmt.Printf("Jetstream: Processing tag event: did=%s, operation=%s, rkey=%s\n", commit.DID, commit.Operation, commit.RKey) return w.processTag(commit) + case atproto.StarCollection: + fmt.Printf("Jetstream: Processing star event: did=%s, operation=%s, rkey=%s\n", + commit.DID, commit.Operation, commit.RKey) + return w.processStar(commit) default: // Ignore other collections return nil @@ -310,12 +315,6 @@ func (w *Worker) processManifest(commit *CommitEvent) error { return nil } - // Serialize full manifest as JSON for storage - manifestJSON, err := json.Marshal(manifestRecord) - if err != nil { - return fmt.Errorf("failed to marshal manifest: %w", err) - } - // Extract OCI annotations from manifest var title, description, sourceURL, documentationURL, licenses, iconURL string if manifestRecord.Annotations != nil { @@ -336,7 +335,6 @@ func (w *Worker) processManifest(commit *CommitEvent) error { SchemaVersion: manifestRecord.SchemaVersion, ConfigDigest: manifestRecord.Config.Digest, ConfigSize: manifestRecord.Config.Size, - RawManifest: string(manifestJSON), HoldEndpoint: manifestRecord.HoldEndpoint, CreatedAt: manifestRecord.CreatedAt, Title: title, @@ -409,6 +407,50 @@ func (w *Worker) processTag(commit *CommitEvent) error { }) } +// processStar processes a star commit event +func (w *Worker) processStar(commit *CommitEvent) error { + // Resolve and upsert the user who starred (starrer) + if err := w.ensureUser(context.Background(), commit.DID); err != nil { + return fmt.Errorf("failed to ensure user: %w", err) + } + + if commit.Operation == "delete" { + // Unstar - parse the record to get the subject (owner DID and repository) + var starRecord atproto.StarRecord + if commit.Record != nil { + recordBytes, err := json.Marshal(commit.Record) + if err != nil { + return fmt.Errorf("failed to marshal record: %w", err) + } + if err := json.Unmarshal(recordBytes, &starRecord); err != nil { + return fmt.Errorf("failed to unmarshal star: %w", err) + } + + // Decrement star count + return db.DecrementStarCount(w.db, starRecord.Subject.DID, starRecord.Subject.Repository) + } + // If no record data, we can't determine what was unstarred + return nil + } + + // Parse star record + var starRecord atproto.StarRecord + if commit.Record != nil { + recordBytes, err := json.Marshal(commit.Record) + if err != nil { + return fmt.Errorf("failed to marshal record: %w", err) + } + if err := json.Unmarshal(recordBytes, &starRecord); err != nil { + return fmt.Errorf("failed to unmarshal star: %w", err) + } + } else { + return nil + } + + // Increment star count for the repository being starred + return db.IncrementStarCount(w.db, starRecord.Subject.DID, starRecord.Subject.Repository) +} + // JetstreamEvent represents a Jetstream event type JetstreamEvent struct { DID string `json:"did"` diff --git a/pkg/appview/templates/components/modal.html b/pkg/appview/templates/components/modal.html index c95798f..d4936d2 100644 --- a/pkg/appview/templates/components/modal.html +++ b/pkg/appview/templates/components/modal.html @@ -25,9 +25,6 @@ - -

Raw Manifest

-
{{ .RawManifest }}
{{ end }} diff --git a/pkg/atproto/lexicon.go b/pkg/atproto/lexicon.go index 225be70..87ebbc6 100644 --- a/pkg/atproto/lexicon.go +++ b/pkg/atproto/lexicon.go @@ -22,6 +22,9 @@ const ( // SailorProfileCollection is the collection name for user profiles SailorProfileCollection = "io.atcr.sailor.profile" + + // StarCollection is the collection name for repository stars + StarCollection = "io.atcr.sailor.star" ) // ManifestRecord represents a container image manifest stored in ATProto @@ -59,14 +62,8 @@ type ManifestRecord struct { Subject *BlobReference `json:"subject,omitempty"` // ManifestBlob is a reference to the manifest blob stored in ATProto blob storage - // This is the new way of storing manifests (replaces RawManifest) ManifestBlob *ATProtoBlobRef `json:"manifestBlob,omitempty"` - // RawManifest stores the original manifest bytes (base64 encoded) - DEPRECATED - // Kept for backward compatibility with old records - // New records should use ManifestBlob instead - RawManifest string `json:"rawManifest,omitempty"` - // CreatedAt timestamp CreatedAt time.Time `json:"createdAt"` } @@ -114,7 +111,6 @@ func NewManifestRecord(repository, digest string, ociManifest []byte) (*Manifest SchemaVersion: ociData.SchemaVersion, Annotations: ociData.Annotations, // ManifestBlob will be set by the caller after uploading to blob storage - // RawManifest no longer stored for new records (backward compat only) CreatedAt: time.Now(), } @@ -143,42 +139,6 @@ func NewManifestRecord(repository, digest string, ociManifest []byte) (*Manifest return record, nil } -// ToOCIManifest converts the manifest record back to OCI manifest JSON -// This should NOT be used directly - use manifest_store.Get() which downloads the blob -// This is kept for backward compatibility only -func (m *ManifestRecord) ToOCIManifest() ([]byte, error) { - // New records: ManifestBlob reference (blob downloaded separately by manifest store) - // This function should not be called for new records - it's a fallback only - - // Backward compatibility: If we have the raw manifest stored, return it - if m.RawManifest != "" { - rawBytes, err := base64.StdEncoding.DecodeString(m.RawManifest) - if err != nil { - return nil, err - } - return rawBytes, nil - } - - // Last resort: reconstruct from fields (will have different digest!) - // This should only happen for very old records - ociManifest := map[string]any{ - "schemaVersion": m.SchemaVersion, - "mediaType": m.MediaType, - "config": m.Config, - "layers": m.Layers, - } - - if m.Subject != nil { - ociManifest["subject"] = m.Subject - } - - if len(m.Annotations) > 0 { - ociManifest["annotations"] = m.Annotations - } - - return json.Marshal(ociManifest) -} - // TagRecord represents a tag pointing to a manifest type TagRecord struct { // Type should be "io.atcr.tag" @@ -299,3 +259,46 @@ func NewSailorProfileRecord(defaultHold string) *SailorProfileRecord { UpdatedAt: now, } } + +// StarSubject represents the subject of a star (the repository being starred) +type StarSubject struct { + // DID is the DID of the repository owner + DID string `json:"did"` + + // Repository is the name of the repository + Repository string `json:"repository"` +} + +// StarRecord represents a user starring a repository +// Stored in the starrer's PDS (like Bluesky likes) +type StarRecord struct { + // Type should be "io.atcr.sailor.star" + Type string `json:"$type"` + + // Subject is the repository being starred + Subject StarSubject `json:"subject"` + + // CreatedAt timestamp + CreatedAt time.Time `json:"createdAt"` +} + +// NewStarRecord creates a new star record +func NewStarRecord(ownerDID, repository string) *StarRecord { + return &StarRecord{ + Type: StarCollection, + Subject: StarSubject{ + DID: ownerDID, + Repository: repository, + }, + CreatedAt: time.Now(), + } +} + +// StarRecordKey generates a record key for a star +// Uses a simple hash to ensure uniqueness and prevent duplicate stars +func StarRecordKey(ownerDID, repository string) string { + // Use base64 encoding of "ownerDID/repository" as the record key + // This is deterministic and prevents duplicate stars + combined := ownerDID + "/" + repository + return base64.RawURLEncoding.EncodeToString([]byte(combined)) +} diff --git a/pkg/atproto/manifest_store.go b/pkg/atproto/manifest_store.go index bfc763e..3964410 100644 --- a/pkg/atproto/manifest_store.go +++ b/pkg/atproto/manifest_store.go @@ -11,6 +11,11 @@ import ( "github.com/opencontainers/go-digest" ) +// DatabaseMetrics interface for tracking push counts +type DatabaseMetrics interface { + IncrementPushCount(did, repository string) error +} + // ManifestStore implements distribution.ManifestService // It stores manifests in ATProto as records type ManifestStore struct { @@ -20,16 +25,18 @@ type ManifestStore struct { did string // User's DID for cache key lastFetchedHoldEndpoint string // Hold endpoint from most recently fetched manifest (for pull) blobStore distribution.BlobStore // Blob store for fetching config during push + database DatabaseMetrics // Database for metrics tracking } // NewManifestStore creates a new ATProto-backed manifest store -func NewManifestStore(client *Client, repository string, holdEndpoint string, did string, blobStore distribution.BlobStore) *ManifestStore { +func NewManifestStore(client *Client, repository string, holdEndpoint string, did string, blobStore distribution.BlobStore, database DatabaseMetrics) *ManifestStore { return &ManifestStore{ client: client, repository: repository, holdEndpoint: holdEndpoint, did: did, blobStore: blobStore, + database: database, } } @@ -75,14 +82,7 @@ func (s *ManifestStore) Get(ctx context.Context, dgst digest.Digest, options ... if err != nil { return nil, fmt.Errorf("failed to download manifest blob: %w", err) } - } else { - // Backward compatibility: Use ToOCIManifest for old records - ociManifest, err = manifestRecord.ToOCIManifest() - if err != nil { - return nil, fmt.Errorf("failed to convert to OCI manifest: %w", err) - } } - // Parse the manifest based on media type // For now, we'll return the raw bytes wrapped in a manifest object // In a full implementation, you'd use distribution's manifest parsing @@ -145,6 +145,15 @@ func (s *ManifestStore) Put(ctx context.Context, manifest distribution.Manifest, return "", fmt.Errorf("failed to store manifest record in ATProto: %w", err) } + // Track push count (increment asynchronously to avoid blocking the response) + if s.database != nil { + go func() { + if err := s.database.IncrementPushCount(s.did, s.repository); err != nil { + fmt.Printf("WARNING: Failed to increment push count for %s/%s: %v\n", s.did, s.repository, err) + } + }() + } + // Also handle tag if specified for _, option := range options { if tagOpt, ok := option.(distribution.WithTagOption); ok { diff --git a/pkg/middleware/registry.go b/pkg/middleware/registry.go index 8e11d8d..a0e9660 100644 --- a/pkg/middleware/registry.go +++ b/pkg/middleware/registry.go @@ -23,11 +23,25 @@ import ( // Global refresher instance (set by main.go) var globalRefresher *oauth.Refresher +// Global database instance (set by main.go for pull tracking) +var globalDatabase interface { + IncrementPullCount(did, repository string) error + IncrementPushCount(did, repository string) error +} + // SetGlobalRefresher sets the global OAuth refresher instance func SetGlobalRefresher(refresher *oauth.Refresher) { globalRefresher = refresher } +// SetGlobalDatabase sets the global database instance for metrics tracking +func SetGlobalDatabase(database interface { + IncrementPullCount(did, repository string) error + IncrementPushCount(did, repository string) error +}) { + globalDatabase = database +} + func init() { // Register the name resolution middleware registrymw.Register("atproto-resolver", initATProtoResolver) @@ -169,7 +183,7 @@ func (nr *NamespaceResolver) Repository(ctx context.Context, name reference.Name // Create routing repository - routes manifests to ATProto, blobs to hold service // The registry is stateless - no local storage is used // Pass storage endpoint and DID as parameters (can't use context as it gets lost) - routingRepo := storage.NewRoutingRepository(repo, atprotoClient, repositoryName, storageEndpoint, did) + routingRepo := storage.NewRoutingRepository(repo, atprotoClient, repositoryName, storageEndpoint, did, globalDatabase) // Cache the repository nr.repositories.Store(cacheKey, routingRepo) diff --git a/pkg/storage/proxy_blob_store.go b/pkg/storage/proxy_blob_store.go index 9a2cd14..f8a0f48 100644 --- a/pkg/storage/proxy_blob_store.go +++ b/pkg/storage/proxy_blob_store.go @@ -32,11 +32,13 @@ type ProxyBlobStore struct { storageEndpoint string httpClient *http.Client did string + database DatabaseMetrics + repository string } // NewProxyBlobStore creates a new proxy blob store -func NewProxyBlobStore(storageEndpoint, did string) *ProxyBlobStore { - fmt.Printf("DEBUG [proxy_blob_store]: NewProxyBlobStore created with endpoint=%s, did=%s\n", storageEndpoint, did) +func NewProxyBlobStore(storageEndpoint, did string, database DatabaseMetrics, repository string) *ProxyBlobStore { + fmt.Printf("DEBUG [proxy_blob_store]: NewProxyBlobStore created with endpoint=%s, did=%s, repo=%s\n", storageEndpoint, did, repository) return &ProxyBlobStore{ storageEndpoint: storageEndpoint, httpClient: &http.Client{ @@ -49,7 +51,9 @@ func NewProxyBlobStore(storageEndpoint, did string) *ProxyBlobStore { IdleConnTimeout: 90 * time.Second, }, }, - did: did, + did: did, + database: database, + repository: repository, } } @@ -179,6 +183,15 @@ func (p *ProxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r return err } + // Track pull count (increment asynchronously to avoid blocking the response) + if p.database != nil && p.repository != "" { + go func() { + if err := p.database.IncrementPullCount(p.did, p.repository); err != nil { + fmt.Printf("WARNING: Failed to increment pull count for %s/%s: %v\n", p.did, p.repository, err) + } + }() + } + // Redirect to presigned URL http.Redirect(w, r, url, http.StatusTemporaryRedirect) return nil diff --git a/pkg/storage/routing_repository.go b/pkg/storage/routing_repository.go index 771b085..6f2f9da 100644 --- a/pkg/storage/routing_repository.go +++ b/pkg/storage/routing_repository.go @@ -9,6 +9,12 @@ import ( "github.com/distribution/distribution/v3" ) +// DatabaseMetrics interface for tracking pull/push counts +type DatabaseMetrics interface { + IncrementPullCount(did, repository string) error + IncrementPushCount(did, repository string) error +} + // RoutingRepository routes manifests to ATProto and blobs to external hold service // The registry (AppView) is stateless and NEVER stores blobs locally type RoutingRepository struct { @@ -19,6 +25,7 @@ type RoutingRepository struct { did string // User's DID for authorization manifestStore *atproto.ManifestStore // Cached manifest store instance blobStore *ProxyBlobStore // Cached blob store instance + database DatabaseMetrics // Database for metrics tracking } // NewRoutingRepository creates a new routing repository @@ -28,6 +35,7 @@ func NewRoutingRepository( repoName string, storageEndpoint string, did string, + database DatabaseMetrics, ) *RoutingRepository { return &RoutingRepository{ Repository: baseRepo, @@ -35,6 +43,7 @@ func NewRoutingRepository( repositoryName: repoName, storageEndpoint: storageEndpoint, did: did, + database: database, } } @@ -45,7 +54,7 @@ func (r *RoutingRepository) Manifests(ctx context.Context, options ...distributi // Ensure blob store is created first (needed for label extraction during push) blobStore := r.Blobs(ctx) - r.manifestStore = atproto.NewManifestStore(r.atprotoClient, r.repositoryName, r.storageEndpoint, r.did, blobStore) + r.manifestStore = atproto.NewManifestStore(r.atprotoClient, r.repositoryName, r.storageEndpoint, r.did, blobStore, r.database) } // After any manifest operation, cache the hold endpoint for blob fetches @@ -94,7 +103,7 @@ func (r *RoutingRepository) Blobs(ctx context.Context) distribution.BlobStore { } // Create and cache proxy blob store - r.blobStore = NewProxyBlobStore(holdEndpoint, r.did) + r.blobStore = NewProxyBlobStore(holdEndpoint, r.did, r.database, r.repositoryName) return r.blobStore } diff --git a/test-registry.sh b/test-registry.sh index d937693..ec372f6 100755 --- a/test-registry.sh +++ b/test-registry.sh @@ -1,6 +1,6 @@ #!/bin/bash -# ATCR Registry Test Script +# ATCR AppView Test Script # Tests various registry operations with ATProto storage # Configuration @@ -341,13 +341,13 @@ test_check_logs() { log_test "Check ATProto records in logs" log_info "Recent manifest PUT operations:" - docker logs atcr-registry 2>&1 | grep "Manifests()" | tail -5 || log_info "No manifest logs found" + docker logs atcr-appview 2>&1 | grep "Manifests()" | tail -5 || log_info "No manifest logs found" log_info "Recent tag operations:" - docker logs atcr-registry 2>&1 | grep "debian_12-slim\|debian_latest\|alpine_latest" | tail -10 || log_info "No tag logs found" + docker logs atcr-appview 2>&1 | grep "debian_12-slim\|debian_latest\|alpine_latest" | tail -10 || log_info "No tag logs found" log_info "Using cached access token:" - docker logs atcr-registry 2>&1 | grep "Using cached access token" | tail -3 || log_info "No token cache logs found" + docker logs atcr-appview 2>&1 | grep "Using cached access token" | tail -3 || log_info "No token cache logs found" log_success "Log check complete" return 0 @@ -367,7 +367,7 @@ test_head_request() { main() { echo -e "${GREEN}" echo "╔═══════════════════════════════════════╗" - echo "║ ATCR Registry Test Suite ║" + echo "║ ATCR AppView Test Suite ║" echo "║ Testing ATProto + OCI Registry ║" echo "╚═══════════════════════════════════════╝" echo -e "${NC}"