diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 84a1eb7..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "permissions": { - "allow": [ - "WebSearch", - "WebFetch(domain:github.com)", - "WebFetch(domain:pkg.go.dev)", - "WebFetch(domain:distribution.github.io)", - "Write(*)", - "Edit(*)", - "Bash(find:*)", - "Bash(curl:*)", - "Bash(sed:*)", - "Bash(grep:*)", - "Bash(gofmt:*)", - "Bash(mkdir:*)", - "Bash(golangci-lint run:*)", - "Bash(go run:*)", - "Bash(go install:*)", - "Bash(go test:*)", - "Bash(go build:*)", - "Bash(go tool:*)", - "Bash(go vet:*)", - "Bash(go get:*)", - "Bash(go mod:*)", - "Bash(go get:*)" - ], - "deny": [], - "ask": [] - } -} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7f8beb7..fac4cce 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ bin/ .env # IDE +.claude/ .vscode/ .idea/ *.swp diff --git a/CLAUDE.md b/CLAUDE.md index 016e36a..1b3a9af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -285,6 +285,42 @@ Later (docker push): - Implements full `distribution.BlobStore` interface - Used when user has `io.atcr.hold` record +#### AppView Web UI (`pkg/appview/`) + +The AppView includes a web interface for browsing the registry: + +**Features:** +- Repository browsing and search +- Star/favorite repositories +- Pull count tracking +- User profiles and settings +- OAuth-based authentication for web users + +**Database Layer** (`pkg/appview/db/`): +- SQLite database for metadata (stars, pulls, repository info) +- Schema migrations via SQL files in `pkg/appview/db/schema.go` +- Stores: OAuth sessions, device flows, repository metadata +- **NOTE:** Simple SQLite for MVP. For production multi-instance: use PostgreSQL + +**Jetstream Integration** (`pkg/appview/jetstream/`): +- Consumes ATProto Jetstream for real-time updates +- Backfills repository records from PDS +- Indexes manifests, tags, and repository metadata +- Worker processes incoming events + +**Web Handlers** (`pkg/appview/handlers/`): +- `home.go` - Landing page +- `repository.go` - Repository detail pages +- `search.go` - Search functionality +- `auth.go` - OAuth login/logout for web +- `settings.go` - User settings management +- `api.go` - JSON API endpoints + +**Static Assets** (`pkg/appview/static/`, `pkg/appview/templates/`): +- Templates use Go html/template +- JavaScript in `static/js/app.js` +- Minimal CSS for clean UI + #### Hold Service (`cmd/hold/`) Lightweight standalone service for BYOS (Bring Your Own Storage): @@ -403,6 +439,8 @@ This ensures: - Name resolver under `middleware.registry` - Default storage endpoint: `middleware.registry.options.default_storage_endpoint` - Auth token signing keys and expiration +- Database path: `db.path` (SQLite database location) +- Jetstream endpoint: `jetstream.endpoint` (for ATProto event streaming) **Hold Service configuration** (environment variables): - Storage driver config via env vars: `STORAGE_DRIVER`, `AWS_*`, `S3_*` @@ -481,6 +519,20 @@ When writing tests: 4. Add case to `buildStorageConfig()` in `cmd/hold/main.go` 5. Update `.env.example` with new driver's env vars +**Working with the database**: +- Schema defined in `pkg/appview/db/schema.go` +- Queries in `pkg/appview/db/queries.go` +- Stores for OAuth, devices, sessions in separate files +- Run migrations automatically on startup +- Database path configurable via config.yml + +**Adding web UI features**: +- Add handler in `pkg/appview/handlers/` +- Register route in `cmd/appview/serve.go` +- Create template in `pkg/appview/templates/pages/` +- Use existing auth middleware for protected routes +- API endpoints return JSON, pages return HTML + ## Important Context Values When working with the codebase, these context values are used for routing: diff --git a/README.md b/README.md index e9b57a6..d095006 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,59 @@ ATCR is an OCI-compliant container registry that integrates with the AT Protocol ### Architecture -- **Manifests**: Stored as ATProto records in user PDSs (small JSON metadata) -- **Blobs/Layers**: Stored in S3 (large binary data) -- **Name Resolution**: Supports both ATProto handles and DIDs +ATCR consists of three main components: + +1. **AppView** - OCI registry server + web UI + - Serves OCI Distribution API (Docker push/pull) + - Resolves identities (handle/DID → PDS endpoint) + - Routes manifests to user's PDS, blobs to storage + - Web interface for browsing and search + - SQLite database for stars, pulls, metadata + +2. **Hold Service** - Optional storage service (BYOS) + - Lightweight HTTP server for presigned URLs + - Supports S3, Storj, Minio, filesystem, etc. + - Authorization via ATProto records + - Users can deploy their own hold + +3. **Credential Helper** - Client-side OAuth + - ATProto OAuth with DPoP + - Exchanges OAuth token for registry JWT + - Seamless Docker integration + +**Storage Model:** +- **Manifests** → ATProto records in user PDSs (small JSON metadata) +- **Blobs/Layers** → S3 or user's hold service (large binary data) +- **Name Resolution** → Supports both handles and DIDs - `atcr.io/alice.bsky.social/myimage:latest` - `atcr.io/did:plc:xyz123/myimage:latest` ## Features -- OCI Distribution Spec compliant -- ATProto-native manifest storage -- S3 blob storage for container layers -- DID/handle resolution -- Decentralized manifest ownership +### Core Registry +- **OCI Distribution Spec compliant** - Works with Docker, containerd, podman +- **ATProto-native manifest storage** - Manifests stored as records in user PDSs +- **Hybrid storage** - Small manifests in ATProto, large blobs in S3/BYOS +- **DID/handle resolution** - Supports both handles and DIDs for image names +- **Decentralized ownership** - Users own their manifest data via their PDS + +### Web Interface +- **Repository browser** - Browse and search container images +- **Star repositories** - Favorite images for quick access +- **Pull tracking** - View popularity and usage metrics +- **OAuth authentication** - Sign in with your ATProto identity +- **User profiles** - Manage your default storage hold + +### Authentication +- **ATProto OAuth with DPoP** - Cryptographic proof-of-possession tokens +- **Docker credential helper** - Seamless `docker push/pull` workflow +- **Token exchange** - OAuth tokens converted to registry JWTs + +### Storage +- **BYOS (Bring Your Own Storage)** - Deploy your own hold service +- **Multi-backend support** - S3, Storj, Minio, Azure, GCS, filesystem +- **Presigned URLs** - Direct client-to-storage uploads/downloads +- **Hold discovery** - Automatic routing based on user preferences ## Building @@ -35,20 +75,6 @@ docker build -t atcr.io/appview:latest . docker build -f Dockerfile.hold -t atcr.io/hold:latest . ``` -## Quick Start (Local Testing) - -**Automated setup:** -```bash -# Run the test script (handles everything) -./test-local.sh -``` - -The script will: -1. Create necessary directories (`/var/lib/atcr/*`) -2. Build all binaries -3. Start registry and hold service -4. Show you how to test - **Manual setup:** ```bash # 1. Create directories @@ -192,13 +218,29 @@ Key settings: ## Usage +### Configure Credential Helper (Recommended) + +```bash +# Build and configure the credential helper +go build -o docker-credential-atcr ./cmd/credential-helper +./docker-credential-atcr configure +# Follow the OAuth flow in your browser + +# Add to Docker config (~/.docker/config.json) +{ + "credHelpers": { + "atcr.io": "atcr" + } +} +``` + ### Pushing an Image ```bash # Tag your image docker tag myapp:latest atcr.io/alice/myapp:latest -# Push to ATCR +# Push to ATCR (credential helper handles auth) docker push atcr.io/alice/myapp:latest ``` @@ -209,19 +251,52 @@ docker push atcr.io/alice/myapp:latest docker pull atcr.io/alice/myapp:latest ``` +### Web Interface + +Visit the AppView URL (default: http://localhost:5000) to: +- Browse repositories +- Search for images +- Star your favorites +- View pull statistics +- Manage your storage settings + ## Development ### Project Structure ``` atcr.io/ -├── cmd/appview/ # AppView entrypoint +├── cmd/ +│ ├── appview/ # AppView entrypoint (registry + web UI) +│ ├── hold/ # Hold service entrypoint (BYOS) +│ └── credential-helper/ # Docker credential helper ├── pkg/ -│ ├── atproto/ # ATProto client and manifest store -│ ├── storage/ # S3 blob store and routing -│ ├── middleware/ # Registry and repository middleware -│ └── server/ # HTTP handlers -├── config/ # Configuration files +│ ├── appview/ # Web UI components +│ │ ├── handlers/ # HTTP handlers (home, repo, search, auth) +│ │ ├── db/ # SQLite database layer +│ │ ├── jetstream/ # ATProto Jetstream consumer +│ │ ├── static/ # JS, CSS assets +│ │ └── templates/ # HTML templates +│ ├── atproto/ # ATProto integration +│ │ ├── client.go # PDS client +│ │ ├── resolver.go # DID/handle resolution +│ │ ├── manifest_store.go # OCI manifest store +│ │ ├── lexicon.go # ATProto record schemas +│ │ └── profile.go # Sailor profile management +│ ├── storage/ # Storage layer +│ │ ├── routing_repository.go # Routes manifests/blobs +│ │ ├── proxy_blob_store.go # BYOS proxy +│ │ ├── s3_blob_store.go # S3 wrapper +│ │ └── hold_cache.go # Hold endpoint cache +│ ├── middleware/ # Registry middleware +│ │ ├── registry.go # Name resolution +│ │ └── repository.go # Storage routing +│ └── auth/ # Authentication +│ ├── oauth/ # ATProto OAuth with DPoP +│ ├── token/ # JWT issuer/validator +│ └── atproto/ # Session validation +├── config/ # Configuration files +├── docs/ # Documentation └── Dockerfile ``` diff --git a/cmd/appview/serve.go b/cmd/appview/serve.go index f3ce948..8c5a7df 100644 --- a/cmd/appview/serve.go +++ b/cmd/appview/serve.go @@ -541,6 +541,8 @@ func initializeUIRoutes(database *sql.DB, readOnlyDB *sql.DB, sessionStore *db.S DB: readOnlyDB, Templates: templates, RegistryURL: uihandlers.TrimRegistryURL(baseURL), + Directory: oauthApp.Directory(), + Refresher: refresher, }, )).Methods("GET") diff --git a/SPEC.md b/docs/SPEC.md similarity index 100% rename from SPEC.md rename to docs/SPEC.md diff --git a/pkg/appview/handlers/repository.go b/pkg/appview/handlers/repository.go index b59cabe..b06b51a 100644 --- a/pkg/appview/handlers/repository.go +++ b/pkg/appview/handlers/repository.go @@ -3,9 +3,14 @@ package handlers import ( "database/sql" "html/template" + "log" "net/http" "atcr.io/pkg/appview/db" + "atcr.io/pkg/appview/middleware" + "atcr.io/pkg/atproto" + "atcr.io/pkg/auth/oauth" + "github.com/bluesky-social/indigo/atproto/identity" "github.com/gorilla/mux" ) @@ -14,6 +19,8 @@ type RepositoryPageHandler struct { DB *sql.DB Templates *template.Template RegistryURL string + Directory identity.Directory + Refresher *oauth.Refresher } func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -45,14 +52,44 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request return } + // Fetch star count + stats, err := db.GetRepositoryStats(h.DB, owner.DID, repository) + if err != nil { + log.Printf("Failed to fetch repository stats: %v", err) + // Continue with zero stats on error + stats = &db.RepositoryStats{StarCount: 0} + } + + // Check if current user has starred this repo + isStarred := false + user := middleware.GetUser(r) + if user != nil && h.Refresher != nil && h.Directory != nil { + // Get OAuth session for the authenticated user + session, err := h.Refresher.GetSession(r.Context(), user.DID) + if err == nil { + // Get user's PDS client + apiClient := session.APIClient() + pdsClient := atproto.NewClientWithIndigoClient(user.PDSEndpoint, user.DID, apiClient) + + // Check if star record exists + rkey := atproto.StarRecordKey(owner.DID, repository) + _, err = pdsClient.GetRecord(r.Context(), atproto.StarCollection, rkey) + isStarred = (err == nil) + } + } + data := struct { PageData Owner *db.User // Repository owner Repository *db.Repository + StarCount int + IsStarred bool }{ PageData: NewPageData(r, h.RegistryURL), Owner: owner, Repository: repo, + StarCount: stats.StarCount, + IsStarred: isStarred, } if err := h.Templates.ExecuteTemplate(w, "repository", data); err != nil { diff --git a/pkg/appview/static/js/app.js b/pkg/appview/static/js/app.js index 50eb0c2..ddb6ca1 100644 --- a/pkg/appview/static/js/app.js +++ b/pkg/appview/static/js/app.js @@ -115,9 +115,6 @@ document.addEventListener('DOMContentLoaded', () => { dropdownMenu.setAttribute('hidden', ''); } } - - // Load star status on repository page - loadStarStatus(); }); // Toggle star on a repository diff --git a/pkg/appview/templates/pages/repository.html b/pkg/appview/templates/pages/repository.html index b5fec9e..263b2df 100644 --- a/pkg/appview/templates/pages/repository.html +++ b/pkg/appview/templates/pages/repository.html @@ -36,9 +36,9 @@
-
diff --git a/test-registry.sh b/test-registry.sh deleted file mode 100755 index ec372f6..0000000 --- a/test-registry.sh +++ /dev/null @@ -1,396 +0,0 @@ -#!/bin/bash - -# ATCR AppView Test Script -# Tests various registry operations with ATProto storage - -# Configuration -REGISTRY="127.0.0.1:5000" -HANDLE="evan.jarrett.net" -IMAGE_PREFIX="${REGISTRY}/${HANDLE}" - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -# Test tracking -declare -a TEST_NAMES -declare -a TEST_RESULTS -declare -a TEST_ERRORS -TEST_COUNT=0 - -# Helper functions -log_test() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}TEST: $1${NC}" - echo -e "${BLUE}========================================${NC}" -} - -log_success() { - echo -e "${GREEN}✓ $1${NC}" -} - -log_info() { - echo -e "${YELLOW}ℹ $1${NC}" -} - -log_error() { - echo -e "${RED}✗ $1${NC}" -} - -# Run a test and track results -run_test() { - local test_name="$1" - local test_func="$2" - - TEST_NAMES[$TEST_COUNT]="$test_name" - - # Capture output and errors - local output - local exit_code - - if output=$($test_func 2>&1); then - TEST_RESULTS[$TEST_COUNT]="PASS" - TEST_ERRORS[$TEST_COUNT]="" - echo "$output" - else - exit_code=$? - TEST_RESULTS[$TEST_COUNT]="FAIL" - TEST_ERRORS[$TEST_COUNT]="$output" - echo "$output" - log_error "Test failed with exit code: $exit_code" - fi - - ((TEST_COUNT++)) -} - -# Display test summary -show_summary() { - local pass_count=0 - local fail_count=0 - - echo -e "\n${BLUE}╔═══════════════════════════════════════╗${NC}" - echo -e "${BLUE}║ TEST SUMMARY ║${NC}" - echo -e "${BLUE}╔═══════════════════════════════════════╗${NC}\n" - - for ((i=0; i/dev/null; then - log_error "Docker not available" - exit 1 - fi - - get_credentials -} - -# Prepare test images -prepare_images() { - log_info "Preparing test images..." - - log_info "Pulling debian:12-slim..." - docker pull debian:12-slim - - log_info "Tagging debian:12-slim..." - docker tag debian:12-slim ${IMAGE_PREFIX}/debian:12-slim - - log_info "Pushing initial debian:12-slim..." - docker push ${IMAGE_PREFIX}/debian:12-slim - - log_success "Test images prepared" -} - -# Test 1: Multiple tags pointing to same manifest -test_multiple_tags() { - log_test "Multiple tags pointing to same manifest" - - log_info "Tagging debian:12-slim with multiple tags..." - docker tag ${IMAGE_PREFIX}/debian:12-slim ${IMAGE_PREFIX}/debian:latest - docker tag ${IMAGE_PREFIX}/debian:12-slim ${IMAGE_PREFIX}/debian:bookworm - - log_info "Pushing tags..." - if ! docker push ${IMAGE_PREFIX}/debian:latest; then - log_error "Failed to push debian:latest" - return 1 - fi - if ! docker push ${IMAGE_PREFIX}/debian:bookworm; then - log_error "Failed to push debian:bookworm" - return 1 - fi - - log_success "Multiple tags pushed successfully" - log_info "All three tags should point to the same manifest digest" - return 0 -} - -# Test 2: Pull by digest -test_pull_by_digest() { - log_test "Pull by digest (immutable reference)" - - # Get the manifest digest from docker inspect - log_info "Getting manifest digest..." - DIGEST=$(docker inspect ${IMAGE_PREFIX}/debian:12-slim --format='{{index .RepoDigests 0}}' | cut -d'@' -f2) - - if [ -z "$DIGEST" ]; then - log_error "Could not get digest" - return 1 - fi - - log_info "Digest: $DIGEST" - - log_info "Removing local image..." - docker rmi ${IMAGE_PREFIX}/debian:12-slim 2>/dev/null || log_info "Image already removed" - - log_info "Pulling by digest..." - if ! docker pull ${IMAGE_PREFIX}/debian@${DIGEST}; then - log_error "Manifest verification failed - known issue with digest storage" - log_info "The registry stores manifests correctly but digest verification may differ" - # Don't fail - this is a known limitation - return 0 - fi - - log_success "Pull by digest successful" - return 0 -} - -# Test 3: Layer deduplication -test_layer_deduplication() { - log_test "Layer deduplication (shared layers)" - - log_info "Pulling debian:12 (larger variant)..." - if ! docker pull debian:12; then - log_error "Failed to pull debian:12" - return 1 - fi - - log_info "Tagging and pushing debian:12..." - docker tag debian:12 ${IMAGE_PREFIX}/debian:12-full - if ! docker push ${IMAGE_PREFIX}/debian:12-full; then - log_error "Failed to push debian:12-full" - return 1 - fi - - log_success "Image with shared layers pushed" - log_info "Check logs - should see 'Layer already exists' or 'Mounted from'" - return 0 -} - -# Test 4: Multiple repositories -test_multiple_repos() { - log_test "Multiple repositories" - - log_info "Pulling alpine:latest..." - if ! docker pull alpine:latest; then - log_error "Failed to pull alpine:latest" - return 1 - fi - - log_info "Tagging alpine..." - docker tag alpine:latest ${IMAGE_PREFIX}/alpine:latest - docker tag alpine:latest ${IMAGE_PREFIX}/alpine:3 - - log_info "Pushing alpine..." - if ! docker push ${IMAGE_PREFIX}/alpine:latest; then - log_error "Failed to push alpine:latest" - return 1 - fi - if ! docker push ${IMAGE_PREFIX}/alpine:3; then - log_error "Failed to push alpine:3" - return 1 - fi - - log_success "Multiple repositories created" - return 0 -} - -# Test 5: Catalog API -test_catalog_api() { - log_test "Catalog API (list repositories)" - - log_info "Fetching repository catalog..." - local response=$(curl -s -u "${CREDENTIALS}" http://${REGISTRY}/v2/_catalog) - - echo "$response" | jq . - - if echo "$response" | grep -q '"errors"'; then - log_info "Expected: Registry requires OAuth tokens for API access (not basic auth)" - log_success "Catalog API responded (OAuth required)" - return 0 - fi - - log_success "Catalog API works" - return 0 -} - -# Test 6: List tags -test_list_tags() { - log_test "List tags for repository" - - log_info "Listing tags for debian repository..." - local debian_response=$(curl -s -u "${CREDENTIALS}" http://${REGISTRY}/v2/${HANDLE}/debian/tags/list) - echo "$debian_response" | jq . - - if echo "$debian_response" | grep -q '"errors"'; then - log_info "Expected: Registry requires OAuth tokens for API access (not basic auth)" - log_success "Tags API responded (OAuth required)" - return 0 - fi - - log_success "Tag listing works" - return 0 -} - -# Test 7: Inspect manifest -test_inspect_manifest() { - log_test "Inspect manifest directly" - - log_info "Fetching manifest for debian:12-slim..." - local manifest_response=$(curl -s -u "${CREDENTIALS}" \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - http://${REGISTRY}/v2/${HANDLE}/debian/manifests/12-slim) - - echo "$manifest_response" | jq . - - if echo "$manifest_response" | grep -q '"errors"'; then - log_info "Expected: Registry requires OAuth tokens for API access (not basic auth)" - log_success "Manifest API responded (OAuth required)" - return 0 - fi - - log_success "Manifest inspection works" - return 0 -} - -# Test 8: Re-pull after clearing cache -test_repull() { - log_test "Re-pull after clearing local cache" - - log_info "Removing all local ATCR images..." - docker images --format "{{.Repository}}:{{.Tag}}" | grep "^${REGISTRY}" | xargs -r docker rmi 2>/dev/null || log_info "No images to remove" - - log_info "Pulling debian:latest from ATCR..." - if ! docker pull ${IMAGE_PREFIX}/debian:latest; then - log_error "Failed to pull debian:latest" - return 1 - fi - - log_info "Pulling alpine:latest from ATCR..." - if ! docker pull ${IMAGE_PREFIX}/alpine:latest; then - log_error "Failed to pull alpine:latest" - return 1 - fi - - log_success "Re-pull from ATProto storage successful" - - log_info "Verifying images..." - docker images | grep "${REGISTRY}" - return 0 -} - -# Test 9: Check ATProto records in logs -test_check_logs() { - log_test "Check ATProto records in logs" - - log_info "Recent manifest PUT operations:" - docker logs atcr-appview 2>&1 | grep "Manifests()" | tail -5 || log_info "No manifest logs found" - - log_info "Recent tag operations:" - docker logs atcr-appview 2>&1 | grep "debian_12-slim\|debian_latest\|alpine_latest" | tail -10 || log_info "No tag logs found" - - log_info "Using cached access token:" - docker logs atcr-appview 2>&1 | grep "Using cached access token" | tail -3 || log_info "No token cache logs found" - - log_success "Log check complete" - return 0 -} - -# Test 10: HEAD request (check blob existence) -test_head_request() { - log_test "HEAD request (check blob existence)" - - log_info "Skipping: Direct API calls require OAuth tokens" - log_info "Docker client handles blob access via credential helper" - log_success "Blob access works via Docker (tested in previous tests)" - return 0 -} - -# Main test runner -main() { - echo -e "${GREEN}" - echo "╔═══════════════════════════════════════╗" - echo "║ ATCR AppView Test Suite ║" - echo "║ Testing ATProto + OCI Registry ║" - echo "╚═══════════════════════════════════════╝" - echo -e "${NC}" - - check_login - prepare_images - - # Run tests - run_test "Multiple tags pointing to same manifest" test_multiple_tags - run_test "Pull by digest (immutable reference)" test_pull_by_digest - run_test "Layer deduplication (shared layers)" test_layer_deduplication - run_test "Multiple repositories" test_multiple_repos - run_test "Catalog API (list repositories)" test_catalog_api - run_test "List tags for repository" test_list_tags - run_test "Inspect manifest directly" test_inspect_manifest - run_test "Re-pull after clearing cache" test_repull - run_test "Check ATProto records in logs" test_check_logs - run_test "HEAD request (blob existence)" test_head_request - - # Show summary - show_summary - exit $? -} - -# Run tests -main "$@"