diff --git a/docs/API_KEY_MIGRATION.md b/docs/API_KEY_MIGRATION.md deleted file mode 100644 index 883cbff..0000000 --- a/docs/API_KEY_MIGRATION.md +++ /dev/null @@ -1,826 +0,0 @@ -# API Key Migration Plan - -## Overview - -Replace the session token system (used only by credential helper) with API keys that link to OAuth sessions. This simplifies authentication while maintaining all use cases. - -## Current State - -### Three Separate Auth Systems - -1. **Session Tokens** (`pkg/auth/session/`) - - JWT-like tokens: `.` - - Created after OAuth callback, shown to user to copy - - User manually pastes into credential helper config - - Validated in `/auth/token` and `/auth/exchange` - - 30-day TTL - - **Problem:** Awkward UX, requires manual copy/paste - -2. **UI Sessions** (`pkg/appview/session/`) - - Cookie-based (`atcr_session`) - - Random session ID, server-side store - - 24-hour TTL - - **Keep this - works well** - -3. **App Password Auth** (via PDS) - - Direct `com.atproto.server.createSession` call - - No AppView involvement until token request - - **Keep this - essential for non-UI users** - -## Target State - -### Two Auth Methods - -1. **API Keys** (NEW - replaces session tokens) - - Generated in UI after OAuth login - - Format: `atcr_<32_bytes_base64>` - - Linked to server-side OAuth refresh token - - Multiple keys per user (laptop, CI/CD, etc.) - - Revocable without re-auth - -2. **App Passwords** (KEEP) - - Direct PDS authentication - - Works without UI/OAuth - -### UI Sessions (UNCHANGED) -- Cookie-based for web UI -- Separate system, no changes needed - ---- - -## Implementation Plan - -### Phase 1: API Key System - -#### 1.1 Create API Key Store (`pkg/appview/apikey/store.go`) - -```go -package apikey - -import ( - "crypto/rand" - "encoding/base64" - "encoding/json" - "fmt" - "os" - "sync" - "time" - "golang.org/x/crypto/bcrypt" -) - -// APIKey represents a user's API key -type APIKey struct { - ID string `json:"id"` // UUID - KeyHash string `json:"key_hash"` // bcrypt hash - DID string `json:"did"` // Owner's DID - Handle string `json:"handle"` // Owner's handle - Name string `json:"name"` // User-provided name - CreatedAt time.Time `json:"created_at"` - LastUsed time.Time `json:"last_used"` -} - -// Store manages API keys -type Store struct { - mu sync.RWMutex - keys map[string]*APIKey // keyHash -> APIKey - byDID map[string][]string // DID -> []keyHash - filePath string // /var/lib/atcr/api-keys.json -} - -// NewStore creates a new API key store -func NewStore(filePath string) (*Store, error) - -// Generate creates a new API key and returns the plaintext key (shown once) -func (s *Store) Generate(did, handle, name string) (key string, keyID string, err error) - -// Validate checks if an API key is valid and returns the associated data -func (s *Store) Validate(key string) (*APIKey, error) - -// List returns all API keys for a DID (without plaintext keys) -func (s *Store) List(did string) []*APIKey - -// Delete removes an API key -func (s *Store) Delete(did, keyID string) error - -// UpdateLastUsed updates the last used timestamp -func (s *Store) UpdateLastUsed(keyHash string) error -``` - -**Key Generation:** -```go -func (s *Store) Generate(did, handle, name string) (string, string, error) { - // Generate 32 random bytes - b := make([]byte, 32) - if _, err := rand.Read(b); err != nil { - return "", "", err - } - - // Format: atcr_ - key := "atcr_" + base64.RawURLEncoding.EncodeToString(b) - - // Hash for storage - keyHash, err := bcrypt.GenerateFromPassword([]byte(key), bcrypt.DefaultCost) - if err != nil { - return "", "", err - } - - // Generate ID - keyID := generateUUID() - - apiKey := &APIKey{ - ID: keyID, - KeyHash: string(keyHash), - DID: did, - Handle: handle, - Name: name, - CreatedAt: time.Now(), - LastUsed: time.Time{}, // Never used yet - } - - s.mu.Lock() - s.keys[string(keyHash)] = apiKey - s.byDID[did] = append(s.byDID[did], string(keyHash)) - s.mu.Unlock() - - s.save() - - // Return plaintext key (only time it's available) - return key, keyID, nil -} -``` - -**Key Validation:** -```go -func (s *Store) Validate(key string) (*APIKey, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - // Try to match against all stored hashes - for hash, apiKey := range s.keys { - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(key)); err == nil { - // Update last used asynchronously - go s.UpdateLastUsed(hash) - return apiKey, nil - } - } - - return nil, fmt.Errorf("invalid API key") -} -``` - -#### 1.2 Add API Key Handlers (`pkg/appview/handlers/apikeys.go`) - -```go -package handlers - -import ( - "encoding/json" - "html/template" - "net/http" - "github.com/gorilla/mux" - "atcr.io/pkg/appview/apikey" - "atcr.io/pkg/appview/middleware" -) - -// GenerateAPIKeyHandler handles POST /api/keys -type GenerateAPIKeyHandler struct { - Store *apikey.Store -} - -func (h *GenerateAPIKeyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - user := middleware.GetUser(r) - if user == nil { - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - - name := r.FormValue("name") - if name == "" { - name = "Unnamed Key" - } - - key, keyID, err := h.Store.Generate(user.DID, user.Handle, name) - if err != nil { - http.Error(w, "Failed to generate key", http.StatusInternalServerError) - return - } - - // Return key (shown once!) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]string{ - "id": keyID, - "key": key, - }) -} - -// ListAPIKeysHandler handles GET /api/keys -type ListAPIKeysHandler struct { - Store *apikey.Store -} - -func (h *ListAPIKeysHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - user := middleware.GetUser(r) - if user == nil { - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - - keys := h.Store.List(user.DID) - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(keys) -} - -// DeleteAPIKeyHandler handles DELETE /api/keys/{id} -type DeleteAPIKeyHandler struct { - Store *apikey.Store -} - -func (h *DeleteAPIKeyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - user := middleware.GetUser(r) - if user == nil { - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - - vars := mux.Vars(r) - keyID := vars["id"] - - if err := h.Store.Delete(user.DID, keyID); err != nil { - http.Error(w, "Failed to delete key", http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} -``` - -### Phase 2: Update Token Handler - -#### 2.1 Modify `/auth/token` Handler (`pkg/auth/token/handler.go`) - -```go -type Handler struct { - issuer *Issuer - validator *atproto.SessionValidator - apiKeyStore *apikey.Store // NEW - defaultHoldEndpoint string -} - -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - username, password, ok := r.BasicAuth() - if !ok { - return unauthorized - } - - var did, handle, accessToken string - - // 1. Check if it's an API key (NEW) - if strings.HasPrefix(password, "atcr_") { - apiKey, err := h.apiKeyStore.Validate(password) - if err != nil { - fmt.Printf("DEBUG [token/handler]: API key validation failed: %v\n", err) - return unauthorized - } - - did = apiKey.DID - handle = apiKey.Handle - fmt.Printf("DEBUG [token/handler]: API key validated for DID=%s, handle=%s\n", did, handle) - - // API key is linked to OAuth session - // OAuth refresher will provide access token when needed via middleware - } - // 2. Try app password (direct PDS) - else { - did, handle, accessToken, err = h.validator.CreateSessionAndGetToken(r.Context(), username, password) - if err != nil { - fmt.Printf("DEBUG [token/handler]: App password validation failed: %v\n", err) - return unauthorized - } - - fmt.Printf("DEBUG [token/handler]: App password validated, DID=%s\n", did) - - // Cache access token for manifest operations - auth.GetGlobalTokenCache().Set(did, accessToken, 2*time.Hour) - - // Ensure profile exists - // ... existing code ... - } - - // Rest of handler: validate access, issue JWT, etc. - // ... existing code ... -} -``` - -**Key Changes:** -- Remove session token validation (`sessionManager.Validate()`) -- Add API key check as first priority -- Keep app password as fallback -- API keys use OAuth refresher (server-side), app passwords use token cache (client-side) - -#### 2.2 Remove `/auth/exchange` Endpoint - -The `/auth/exchange` endpoint was only used for exchanging session tokens for registry JWTs. With API keys, this is no longer needed. - -**Files to delete:** -- `pkg/auth/exchange/handler.go` - -**Files to update:** -- `cmd/appview/serve.go` - Remove exchange handler registration - -### Phase 3: Update UI - -#### 3.1 Add API Keys Section to Settings Page - -**Template** (`pkg/appview/templates/settings.html`): - -```html - -
-

API Keys

-

Generate API keys for Docker CLI and CI/CD. Each key is linked to your OAuth session.

- - -
-

Generate New API Key

-
- - -
-
- - - - - -
-

Your API Keys

- - - - - - - - - - - - -
NameCreatedLast UsedActions
-
-
- - - - -``` - -#### 3.2 Register API Key Routes (`cmd/appview/serve.go`) - -```go -// In initializeUI() function, add: - -// API key management routes (authenticated) -authRouter.Handle("/api/keys", &uihandlers.GenerateAPIKeyHandler{ - Store: apiKeyStore, -}).Methods("POST") - -authRouter.Handle("/api/keys", &uihandlers.ListAPIKeysHandler{ - Store: apiKeyStore, -}).Methods("GET") - -authRouter.Handle("/api/keys/{id}", &uihandlers.DeleteAPIKeyHandler{ - Store: apiKeyStore, -}).Methods("DELETE") -``` - -### Phase 4: Update Credential Helper - -#### 4.1 Simplify Configuration (`cmd/credential-helper/main.go`) - -```go -// SessionStore becomes CredentialStore -type CredentialStore struct { - Handle string `json:"handle"` - APIKey string `json:"api_key"` - AppViewURL string `json:"appview_url"` -} - -func handleConfigure(handle string) { - fmt.Println("ATCR Credential Helper Configuration") - fmt.Println("=====================================") - fmt.Println() - fmt.Println("You need an API key from the ATCR web UI.") - fmt.Println() - - appViewURL := os.Getenv("ATCR_APPVIEW_URL") - if appViewURL == "" { - appViewURL = defaultAppViewURL - } - - // Auto-open settings page - settingsURL := appViewURL + "/settings" - fmt.Printf("Opening settings page: %s\n", settingsURL) - fmt.Println("Log in and generate an API key if you haven't already.") - fmt.Println() - - if err := oauth.OpenBrowser(settingsURL); err != nil { - fmt.Printf("Could not open browser. Please visit: %s\n\n", settingsURL) - } - - // Prompt for credentials - if handle == "" { - fmt.Print("Enter your ATProto handle (e.g., alice.bsky.social): ") - fmt.Scanln(&handle) - } else { - fmt.Printf("Using handle: %s\n", handle) - } - - fmt.Print("Enter your API key (from settings page): ") - var apiKey string - fmt.Scanln(&apiKey) - - // Validate key format - if !strings.HasPrefix(apiKey, "atcr_") { - fmt.Fprintf(os.Stderr, "Invalid API key format. Key should start with 'atcr_'\n") - os.Exit(1) - } - - // Save credentials - creds := &CredentialStore{ - Handle: handle, - APIKey: apiKey, - AppViewURL: appViewURL, - } - - if err := saveCredentials(getCredentialsPath(), creds); err != nil { - fmt.Fprintf(os.Stderr, "Error saving credentials: %v\n", err) - os.Exit(1) - } - - fmt.Println() - fmt.Println("✓ Configuration complete!") - fmt.Println("You can now use docker push/pull with atcr.io") -} - -func handleGet() { - var serverURL string - fmt.Fscanln(os.Stdin, &serverURL) - - // Load credentials - creds, err := loadCredentials(getCredentialsPath()) - if err != nil { - fmt.Fprintf(os.Stderr, "Error loading credentials: %v\n", err) - fmt.Fprintf(os.Stderr, "Please run: docker-credential-atcr configure\n") - os.Exit(1) - } - - // Return credentials for Docker - // Docker will send these as Basic Auth to /auth/token - response := Credentials{ - ServerURL: serverURL, - Username: creds.Handle, - Secret: creds.APIKey, // API key as password - } - - json.NewEncoder(os.Stdout).Encode(response) -} -``` - -**File Rename:** -- `~/.atcr/session.json` → `~/.atcr/credentials.json` - -### Phase 5: Remove Session Token System - -#### 5.1 Delete Session Token Files - -**Files to delete:** -- `pkg/auth/session/handler.go` -- `pkg/auth/exchange/handler.go` - -#### 5.2 Update OAuth Server (`pkg/auth/oauth/server.go`) - -**Remove session token creation:** -```go -// OLD (delete this): -sessionToken, err := s.sessionManager.Create(did, handle) -if err != nil { - s.renderError(w, fmt.Sprintf("Failed to create session token: %v", err)) - return -} - -// Check if this is a UI login... -if cookie, err := r.Cookie("oauth_return_to"); err == nil && s.uiSessionStore != nil { - // UI flow... -} else { - // Render success page with session token (for credential helper) - s.renderSuccess(w, sessionToken, handle) -} -``` - -**NEW (replace with):** -```go -// Check if this is a UI login -if cookie, err := r.Cookie("oauth_return_to"); err == nil && s.uiSessionStore != nil { - // Create UI session - uiSessionID, err := s.uiSessionStore.Create(did, handle, sessionData.HostURL, 24*time.Hour) - // ... set cookie, redirect ... -} else { - // Non-UI flow: redirect to settings to get API key - s.renderRedirectToSettings(w, handle) -} -``` - -**Add redirect to settings template:** -```go -func (s *Server) renderRedirectToSettings(w http.ResponseWriter, handle string) { - tmpl := template.Must(template.New("redirect").Parse(` - - - - Authorization Successful - ATCR - - - -

✓ Authorization Successful!

-

Redirecting to settings page to generate your API key...

-

If not redirected, click here.

- - - `)) - w.Header().Set("Content-Type", "text/html") - tmpl.Execute(w, nil) -} -``` - -#### 5.3 Update Server Constructor - -```go -// Remove sessionManager parameter -func NewServer(app *App) *Server { - return &Server{ - app: app, - } -} -``` - -#### 5.4 Update Registry Initialization (`cmd/appview/serve.go`) - -```go -// REMOVE session manager creation: -// sessionManager, err := session.NewManagerWithPersistentSecret(secretPath, 30*24*time.Hour) - -// Create API key store -apiKeyStorePath := filepath.Join(filepath.Dir(storagePath), "api-keys.json") -apiKeyStore, err := apikey.NewStore(apiKeyStorePath) -if err != nil { - return fmt.Errorf("failed to create API key store: %w", err) -} - -// OAuth server doesn't need session manager anymore -oauthServer := oauth.NewServer(oauthApp) -oauthServer.SetRefresher(refresher) -if uiSessionStore != nil { - oauthServer.SetUISessionStore(uiSessionStore) -} - -// Token handler gets API key store instead of session manager -if issuer != nil { - tokenHandler := token.NewHandler(issuer, apiKeyStore, defaultHoldEndpoint) - tokenHandler.RegisterRoutes(mux) - - // Remove exchange handler registration (no longer needed) -} -``` - ---- - -## Migration Path - -### For Existing Users - -**Option 1: Smooth Migration (Recommended)** -1. Keep session token validation temporarily with deprecation warning -2. When session token is used, log warning and return special response header -3. Docker client shows warning: "Session tokens deprecated, please regenerate API key" -4. Remove session token support in next major version - -**Option 2: Hard Cutover** -1. Deploy new version with API keys -2. Session tokens stop working immediately -3. Users must reconfigure: `docker-credential-atcr configure` -4. Cleaner but disruptive - -### Rollout Plan - -**Week 1: Deploy API Keys** -- Add API key system -- Keep session token validation -- Add deprecation notice to OAuth callback - -**Week 2-4: Migration Period** -- Monitor API key adoption -- Email users about migration -- Provide migration guide - -**Week 5: Remove Session Tokens** -- Delete session token code -- Force users to API keys - ---- - -## Testing Plan - -### Unit Tests - -1. **API Key Store** - - Test key generation (format, uniqueness) - - Test key validation (correct/incorrect keys) - - Test bcrypt hashing - - Test key listing/deletion - -2. **Token Handler** - - Test API key authentication - - Test app password authentication - - Test invalid credentials - - Test key format validation - -### Integration Tests - -1. **Full Auth Flow** - - UI login → OAuth → API key generation - - Credential helper → API key → registry JWT - - App password → registry JWT - -2. **Docker Client Tests** - - `docker login -u handle -p api_key` - - `docker login -u handle -p app_password` - - `docker push` with API key - - `docker pull` with API key - -### Security Tests - -1. **Key Security** - - Verify bcrypt hashing (not plaintext storage) - - Test key shown only once - - Test key revocation - - Test unauthorized key access - -2. **OAuth Security** - - Verify API key links to correct OAuth session - - Test expired refresh token handling - - Test multiple keys for same user - ---- - -## Files Changed - -### New Files -- `pkg/appview/apikey/store.go` - API key storage and validation -- `pkg/appview/handlers/apikeys.go` - API key HTTP handlers -- `docs/API_KEY_MIGRATION.md` - This document - -### Modified Files -- `pkg/auth/token/handler.go` - Add API key validation, remove session token -- `pkg/auth/oauth/server.go` - Remove session token creation, redirect to settings -- `pkg/appview/handlers/settings.go` - Add API key management UI -- `pkg/appview/templates/settings.html` - Add API key section -- `cmd/credential-helper/main.go` - Simplify to use API keys -- `cmd/appview/serve.go` - Initialize API key store, remove session manager - -### Deleted Files -- `pkg/auth/session/handler.go` - Session token system -- `pkg/auth/exchange/handler.go` - Exchange endpoint (no longer needed) - ---- - -## Advantages - -✅ **Simpler Auth:** Two methods instead of three (API keys + app passwords) -✅ **Better UX:** No manual copy/paste of session tokens -✅ **Multiple Keys:** Users can have laptop key, CI key, etc. -✅ **Revocable:** Revoke individual keys without re-auth -✅ **Server-Side OAuth:** Refresh tokens stay on server, not in client files -✅ **Familiar Pattern:** Matches AWS ECR, GitHub tokens, etc. - -## Backward Compatibility - -⚠️ **Breaking Change:** Session tokens will stop working -✅ **App passwords:** Still work (no changes) -✅ **UI sessions:** Still work (separate system) - -**Migration Required:** Users with session tokens must run `docker-credential-atcr configure` again to get API keys. diff --git a/docs/OAUTH.md b/docs/OAUTH.md deleted file mode 100644 index 6971e1a..0000000 --- a/docs/OAUTH.md +++ /dev/null @@ -1,281 +0,0 @@ -# ATCR OAuth Implementation - -## Overview - -ATCR now supports ATProto OAuth authentication via Docker credential helpers. This allows users to authenticate with their ATProto identity (Bluesky account) and use Docker push/pull commands seamlessly. - -## Architecture - -### Components - -1. **OAuth Client** (`pkg/auth/oauth/`) - - Full ATProto OAuth implementation with DPoP support - - Uses `authelia.com/client/oauth2` for OAuth + PAR - - Uses `github.com/AxisCommunications/go-dpop` for DPoP proof generation - - Automatic authorization server discovery - - PKCE support for security - -2. **Credential Helper** (`cmd/credential-helper/`) - - Standalone binary: `docker-credential-atcr` - - Implements Docker credential helper protocol - - Manages OAuth flow with browser - - Stores tokens securely in `~/.atcr/oauth-token.json` - -3. **Registry Integration** - - `/auth/exchange` endpoint exchanges OAuth tokens for registry JWTs - - Existing `/auth/token` endpoint for standard Docker auth - -## Dependencies - -- `authelia.com/client/oauth2` - OAuth client with PAR support (2⭐, Authelia-backed) -- `github.com/AxisCommunications/go-dpop` - DPoP implementation (10⭐, RFC 9449 compliant) -- `github.com/golang-jwt/jwt/v5` - JWT library (transitive, 11k+⭐) - -## Usage - -### Setup - -1. Build the credential helper: -```bash -go build -o docker-credential-atcr ./cmd/credential-helper -``` - -2. Install it in your PATH: -```bash -sudo mv docker-credential-atcr /usr/local/bin/ -``` - -3. Configure Docker to use it by editing `~/.docker/config.json`: -```json -{ - "credsStore": "atcr" -} -``` - -### Configuration - -Run the OAuth flow: -```bash -docker-credential-atcr configure -``` - -This will: -1. Prompt for your ATProto handle (e.g., `alice.bsky.social`) -2. Open your browser for OAuth authorization -3. Store the OAuth token and DPoP key in `~/.atcr/oauth-token.json` - -### Using with Docker - -Once configured, use Docker normally: - -```bash -# Push an image -docker push atcr.io/alice/myapp:latest - -# Pull an image -docker pull atcr.io/alice/myapp:latest -``` - -The credential helper automatically: -1. Loads your stored OAuth token -2. Refreshes it if expired -3. Exchanges it for a registry JWT -4. Provides the JWT to Docker - -## How It Works - -### OAuth Flow - -1. **User runs** `docker-credential-atcr configure` -2. **Resolve identity**: alice.bsky.social → DID → PDS endpoint -3. **Discover auth server**: GET `{pds}/.well-known/oauth-authorization-server` -4. **Generate DPoP key**: ECDSA P-256 key pair -5. **PAR request**: POST to PAR endpoint with DPoP header + PKCE challenge -6. **Open browser**: User authorizes on their PDS -7. **Receive code**: Callback to `localhost:8888/callback` -8. **Exchange code**: POST to token endpoint with DPoP header + PKCE verifier -9. **Save tokens**: Store OAuth token + DPoP key + DID/handle - -### Docker Push/Pull Flow - -1. **Docker needs credentials** for `atcr.io` -2. **Calls credential helper**: `docker-credential-atcr get` -3. **Helper loads token** from `~/.atcr/oauth-token.json` -4. **Refresh if needed**: Uses refresh token + DPoP if expired -5. **Exchange for registry JWT**: POST to `/auth/exchange` with OAuth token + handle -6. **Registry validates token**: Calls `getSession` on PDS to validate token -7. **Registry issues JWT**: Creates registry JWT with validated DID/handle -8. **Return to Docker**: `{"Username": "oauth2", "Secret": ""}` -9. **Docker uses JWT**: For authentication to registry API - -## Security - -### DPoP (Demonstrating Proof-of-Possession) - -Every OAuth request includes a DPoP proof: -- Unique JWT signed with ECDSA private key -- Contains HTTP method, URL, timestamp, nonce -- Public key (JWK) included in JWT header -- Binds the token to the specific client - -### PKCE (Proof Key for Code Exchange) - -- Code verifier generated locally -- Code challenge sent in authorization request -- Verifier sent in token exchange -- Prevents authorization code interception - -### Token Storage - -- Tokens stored in `~/.atcr/oauth-token.json` -- File permissions: 0600 (owner read/write only) -- DPoP key stored in PEM format -- Refresh tokens for long-term access - -## Implementation Details - -### Code Structure - -``` -pkg/auth/oauth/ -├── client.go # OAuth client with DPoP -├── discovery.go # Authorization server discovery -├── metadata.go # Client metadata document -├── storage.go # Token persistence -└── transport.go # DPoP HTTP transport - -pkg/auth/atproto/ -├── session.go # ATProto session validation (Basic auth) -└── validator.go # OAuth token validation via getSession - -cmd/credential-helper/ -├── main.go # Docker credential helper protocol -├── oauth.go # OAuth flow orchestration -└── token.go # Token management - -pkg/auth/exchange/ -└── handler.go # OAuth → Registry JWT exchange -``` - -### Key Classes - -**OAuth Client** (`pkg/auth/oauth/client.go`) -- `NewClient()` - Create client with DPoP key -- `InitializeForHandle()` - Discover auth server -- `AuthorizeURL()` - Generate authorization URL with PAR + PKCE -- `Exchange()` - Exchange code for token with DPoP -- `RefreshToken()` - Refresh expired token with DPoP - -**DPoP Transport** (`pkg/auth/oauth/transport.go`) -- Implements `http.RoundTripper` -- Automatically adds DPoP header to all requests -- Handles nonce management and retries -- Used by OAuth client for all HTTP requests - -**Token Store** (`pkg/auth/oauth/storage.go`) -- Persists OAuth tokens and DPoP key -- PEM encoding for private key -- Expiration checking -- Secure file permissions - -**Token Validator** (`pkg/auth/atproto/validator.go`) -- `ValidateToken()` - Validate token via PDS getSession -- `ValidateTokenWithResolver()` - Auto-resolve PDS from handle -- Returns validated DID and handle -- Used by registry to verify OAuth tokens - -## Testing - -### Manual Testing - -1. Configure the helper: -```bash -./docker-credential-atcr configure -# Enter handle: alice.bsky.social -# Browser opens for authorization -# Token saved to ~/.atcr/oauth-token.json -``` - -2. Test credential retrieval: -```bash -echo '{"ServerURL": "atcr.io"}' | ./docker-credential-atcr get -# Should return: {"Username":"oauth2","Secret":""} -``` - -3. Test with Docker: -```bash -docker push atcr.io/alice/test:latest -``` - -### Integration Testing - -TODO: Add automated tests for: -- OAuth flow with mock PDS -- DPoP proof generation -- Token exchange -- Credential helper protocol - -## Security Features - -### OAuth Token Validation - -The registry validates ATProto OAuth tokens by calling `com.atproto.server.getSession` on the user's PDS. This ensures: -- Token is valid and not expired -- Token belongs to the claimed user -- User's DID and handle are extracted from the PDS response -- No trust in client-provided identity information - -**Flow:** -1. Client sends OAuth token + handle to `/auth/exchange` -2. Registry resolves handle → PDS endpoint -3. Registry calls `{pds}/xrpc/com.atproto.server.getSession` with token -4. PDS validates token and returns session info (DID, handle) -5. Registry uses validated DID/handle to issue registry JWT - -## Future Improvements - -1. **Token refresh in background** - - Proactively refresh before expiry - - Reduce latency on Docker commands - -3. **Multiple account support** - - Store tokens for multiple handles - - Allow selecting which account to use - -4. **Revocation support** - - Implement token revocation - - Clean up on logout - -5. **Better error messages** - - User-friendly OAuth error handling - - Guide users through common issues - -## Troubleshooting - -### "Failed to resolve identity" -- Check internet connection -- Verify handle is correct (e.g., `alice.bsky.social`) -- Ensure PDS is accessible - -### "Authorization timed out" -- Complete authorization within 5 minutes -- Check if browser opened correctly -- Try running `configure` again - -### "Token expired" -- Credential helper should auto-refresh -- If persistent, run `configure` again -- Check `~/.atcr/oauth-token.json` permissions - -### "Failed to exchange token" -- Ensure registry is running -- Check `/auth/exchange` endpoint is accessible -- Verify token hasn't been revoked - -## References - -- [ATProto OAuth Specification](https://atproto.com/specs/oauth) -- [RFC 9449: DPoP](https://datatracker.ietf.org/doc/html/rfc9449) -- [RFC 9126: PAR](https://datatracker.ietf.org/doc/html/rfc9126) -- [RFC 7636: PKCE](https://datatracker.ietf.org/doc/html/rfc7636) -- [Docker Credential Helpers](https://github.com/docker/docker-credential-helpers) diff --git a/docs/QUOTAS.md b/docs/QUOTAS.md new file mode 100644 index 0000000..d30effc --- /dev/null +++ b/docs/QUOTAS.md @@ -0,0 +1,1289 @@ +# ATCR Quota System + +This document describes ATCR's storage quota implementation, inspired by Harbor's proven approach to per-project blob tracking with deduplication. + +## Table of Contents + +- [Overview](#overview) +- [Harbor's Approach (Reference Implementation)](#harbors-approach-reference-implementation) +- [Storage Options](#storage-options) +- [Quota Data Model](#quota-data-model) +- [Push Flow (Detailed)](#push-flow-detailed) +- [Delete Flow](#delete-flow) +- [Garbage Collection](#garbage-collection) +- [Quota Reconciliation](#quota-reconciliation) +- [Configuration](#configuration) +- [Trade-offs & Design Decisions](#trade-offs--design-decisions) +- [Future Enhancements](#future-enhancements) + +## Overview + +ATCR implements per-user storage quotas to: +1. **Limit storage consumption** on shared hold services +2. **Track actual S3 costs** (what new data was added) +3. **Benefit from deduplication** (users only pay once per layer) +4. **Provide transparency** (show users their storage usage) + +**Key principle:** Users pay for layers they've uploaded, but only ONCE per layer regardless of how many images reference it. + +### Example Scenario + +``` +Alice pushes myapp:v1 (layers A, B, C - each 100MB) +→ Alice's quota: +300MB (all new layers) + +Alice pushes myapp:v2 (layers A, B, D) +→ Layers A, B already claimed by Alice +→ Layer D is new (100MB) +→ Alice's quota: +100MB (only D is new) +→ Total: 400MB + +Bob pushes his-app:latest (layers A, E) +→ Layer A already exists in S3 (uploaded by Alice) +→ Bob claims it for first time → +100MB to Bob's quota +→ Layer E is new → +100MB to Bob's quota +→ Bob's quota: 200MB + +Physical S3 storage: 500MB (A, B, C, D, E) +Claimed storage: 600MB (Alice: 400MB, Bob: 200MB) +Deduplication savings: 100MB (layer A shared) +``` + +## Harbor's Approach (Reference Implementation) + +Harbor is built on distribution/distribution (same as ATCR) and implements quotas as middleware. Their approach: + +### Key Insights from Harbor + +1. **"Shared blobs are only computed once per project"** + - Each project tracks which blobs it has uploaded + - Same blob used in multiple images counts only once per project + - Different projects claiming the same blob each pay for it + +2. **Quota checked when manifest is pushed** + - Blobs upload first (presigned URLs, can't intercept) + - Manifest pushed last → quota check happens here + - Can reject manifest if quota exceeded (orphaned blobs cleaned by GC) + +3. **Middleware-based implementation** + - distribution/distribution has NO built-in quota support + - Harbor added it as request preprocessing middleware + - Uses database (PostgreSQL) or Redis for quota storage + +4. **Per-project ownership model** + - Blobs are physically deduplicated globally + - Quota accounting is logical (per-project claims) + - Total claimed storage can exceed physical storage + +### References + +- Harbor Quota Documentation: https://goharbor.io/docs/1.10/administration/configure-project-quotas/ +- Harbor Source: https://github.com/goharbor/harbor (see `src/controller/quota`) + +## Storage Options + +The hold service needs to store quota data somewhere. Two options: + +### Option 1: S3-Based Storage (Recommended for BYOS) + +Store quota metadata alongside blobs in the same S3 bucket: + +``` +Bucket structure: +/docker/registry/v2/blobs/sha256/ab/abc123.../data ← actual blobs +/atcr/quota/did:plc:alice.json ← quota tracking +/atcr/quota/did:plc:bob.json +``` + +**Pros:** +- ✅ No separate database needed +- ✅ Single S3 bucket (better UX - no second bucket to configure) +- ✅ Quota data lives with the blobs +- ✅ Hold service stays relatively stateless +- ✅ Works with any S3-compatible service (Storj, Minio, Upcloud, Fly.io) + +**Cons:** +- ❌ Slower than local database (network round-trip) +- ❌ Eventual consistency issues +- ❌ Race conditions on concurrent updates +- ❌ Extra S3 API costs (GET/PUT per upload) + +**Performance:** +- Each blob upload: 1 HEAD (blob exists?) + 1 GET (quota) + 1 PUT (update quota) +- Typical latency: 100-200ms total overhead +- For high-throughput registries, consider SQLite + +### Option 2: SQLite Database (Recommended for Shared Holds) + +Local database in hold service: + +```bash +/var/lib/atcr/hold-quota.db +``` + +**Pros:** +- ✅ Fast local queries (no network latency) +- ✅ ACID transactions (no race conditions) +- ✅ Efficient for high-throughput registries +- ✅ Can use foreign keys and joins + +**Cons:** +- ❌ Makes hold service stateful (persistent volume needed) +- ❌ Not ideal for ephemeral BYOS deployments +- ❌ Backup/restore complexity +- ❌ Multi-instance scaling requires shared database + +**Schema:** +```sql +CREATE TABLE user_quotas ( + did TEXT PRIMARY KEY, + quota_limit INTEGER NOT NULL DEFAULT 10737418240, -- 10GB + quota_used INTEGER NOT NULL DEFAULT 0, + updated_at TIMESTAMP +); + +CREATE TABLE claimed_layers ( + did TEXT NOT NULL, + digest TEXT NOT NULL, + size INTEGER NOT NULL, + claimed_at TIMESTAMP, + PRIMARY KEY(did, digest) +); +``` + +### Recommendation + +- **BYOS (user-owned holds):** S3-based (keeps hold service ephemeral) +- **Shared holds (multi-user):** SQLite (better performance and consistency) +- **High-traffic production:** SQLite or PostgreSQL (Harbor uses this) + +## Quota Data Model + +### Quota File Format (S3-based) + +```json +{ + "did": "did:plc:alice123", + "limit": 10737418240, + "used": 5368709120, + "claimed_layers": { + "sha256:abc123...": 104857600, + "sha256:def456...": 52428800, + "sha256:789ghi...": 209715200 + }, + "last_updated": "2025-10-09T12:34:56Z", + "version": 1 +} +``` + +**Fields:** +- `did`: User's ATProto DID +- `limit`: Maximum storage in bytes (default: 10GB) +- `used`: Current storage usage in bytes (sum of claimed_layers) +- `claimed_layers`: Map of digest → size for all layers user has uploaded +- `last_updated`: Timestamp of last quota update +- `version`: Schema version for future migrations + +### Why Track Individual Layers? + +**Q: Can't we just track a counter?** + +**A: We need layer tracking for:** + +1. **Deduplication detection** + - Check if user already claimed a layer → free upload + - Example: Updating an image reuses most layers + +2. **Accurate deletes** + - When manifest deleted, only decrement unclaimed layers + - User may have 5 images sharing layer A - deleting 1 image doesn't free layer A + +3. **Quota reconciliation** + - Verify quota matches reality by listing user's manifests + - Recalculate from layers in manifests vs claimed_layers map + +4. **Auditing** + - "Show me what I'm storing" + - Users can see which layers consume their quota + +## Push Flow (Detailed) + +### Step-by-Step: User Pushes Image + +``` +┌──────────┐ ┌──────────┐ ┌──────────┐ +│ Client │ │ Hold │ │ S3 │ +│ (Docker) │ │ Service │ │ Bucket │ +└──────────┘ └──────────┘ └──────────┘ + │ │ │ + │ 1. PUT /v2/.../blobs/ │ │ + │ upload?digest=sha256:abc│ │ + ├───────────────────────────>│ │ + │ │ │ + │ │ 2. Check if blob exists │ + │ │ (Stat/HEAD request) │ + │ ├───────────────────────────>│ + │ │<───────────────────────────┤ + │ │ 200 OK (exists) or │ + │ │ 404 Not Found │ + │ │ │ + │ │ 3. Read user quota │ + │ │ GET /atcr/quota/{did} │ + │ ├───────────────────────────>│ + │ │<───────────────────────────┤ + │ │ quota.json │ + │ │ │ + │ │ 4. Calculate quota impact │ + │ │ - If digest in │ + │ │ claimed_layers: 0 │ + │ │ - Else: size │ + │ │ │ + │ │ 5. Check quota limit │ + │ │ used + impact <= limit? │ + │ │ │ + │ │ 6. Update quota │ + │ │ PUT /atcr/quota/{did} │ + │ ├───────────────────────────>│ + │ │<───────────────────────────┤ + │ │ 200 OK │ + │ │ │ + │ 7. Presigned URL │ │ + │<───────────────────────────┤ │ + │ {url: "https://s3..."} │ │ + │ │ │ + │ 8. Upload blob to S3 │ │ + ├────────────────────────────┼───────────────────────────>│ + │ │ │ + │ 9. 200 OK │ │ + │<───────────────────────────┼────────────────────────────┤ + │ │ │ +``` + +### Implementation (Pseudocode) + +```go +// cmd/hold/main.go - HandlePutPresignedURL + +func (s *HoldService) HandlePutPresignedURL(w http.ResponseWriter, r *http.Request) { + var req PutPresignedURLRequest + json.NewDecoder(r.Body).Decode(&req) + + // Step 1: Check if blob already exists in S3 + blobPath := fmt.Sprintf("/docker/registry/v2/blobs/%s/%s/%s/data", + algorithm, digest[:2], digest) + + _, err := s.driver.Stat(ctx, blobPath) + blobExists := (err == nil) + + // Step 2: Read quota from S3 (or SQLite) + quota, err := s.quotaManager.GetQuota(req.DID) + if err != nil { + // First upload - create quota with defaults + quota = &Quota{ + DID: req.DID, + Limit: s.config.QuotaDefaultLimit, + Used: 0, + ClaimedLayers: make(map[string]int64), + } + } + + // Step 3: Calculate quota impact + quotaImpact := req.Size // Default: assume new layer + + if _, alreadyClaimed := quota.ClaimedLayers[req.Digest]; alreadyClaimed { + // User already uploaded this layer before + quotaImpact = 0 + log.Printf("Layer %s already claimed by %s, no quota impact", + req.Digest, req.DID) + } else if blobExists { + // Blob exists in S3 (uploaded by another user) + // But this user is claiming it for first time + // Still counts against their quota + log.Printf("Layer %s exists globally but new to %s, quota impact: %d", + req.Digest, req.DID, quotaImpact) + } else { + // Brand new blob - will be uploaded to S3 + log.Printf("New layer %s for %s, quota impact: %d", + req.Digest, req.DID, quotaImpact) + } + + // Step 4: Check quota limit + if quota.Used + quotaImpact > quota.Limit { + http.Error(w, fmt.Sprintf( + "quota exceeded: used=%d, impact=%d, limit=%d", + quota.Used, quotaImpact, quota.Limit, + ), http.StatusPaymentRequired) // 402 + return + } + + // Step 5: Update quota (optimistic - before upload completes) + quota.Used += quotaImpact + if quotaImpact > 0 { + quota.ClaimedLayers[req.Digest] = req.Size + } + quota.LastUpdated = time.Now() + + if err := s.quotaManager.SaveQuota(quota); err != nil { + http.Error(w, "failed to update quota", http.StatusInternalServerError) + return + } + + // Step 6: Generate presigned URL + presignedURL, err := s.getUploadURL(ctx, req.Digest, req.Size, req.DID) + if err != nil { + // Rollback quota update on error + quota.Used -= quotaImpact + delete(quota.ClaimedLayers, req.Digest) + s.quotaManager.SaveQuota(quota) + + http.Error(w, "failed to generate presigned URL", http.StatusInternalServerError) + return + } + + // Step 7: Return presigned URL + quota info + resp := PutPresignedURLResponse{ + URL: presignedURL, + ExpiresAt: time.Now().Add(15 * time.Minute), + QuotaInfo: QuotaInfo{ + Used: quota.Used, + Limit: quota.Limit, + Available: quota.Limit - quota.Used, + Impact: quotaImpact, + AlreadyClaimed: quotaImpact == 0, + }, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} +``` + +### Race Condition Handling + +**Problem:** Two concurrent uploads of the same blob + +``` +Time User A User B +0ms Upload layer X (100MB) +10ms Upload layer X (100MB) +20ms Check exists: NO Check exists: NO +30ms Quota impact: 100MB Quota impact: 100MB +40ms Update quota A: +100MB Update quota B: +100MB +50ms Generate presigned URL Generate presigned URL +100ms Upload to S3 completes Upload to S3 (overwrites A's) +``` + +**Result:** Both users charged 100MB, but only 100MB stored in S3. + +**Mitigation strategies:** + +1. **Accept eventual consistency** (recommended for S3-based) + - Run periodic reconciliation to fix discrepancies + - Small inconsistency window (minutes) is acceptable + - Reconciliation uses PDS as source of truth + +2. **Optimistic locking** (S3 ETags) + ```go + // Use S3 ETags for conditional writes + oldETag := getQuotaFileETag(did) + err := putQuotaFileWithCondition(quota, oldETag) + if err == PreconditionFailed { + // Retry with fresh read + } + ``` + +3. **Database transactions** (SQLite-based) + ```sql + BEGIN TRANSACTION; + SELECT * FROM user_quotas WHERE did = ? FOR UPDATE; + UPDATE user_quotas SET used = used + ? WHERE did = ?; + COMMIT; + ``` + +## Delete Flow + +### Manifest Deletion via AppView UI + +When a user deletes a manifest through the AppView web interface: + +``` +┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ +│ User │ │ AppView │ │ Hold │ │ PDS │ +│ UI │ │ Database │ │ Service │ │ │ +└──────────┘ └──────────┘ └──────────┘ └──────────┘ + │ │ │ │ + │ DELETE manifest │ │ │ + ├─────────────────────>│ │ │ + │ │ │ │ + │ │ 1. Get manifest │ │ + │ │ and layers │ │ + │ │ │ │ + │ │ 2. Check which │ │ + │ │ layers still │ │ + │ │ referenced by │ │ + │ │ user's other │ │ + │ │ manifests │ │ + │ │ │ │ + │ │ 3. DELETE manifest │ │ + │ │ from PDS │ │ + │ ├──────────────────────┼─────────────────────>│ + │ │ │ │ + │ │ 4. POST /quota/decrement │ + │ ├─────────────────────>│ │ + │ │ {layers: [...]} │ │ + │ │ │ │ + │ │ │ 5. Update quota │ + │ │ │ Remove unclaimed │ + │ │ │ layers │ + │ │ │ │ + │ │ 6. 200 OK │ │ + │ │<─────────────────────┤ │ + │ │ │ │ + │ │ 7. Delete from DB │ │ + │ │ │ │ + │ 8. Success │ │ │ + │<─────────────────────┤ │ │ + │ │ │ │ +``` + +### AppView Implementation + +```go +// pkg/appview/handlers/manifest.go + +func (h *ManifestHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { + did := r.Context().Value("auth.did").(string) + repository := chi.URLParam(r, "repository") + digest := chi.URLParam(r, "digest") + + // Step 1: Get manifest and its layers from database + manifest, err := db.GetManifest(h.db, digest) + if err != nil { + http.Error(w, "manifest not found", 404) + return + } + + layers, err := db.GetLayersForManifest(h.db, manifest.ID) + if err != nil { + http.Error(w, "failed to get layers", 500) + return + } + + // Step 2: For each layer, check if user still references it + // in other manifests + layersToDecrement := []LayerInfo{} + + for _, layer := range layers { + // Query: does this user have other manifests using this layer? + stillReferenced, err := db.CheckLayerReferencedByUser( + h.db, did, repository, layer.Digest, manifest.ID, + ) + + if err != nil { + http.Error(w, "failed to check layer references", 500) + return + } + + if !stillReferenced { + // This layer is no longer used by user + layersToDecrement = append(layersToDecrement, LayerInfo{ + Digest: layer.Digest, + Size: layer.Size, + }) + } + } + + // Step 3: Delete manifest from user's PDS + atprotoClient := atproto.NewClient(manifest.PDSEndpoint, did, accessToken) + err = atprotoClient.DeleteRecord(ctx, atproto.ManifestCollection, manifestRKey) + if err != nil { + http.Error(w, "failed to delete from PDS", 500) + return + } + + // Step 4: Notify hold service to decrement quota + if len(layersToDecrement) > 0 { + holdClient := &http.Client{} + + decrementReq := QuotaDecrementRequest{ + DID: did, + Layers: layersToDecrement, + } + + body, _ := json.Marshal(decrementReq) + resp, err := holdClient.Post( + manifest.HoldEndpoint + "/quota/decrement", + "application/json", + bytes.NewReader(body), + ) + + if err != nil || resp.StatusCode != 200 { + log.Printf("Warning: failed to update quota on hold service: %v", err) + // Continue anyway - GC reconciliation will fix it + } + } + + // Step 5: Delete from AppView database + err = db.DeleteManifest(h.db, did, repository, digest) + if err != nil { + http.Error(w, "failed to delete from database", 500) + return + } + + w.WriteHeader(http.StatusNoContent) +} +``` + +### Hold Service Decrement Endpoint + +```go +// cmd/hold/main.go + +type QuotaDecrementRequest struct { + DID string `json:"did"` + Layers []LayerInfo `json:"layers"` +} + +type LayerInfo struct { + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +func (s *HoldService) HandleQuotaDecrement(w http.ResponseWriter, r *http.Request) { + var req QuotaDecrementRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid request", 400) + return + } + + // Read current quota + quota, err := s.quotaManager.GetQuota(req.DID) + if err != nil { + http.Error(w, "quota not found", 404) + return + } + + // Decrement quota for each layer + for _, layer := range req.Layers { + if size, claimed := quota.ClaimedLayers[layer.Digest]; claimed { + // Remove from claimed layers + delete(quota.ClaimedLayers, layer.Digest) + quota.Used -= size + + log.Printf("Decremented quota for %s: layer %s (%d bytes)", + req.DID, layer.Digest, size) + } else { + log.Printf("Warning: layer %s not in claimed_layers for %s", + layer.Digest, req.DID) + } + } + + // Ensure quota.Used doesn't go negative (defensive) + if quota.Used < 0 { + log.Printf("Warning: quota.Used went negative for %s, resetting to 0", req.DID) + quota.Used = 0 + } + + // Save updated quota + quota.LastUpdated = time.Now() + if err := s.quotaManager.SaveQuota(quota); err != nil { + http.Error(w, "failed to save quota", 500) + return + } + + // Return updated quota info + json.NewEncoder(w).Encode(map[string]any{ + "used": quota.Used, + "limit": quota.Limit, + }) +} +``` + +### SQL Query: Check Layer References + +```sql +-- pkg/appview/db/queries.go + +-- Check if user still references this layer in other manifests +SELECT COUNT(*) +FROM layers l +JOIN manifests m ON l.manifest_id = m.id +WHERE m.did = ? -- User's DID + AND l.digest = ? -- Layer digest + AND m.id != ? -- Exclude the manifest being deleted +``` + +## Garbage Collection + +### Background: Orphaned Blobs + +Orphaned blobs accumulate when: +1. Manifest push fails after blobs uploaded (presigned URLs bypass hold) +2. Quota exceeded - manifest rejected, blobs already in S3 +3. User deletes manifest - blobs no longer referenced + +**GC periodically cleans these up.** + +### GC Cron Implementation + +Similar to AppView's backfill worker, the hold service can run periodic GC: + +```go +// cmd/hold/gc/gc.go + +type GarbageCollector struct { + driver storagedriver.StorageDriver + appviewURL string + holdURL string + quotaManager *quota.Manager +} + +// Run garbage collection +func (gc *GarbageCollector) Run(ctx context.Context) error { + log.Println("Starting garbage collection...") + + // Step 1: Get list of referenced blobs from AppView + referenced, err := gc.getReferencedBlobs() + if err != nil { + return fmt.Errorf("failed to get referenced blobs: %w", err) + } + + referencedSet := make(map[string]bool) + for _, digest := range referenced { + referencedSet[digest] = true + } + + log.Printf("AppView reports %d referenced blobs", len(referenced)) + + // Step 2: Walk S3 blobs + deletedCount := 0 + reclaimedBytes := int64(0) + + err = gc.driver.Walk(ctx, "/docker/registry/v2/blobs", func(fileInfo storagedriver.FileInfo) error { + if fileInfo.IsDir() { + return nil // Skip directories + } + + // Extract digest from path + // Path: /docker/registry/v2/blobs/sha256/ab/abc123.../data + digest := extractDigestFromPath(fileInfo.Path()) + + if !referencedSet[digest] { + // Unreferenced blob - delete it + size := fileInfo.Size() + + if err := gc.driver.Delete(ctx, fileInfo.Path()); err != nil { + log.Printf("Failed to delete blob %s: %v", digest, err) + return nil // Continue anyway + } + + deletedCount++ + reclaimedBytes += size + + log.Printf("GC: Deleted unreferenced blob %s (%d bytes)", digest, size) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to walk blobs: %w", err) + } + + log.Printf("GC complete: deleted %d blobs, reclaimed %d bytes", + deletedCount, reclaimedBytes) + + return nil +} + +// Get referenced blobs from AppView +func (gc *GarbageCollector) getReferencedBlobs() ([]string, error) { + // Query AppView for all blobs referenced by manifests + // stored in THIS hold service + url := fmt.Sprintf("%s/internal/blobs/referenced?hold=%s", + gc.appviewURL, url.QueryEscape(gc.holdURL)) + + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Blobs []string `json:"blobs"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + return result.Blobs, nil +} +``` + +### AppView Internal API + +```go +// pkg/appview/handlers/internal.go + +// Get all referenced blobs for a specific hold +func (h *InternalHandler) GetReferencedBlobs(w http.ResponseWriter, r *http.Request) { + holdEndpoint := r.URL.Query().Get("hold") + if holdEndpoint == "" { + http.Error(w, "missing hold parameter", 400) + return + } + + // Query database for all layers in manifests stored in this hold + query := ` + SELECT DISTINCT l.digest + FROM layers l + JOIN manifests m ON l.manifest_id = m.id + WHERE m.hold_endpoint = ? + ` + + rows, err := h.db.Query(query, holdEndpoint) + if err != nil { + http.Error(w, "database error", 500) + return + } + defer rows.Close() + + blobs := []string{} + for rows.Next() { + var digest string + if err := rows.Scan(&digest); err != nil { + continue + } + blobs = append(blobs, digest) + } + + json.NewEncoder(w).Encode(map[string]any{ + "blobs": blobs, + "count": len(blobs), + "hold": holdEndpoint, + }) +} +``` + +### GC Cron Schedule + +```go +// cmd/hold/main.go + +func main() { + // ... service setup ... + + // Start GC cron if enabled + if os.Getenv("GC_ENABLED") == "true" { + gcInterval := 24 * time.Hour // Daily by default + + go func() { + ticker := time.NewTicker(gcInterval) + defer ticker.Stop() + + for range ticker.C { + if err := garbageCollector.Run(context.Background()); err != nil { + log.Printf("GC error: %v", err) + } + } + }() + + log.Printf("GC cron started: runs every %v", gcInterval) + } + + // Start server... +} +``` + +## Quota Reconciliation + +### PDS as Source of Truth + +**Key insight:** Manifest records in PDS are publicly readable (no OAuth needed for reads). + +Each manifest contains: +- Repository name +- Digest +- Layers array with digest + size +- Hold endpoint + +The hold service can query the PDS to calculate the user's true quota: + +``` +1. List all io.atcr.manifest records for user +2. Filter manifests where holdEndpoint == this hold service +3. Extract unique layers (deduplicate by digest) +4. Sum layer sizes = true quota usage +5. Compare to quota file +6. Fix discrepancies +``` + +### Implementation + +```go +// cmd/hold/quota/reconcile.go + +type Reconciler struct { + quotaManager *Manager + atprotoResolver *atproto.Resolver + holdURL string +} + +// ReconcileUser recalculates quota from PDS manifests +func (r *Reconciler) ReconcileUser(ctx context.Context, did string) error { + log.Printf("Reconciling quota for %s", did) + + // Step 1: Resolve user's PDS endpoint + identity, err := r.atprotoResolver.ResolveIdentity(ctx, did) + if err != nil { + return fmt.Errorf("failed to resolve DID: %w", err) + } + + // Step 2: Create unauthenticated ATProto client + // (manifest records are public - no OAuth needed) + client := atproto.NewClient(identity.PDSEndpoint, did, "") + + // Step 3: List all manifest records for this user + manifests, err := client.ListRecords(ctx, atproto.ManifestCollection, 1000) + if err != nil { + return fmt.Errorf("failed to list manifests: %w", err) + } + + // Step 4: Filter manifests stored in THIS hold service + // and extract unique layers + uniqueLayers := make(map[string]int64) // digest -> size + + for _, record := range manifests { + var manifest atproto.ManifestRecord + if err := json.Unmarshal(record.Value, &manifest); err != nil { + log.Printf("Warning: failed to parse manifest: %v", err) + continue + } + + // Only count manifests stored in this hold + if manifest.HoldEndpoint != r.holdURL { + continue + } + + // Add config blob + if manifest.Config.Digest != "" { + uniqueLayers[manifest.Config.Digest] = manifest.Config.Size + } + + // Add layer blobs + for _, layer := range manifest.Layers { + uniqueLayers[layer.Digest] = layer.Size + } + } + + // Step 5: Calculate true quota usage + trueUsage := int64(0) + for _, size := range uniqueLayers { + trueUsage += size + } + + log.Printf("User %s true usage from PDS: %d bytes (%d unique layers)", + did, trueUsage, len(uniqueLayers)) + + // Step 6: Compare with current quota file + quota, err := r.quotaManager.GetQuota(did) + if err != nil { + log.Printf("No existing quota for %s, creating new", did) + quota = &Quota{ + DID: did, + Limit: r.quotaManager.DefaultLimit, + ClaimedLayers: make(map[string]int64), + } + } + + // Step 7: Fix discrepancies + if quota.Used != trueUsage || len(quota.ClaimedLayers) != len(uniqueLayers) { + log.Printf("Quota mismatch for %s: recorded=%d, actual=%d (diff=%d)", + did, quota.Used, trueUsage, trueUsage - quota.Used) + + // Update quota to match PDS truth + quota.Used = trueUsage + quota.ClaimedLayers = uniqueLayers + quota.LastUpdated = time.Now() + + if err := r.quotaManager.SaveQuota(quota); err != nil { + return fmt.Errorf("failed to save reconciled quota: %w", err) + } + + log.Printf("Reconciled quota for %s: %d bytes", did, trueUsage) + } else { + log.Printf("Quota for %s is accurate", did) + } + + return nil +} + +// ReconcileAll reconciles all users (run periodically) +func (r *Reconciler) ReconcileAll(ctx context.Context) error { + // Get list of all users with quota files + users, err := r.quotaManager.ListUsers() + if err != nil { + return err + } + + log.Printf("Starting reconciliation for %d users", len(users)) + + for _, did := range users { + if err := r.ReconcileUser(ctx, did); err != nil { + log.Printf("Failed to reconcile %s: %v", did, err) + // Continue with other users + } + } + + log.Println("Reconciliation complete") + return nil +} +``` + +### Reconciliation Cron + +```go +// cmd/hold/main.go + +func main() { + // ... setup ... + + // Start reconciliation cron + if os.Getenv("QUOTA_RECONCILE_ENABLED") == "true" { + reconcileInterval := 24 * time.Hour // Daily + + go func() { + ticker := time.NewTicker(reconcileInterval) + defer ticker.Stop() + + for range ticker.C { + if err := reconciler.ReconcileAll(context.Background()); err != nil { + log.Printf("Reconciliation error: %v", err) + } + } + }() + + log.Printf("Quota reconciliation cron started: runs every %v", reconcileInterval) + } + + // ... start server ... +} +``` + +### Why PDS as Source of Truth Works + +1. **Manifests are canonical** - If manifest exists in PDS, user owns those layers +2. **Public reads** - No OAuth needed, just resolve DID → PDS endpoint +3. **ATProto durability** - PDS is user's authoritative data store +4. **AppView is cache** - AppView database might lag or have inconsistencies +5. **Reconciliation fixes drift** - Periodic sync from PDS ensures accuracy + +**Example reconciliation scenarios:** + +- **Orphaned quota entries:** User deleted manifest from PDS, but hold quota still has it + → Reconciliation removes from claimed_layers + +- **Missing quota entries:** User pushed manifest, but quota update failed + → Reconciliation adds to claimed_layers + +- **Race condition duplicates:** Two concurrent pushes double-counted a layer + → Reconciliation fixes to actual usage + +## Configuration + +### Hold Service Environment Variables + +```bash +# .env.hold + +# ============================================================================ +# Quota Configuration +# ============================================================================ + +# Enable quota enforcement +QUOTA_ENABLED=true + +# Default quota limit per user (bytes) +# 10GB = 10737418240 +# 50GB = 53687091200 +# 100GB = 107374182400 +QUOTA_DEFAULT_LIMIT=10737418240 + +# Storage backend for quota data +# Options: s3, sqlite +QUOTA_STORAGE_BACKEND=s3 + +# For S3-based storage: +# Quota files stored in same bucket as blobs +QUOTA_STORAGE_PREFIX=/atcr/quota/ + +# For SQLite-based storage: +QUOTA_DB_PATH=/var/lib/atcr/hold-quota.db + +# ============================================================================ +# Garbage Collection +# ============================================================================ + +# Enable periodic garbage collection +GC_ENABLED=true + +# GC interval (default: 24h) +GC_INTERVAL=24h + +# AppView URL for GC reference checking +APPVIEW_URL=https://atcr.io + +# ============================================================================ +# Quota Reconciliation +# ============================================================================ + +# Enable quota reconciliation from PDS +QUOTA_RECONCILE_ENABLED=true + +# Reconciliation interval (default: 24h) +QUOTA_RECONCILE_INTERVAL=24h + +# ============================================================================ +# Hold Service Identity (Required) +# ============================================================================ + +# Public URL of this hold service +HOLD_PUBLIC_URL=https://hold1.example.com + +# Owner DID (for auto-registration) +HOLD_OWNER=did:plc:xyz123 +``` + +### AppView Configuration + +```bash +# .env.appview + +# Internal API endpoint for hold services +# Used for GC reference checking +ATCR_INTERNAL_API_ENABLED=true + +# Optional: authentication token for internal APIs +ATCR_INTERNAL_API_TOKEN=secret123 +``` + +## Trade-offs & Design Decisions + +### 1. Claimed Storage vs Physical Storage + +**Decision:** Track claimed storage (logical accounting) + +**Why:** +- Predictable for users: "you pay for what you upload" +- No complex cross-user dependencies +- Delete always gives you quota back +- Matches Harbor's proven model + +**Trade-off:** +- Total claimed can exceed physical storage +- Users might complain "I uploaded 10GB but S3 only has 6GB" + +**Mitigation:** +- Show deduplication savings metric +- Educate users: "You claimed 10GB, but deduplication saved 4GB" + +### 2. S3 vs SQLite for Quota Storage + +**Decision:** Support both, recommend based on use case + +**S3 Pros:** +- No database to manage +- Quota data lives with blobs +- Better for ephemeral BYOS + +**SQLite Pros:** +- Faster (no network) +- ACID transactions (no race conditions) +- Better for high-traffic shared holds + +**Trade-off:** +- S3: eventual consistency, race conditions +- SQLite: stateful service, scaling challenges + +**Mitigation:** +- Reconciliation fixes S3 inconsistencies +- SQLite can use shared DB for multi-instance + +### 3. Optimistic Quota Update + +**Decision:** Update quota BEFORE upload completes + +**Why:** +- Prevent race conditions (two users uploading simultaneously) +- Can reject before presigned URL generated +- Simpler flow + +**Trade-off:** +- If upload fails, quota already incremented (user "paid" for nothing) + +**Mitigation:** +- Reconciliation from PDS fixes orphaned quota entries +- Acceptable for MVP (upload failures are rare) + +### 4. AppView as Intermediary + +**Decision:** AppView notifies hold service on deletes + +**Why:** +- AppView already has manifest/layer database +- Can efficiently check if layer still referenced +- Hold service doesn't need to query PDS on every delete + +**Trade-off:** +- AppView → Hold dependency +- Network hop on delete + +**Mitigation:** +- If notification fails, reconciliation fixes quota +- Eventually consistent is acceptable + +### 5. PDS as Source of Truth + +**Decision:** Use PDS manifests for reconciliation + +**Why:** +- Manifests in PDS are canonical user data +- Public reads (no OAuth for reconciliation) +- AppView database might lag or be inconsistent + +**Trade-off:** +- Reconciliation requires PDS queries (slower) +- Limited to 1000 manifests per query + +**Mitigation:** +- Run reconciliation daily (not real-time) +- Paginate if user has >1000 manifests + +## Future Enhancements + +### 1. Quota API Endpoints + +``` +GET /quota/usage - Get current user's quota +GET /quota/breakdown - Get storage by repository +POST /quota/limit - Update user's quota limit (admin) +GET /quota/stats - Get hold-wide statistics +``` + +### 2. Quota Alerts + +Notify users when approaching limit: +- Email/webhook at 80%, 90%, 95% +- Reject uploads at 100% (currently implemented) +- Grace period: allow 105% temporarily + +### 3. Tiered Quotas + +Different limits based on user tier: +- Free: 10GB +- Pro: 100GB +- Enterprise: unlimited + +### 4. Quota Purchasing + +Allow users to buy additional storage: +- Stripe integration +- $0.10/GB/month pricing +- Dynamic limit updates + +### 5. Cross-Hold Deduplication + +If multiple holds share same S3 bucket: +- Track blob ownership globally +- Split costs proportionally +- More complex, but maximizes deduplication + +### 6. Manifest-Based Quota (Alternative Model) + +Instead of tracking layers, track manifests: +- Simpler: just count manifest sizes +- No deduplication benefits for users +- Might be acceptable for some use cases + +### 7. Redis-Based Quota (High Performance) + +For high-traffic registries: +- Use Redis instead of S3/SQLite +- Sub-millisecond quota checks +- Harbor-proven approach + +### 8. Quota Visualizations + +Web UI showing: +- Storage usage over time +- Top consumers by repository +- Deduplication savings graph +- Layer size distribution + +## Appendix: SQL Queries + +### Check if User Still References Layer + +```sql +-- After deleting manifest, check if user has other manifests using this layer +SELECT COUNT(*) +FROM layers l +JOIN manifests m ON l.manifest_id = m.id +WHERE m.did = ? -- User's DID + AND l.digest = ? -- Layer digest to check + AND m.id != ? -- Exclude the manifest being deleted +``` + +### Get All Unique Layers for User + +```sql +-- Calculate true quota usage for a user +SELECT DISTINCT l.digest, l.size +FROM layers l +JOIN manifests m ON l.manifest_id = m.id +WHERE m.did = ? + AND m.hold_endpoint = ? +``` + +### Get Referenced Blobs for Hold + +```sql +-- For GC: get all blobs still referenced by any user of this hold +SELECT DISTINCT l.digest +FROM layers l +JOIN manifests m ON l.manifest_id = m.id +WHERE m.hold_endpoint = ? +``` + +### Get Storage Stats by Repository + +```sql +-- User's storage broken down by repository +SELECT + m.repository, + COUNT(DISTINCT m.id) as manifest_count, + COUNT(DISTINCT l.digest) as unique_layers, + SUM(l.size) as total_size +FROM manifests m +JOIN layers l ON l.manifest_id = m.id +WHERE m.did = ? + AND m.hold_endpoint = ? +GROUP BY m.repository +ORDER BY total_size DESC +``` + +## References + +- **Harbor Quotas:** https://goharbor.io/docs/1.10/administration/configure-project-quotas/ +- **Harbor Source:** https://github.com/goharbor/harbor +- **ATProto Spec:** https://atproto.com/specs/record +- **OCI Distribution Spec:** https://github.com/opencontainers/distribution-spec +- **S3 API Reference:** https://docs.aws.amazon.com/AmazonS3/latest/API/ +- **Distribution GC:** https://github.com/distribution/distribution/blob/main/registry/storage/garbagecollect.go + +--- + +**Document Version:** 1.0 +**Last Updated:** 2025-10-09 +**Author:** Generated from implementation research and Harbor analysis diff --git a/docs/SPEC.md b/docs/SPEC.md deleted file mode 100644 index b24eeb9..0000000 --- a/docs/SPEC.md +++ /dev/null @@ -1,460 +0,0 @@ -ATProto Container Registry (atcr.io) Implementation Plan - - Project Structure - - /home/data/atcr.io/ - ├── cmd/ - │ └── registry/ - │ └── main.go # Entrypoint that imports distribution - ├── pkg/ - │ ├── atproto/ - │ │ ├── client.go # ATProto client wrapper (using indigo) - │ │ ├── manifest_store.go # Implements distribution.ManifestService - │ │ ├── resolver.go # DID/handle resolution (alice → did:plc:...) - │ │ └── lexicon.go # ATProto record schemas for manifests - │ ├── storage/ - │ │ ├── s3_blob_store.go # Wraps distribution's S3 driver for blobs - │ │ └── routing_repository.go # Routes manifests→ATProto, blobs→S3 - │ ├── middleware/ - │ │ ├── repository.go # Repository middleware registration - │ │ └── registry.go # Registry middleware for name resolution - │ └── server/ - │ └── handler.go # HTTP wrapper for custom name resolution - ├── config/ - │ └── config.yml # Registry configuration - ├── go.mod - ├── go.sum - ├── Dockerfile - ├── README.md - └── CLAUDE.md # Updated with architecture docs - - - Implementation Steps - - Phase 1: Project Setup - - 1. Initialize Go module with github.com/distribution/distribution/v3 and github.com/bluesky-social/indigo - 2. Create basic project structure - 3. Set up cmd/appview/main.go that imports distribution and registers middleware - - Phase 2: Core ATProto Integration - - 4. Implement DID/handle resolver (pkg/atproto/resolver.go) - - Resolve handles to DIDs (alice.bsky.social → did:plc:xyz) - - Discover PDS endpoints from DID documents - 5. Create ATProto client wrapper (pkg/atproto/client.go) - - Wrap indigo SDK for manifest storage - - Handle authentication with PDS - 6. Design ATProto lexicon for manifest records (pkg/atproto/lexicon.go) - - Define schema for storing OCI manifests as ATProto records - - Phase 3: Storage Layer - - 7. Implement ATProto manifest store (pkg/atproto/manifest_store.go) - - Implements distribution.ManifestService - - Stores/retrieves manifests from PDS - 8. Implement S3 blob store wrapper (pkg/storage/s3_blob_store.go) - - Wraps distribution's built-in S3 driver - 9. Create routing repository (pkg/storage/routing_repository.go) - - Returns ATProto store for Manifests() - - Returns S3 store for Blobs() - - Phase 4: Middleware Layer - - 10. Implement repository middleware (pkg/middleware/repository.go) - - Registers routing repository - - Configurable via YAML - 11. Implement registry/namespace middleware (pkg/middleware/registry.go) - - Intercepts Repository(name) calls - - Performs name resolution before repository creation - - Phase 5: HTTP Layer (if needed) - - 12. Create custom HTTP handler (pkg/server/handler.go) - - Wraps distribution's HTTP handlers - - Performs early name resolution: atcr.io/alice/myimage → resolve alice - - Delegates to distribution handlers - - Phase 6: Configuration & Deployment - - 13. Create registry configuration (config/config.yml) - 14. Create Dockerfile for building atcr-appview binary - 16. Write README.md with usage instructions - - Phase 7: Documentation - - 17. Update CLAUDE.md with: - - Architecture overview (ATProto for manifests, S3 for blobs) - - Build/run/test commands - - How name resolution works - - Middleware registration flow - - Key design decisions - - Key Technical Decisions - - ATProto Storage Design: - - Manifests stored as ATProto records in user's PDS - - Each image manifest is a record in a io.atcr.manifest collection - - Record key = image digest (sha256:...) - - Name Resolution: - - atcr.io/alice/myimage → resolve alice to DID → discover PDS - - Support both handles (alice.bsky.social) and DIDs (did:plc:xyz) - - Blob Storage: - - All layers/blobs in S3 (content-addressable by digest) - - Manifests reference S3-stored blobs by digest - - S3 provides cheap, durable blob storage - - Middleware Strategy: - - Repository middleware for storage routing - - Registry middleware (or HTTP wrapper) for name resolution - - No fork of distribution core needed - -Perfect. To match Docker Hub/ghcr.io/gcr.io, here's what we need: - -● Implementation Plan (Drop-in replacement for Docker Hub/ghcr.io) - - Flow 1: Standard Token Auth (Like Docker Hub) - PRIMARY - - # User experience - docker login atcr.io -u alice.bsky.social -p - docker push atcr.io/alice/myapp:latest - - # Behind the scenes - 1. docker login stores credentials locally - 2. docker push → Registry returns 401 with WWW-Authenticate: Bearer realm="https://atcr.io/auth/token"... - 3. Docker auto-calls /auth/token with Basic auth (alice.bsky.social:app-password) - 4. Auth service validates against ATProto createSession - 5. Returns JWT token with scope for alice/myapp - 6. Docker uses JWT for manifest/blob uploads - 7. Registry validates JWT signature and scope - - Components: - - /auth/token endpoint (standalone service or embedded) - - ATProto session validator (username/password → validate via PDS) - - JWT issuer/signer - - JWT validator middleware for registry - - Flow 2: Credential Helper (Like gcr.io) - ADVANCED - - # User experience - docker-credential-atcr configure - # Opens browser for ATProto OAuth - docker push atcr.io/alice/myapp:latest - # No manual login needed - - # Behind the scenes - 1. Helper does OAuth flow → gets ATProto access token - 2. Caches token securely - 3. When Docker needs credentials, calls helper via stdin/stdout - 4. Helper exchanges ATProto token for registry JWT at /auth/exchange - 5. Returns JWT to Docker - 6. Docker uses JWT for requests - - Components: - - cmd/credential-helper/main.go - Standalone binary - - ATProto OAuth client - - Token exchange endpoint (/auth/exchange) - - Secure token cache - - Architecture: - - pkg/auth/ - ├── token/ - │ ├── service.go # HTTP handler for /auth/token - │ ├── claims.go # JWT claims structure - │ ├── issuer.go # Signs JWTs - │ └── validator.go # Validates JWTs (middleware for registry) - ├── atproto/ - │ ├── session.go # Validates username/password via ATProto - │ └── oauth.go # OAuth flow implementation - ├── exchange/ - │ └── handler.go # /auth/exchange endpoint (OAuth → JWT) - └── scope.go # Parses/validates Docker scopes - - cmd/ - ├── registry/main.go # Registry server (existing) - ├── auth/main.go # Standalone auth service (optional) - └── credential-helper/ - └── main.go # docker-credential-atcr binary - - Config: - - auth: - token: - realm: https://atcr.io/auth/token # Where Docker gets tokens - service: atcr.io - issuer: atcr.io - rootcertbundle: /etc/atcr/token-signing.crt - privatekey: /etc/atcr/token-signing.pem - expiration: 300 - - atproto: - # Used by auth service to validate credentials - pds_endpoint: https://bsky.social - client_id: atcr-appview - oauth_redirect: http://localhost:8888/callback - -ATProto OAuth Implementation Plan - - Architecture - - Dependencies: - - authelia.com/client/oauth2 - OAuth + PAR support - - github.com/AxisCommunications/go-dpop - DPoP proof generation (handles JWK automatically) - - github.com/golang-jwt/jwt/v5 - JWT library (transitive via go-dpop) - - Our existing pkg/atproto/resolver.go - ATProto identity resolution - - Implementation Components - - 1. OAuth Client (pkg/auth/oauth/client.go) - ~100 lines - - type Client struct { - config *oauth2.Config - dpopKey *ecdsa.PrivateKey - resolver *atproto.Resolver - clientID string // URL to our metadata document - redirectURI string - dpopNonce string // Server-provided nonce - } - - func NewClient(clientID, redirectURI string) (*Client, error) - func (c *Client) AuthorizeURL(handle string, scopes []string) (string, error) - func (c *Client) Exchange(code string) (*Token, error) - func (c *Client) addDPoPHeader(req *http.Request, method, url string) error - - Flow: - 1. Generate ECDSA P-256 key for DPoP - 2. Discover authorization server from handle/DID - 3. Use authelia's PushedAuth() for PAR with DPoP header - 4. Exchange code for token with DPoP proof - - 2. Authorization Server Discovery (pkg/auth/oauth/discovery.go) - ~30 lines - - type AuthServerMetadata struct { - Issuer string `json:"issuer"` - AuthorizationEndpoint string `json:"authorization_endpoint"` - TokenEndpoint string `json:"token_endpoint"` - PushedAuthorizationRequestEndpoint string `json:"pushed_authorization_request_endpoint"` - DPoPSigningAlgValuesSupported []string `json:"dpop_signing_alg_values_supported"` - } - - func DiscoverAuthServer(pdsEndpoint string) (*AuthServerMetadata, error) - - Implementation: - - GET {pds}/.well-known/oauth-authorization-server - - Parse JSON metadata - - Validate required endpoints exist - - 3. Client Metadata Server (pkg/auth/oauth/metadata.go) - ~40 lines - - type ClientMetadata struct { - ClientID string `json:"client_id"` - RedirectURIs []string `json:"redirect_uris"` - GrantTypes []string `json:"grant_types"` - ResponseTypes []string `json:"response_types"` - Scope string `json:"scope"` - DPoPBoundAccessTokens bool `json:"dpop_bound_access_tokens"` - } - - func ServeMetadata(clientID string, redirectURIs []string) http.Handler - - Serves: https://atcr.io/oauth/client-metadata.json - - 4. Token Storage (pkg/auth/oauth/storage.go) - ~50 lines - - type TokenStore struct { - AccessToken string - RefreshToken string - DPoPKey *ecdsa.PrivateKey // Persist for refresh - ExpiresAt time.Time - } - - func (s *TokenStore) Save(path string) error - func LoadTokenStore(path string) (*TokenStore, error) - - Storage location: ~/.atcr/oauth-tokens.json - - 5. Credential Helper (cmd/credential-helper/main.go) - ~80 lines - - // Docker credential helper protocol - // Reads JSON from stdin, writes to stdout - - func main() { - if len(os.Args) < 2 { - os.Exit(1) - } - - switch os.Args[1] { - case "get": - handleGet() // Return credentials for registry - case "store": - handleStore() // Store credentials - case "erase": - handleErase() // Remove credentials - } - } - - func handleGet() { - var request struct { - ServerURL string `json:"ServerURL"` - } - json.NewDecoder(os.Stdin).Decode(&request) - - // Load token from storage - // Exchange for registry JWT if needed - // Output: {"Username": "oauth2", "Secret": ""} - } - - 6. OAuth Flow (cmd/credential-helper/oauth.go) - ~60 lines - - func RunOAuthFlow(handle string) (*TokenStore, error) { - // 1. Start local HTTP server on :8888 - // 2. Open browser to authorization URL - // 3. Wait for callback with code - // 4. Exchange code for token - // 5. Save token store - // 6. Return token - } - - func startCallbackServer() (chan string, *http.Server) - - Complete Flow Example - - User runs: - docker-credential-atcr configure - - What happens: - - 1. Generate DPoP key (client.go) - dpopKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - - 2. Resolve handle → DID → PDS (using our resolver) - did, pds, _ := resolver.ResolveIdentity(ctx, "alice.bsky.social") - - 3. Discover auth server (discovery.go) - metadata, _ := DiscoverAuthServer(pds) - // Returns: PAR endpoint, token endpoint, etc. - - 4. Create PAR request with DPoP (client.go + go-dpop) - // Generate DPoP proof for PAR endpoint - claims := &dpop.ProofTokenClaims{ - Method: dpop.POST, - URL: metadata.PushedAuthorizationRequestEndpoint, - RegisteredClaims: &jwt.RegisteredClaims{ - IssuedAt: jwt.NewNumericDate(time.Now()), - }, - } - dpopProof, _ := dpop.Create(jwt.SigningMethodES256, claims, dpopKey) - - // Use authelia for PAR - config := &oauth2.Config{ - ClientID: "https://atcr.io/oauth/client-metadata.json", - Endpoint: oauth2.Endpoint{ - AuthURL: metadata.AuthorizationEndpoint, - TokenURL: metadata.TokenEndpoint, - }, - } - - // Create custom HTTP client that adds DPoP header - client := &http.Client{ - Transport: &dpopTransport{ - base: http.DefaultTransport, - dpopKey: dpopKey, - }, - } - ctx := context.WithValue(context.Background(), oauth2.HTTPClient, client) - - // PAR request (authelia handles this) - authURL, parResp, _ := config.PushedAuth(ctx, state, - oauth2.SetAuthURLParam("code_challenge", pkceChallenge), - oauth2.SetAuthURLParam("code_challenge_method", "S256"), - ) - - 5. Open browser, get code (oauth.go) - exec.Command("open", authURL).Run() - // User authorizes - // Callback: http://localhost:8888?code=xyz&state=abc - - 6. Exchange code for token with DPoP (client.go + go-dpop) - // Generate DPoP proof for token endpoint - claims := &dpop.ProofTokenClaims{ - Method: dpop.POST, - URL: metadata.TokenEndpoint, - RegisteredClaims: &jwt.RegisteredClaims{ - IssuedAt: jwt.NewNumericDate(time.Now()), - }, - } - dpopProof, _ := dpop.Create(jwt.SigningMethodES256, claims, dpopKey) - - // Exchange (with DPoP header added by our transport) - token, _ := config.Exchange(ctx, code, - oauth2.SetAuthURLParam("code_verifier", pkceVerifier), - ) - - 7. Save token + DPoP key (storage.go) - store := &TokenStore{ - AccessToken: token.AccessToken, - RefreshToken: token.RefreshToken, - DPoPKey: dpopKey, - ExpiresAt: token.Expiry, - } - store.Save("~/.atcr/oauth-tokens.json") - - Later, when docker push happens: - docker push atcr.io/alice/myapp:latest - - 1. Docker calls credential helper: docker-credential-atcr get - 2. Helper loads stored token - 3. Helper calls /auth/exchange with OAuth token → gets registry JWT - 4. Returns JWT to Docker - 5. Docker uses JWT for push - - Directory Structure - - pkg/auth/oauth/ - ├── client.go # OAuth client with DPoP integration - ├── discovery.go # Authorization server discovery - ├── metadata.go # Client metadata server - ├── storage.go # Token persistence - └── transport.go # HTTP transport that adds DPoP headers - - cmd/credential-helper/ - ├── main.go # Docker credential helper protocol - ├── oauth.go # OAuth flow (browser, callback) - └── config.go # Configuration - - go.mod additions: - authelia.com/client/oauth2 v0.25.0 - github.com/AxisCommunications/go-dpop v1.1.2 - -Unified Model - - Every hold service requires HOLD_OWNER: - - Owner's PDS has the io.atcr.hold record - - Owner's PDS has all io.atcr.hold.crew records - - Authorization is always governed by PDS records - - For "public" hold (like Tangled's public knot): - - Owner creates hold with public: true - - Anyone can push/pull without being crew - - Owner can add crew records for special privileges/tracking if desired - - Config has emergency override: - auth: - # Emergency freeze: ignore public setting, restrict to crew only - # Use this to stop abuse without changing PDS records - freeze: false - - Authorization logic: - 1. Check freeze in config → if true, skip to crew check - 2. Query owner's PDS for io.atcr.hold record - 3. If public: true → allow all operations (unless frozen) - 4. If public: false OR frozen → query io.atcr.hold.crew records, check membership - - Remove from config: - - allow_all (replaced by public: true in PDS) - - allowed_dids (replaced by crew records in PDS) - - This way the hold owner at atcr.io can run a public hold at hold1.atcr.io that anyone can use, but can freeze it instantly if needed without touching PDS records. diff --git a/docs/TESTING.md b/docs/TESTING.md deleted file mode 100644 index ee4ffa1..0000000 --- a/docs/TESTING.md +++ /dev/null @@ -1,334 +0,0 @@ -# Local Testing Guide - -## Quick Start - -```bash -./test-local.sh -``` - -This automated script will: -1. Create storage directories -2. Build all binaries -3. Start both services -4. Show test commands - -## Manual Testing Steps - -### 1. Setup Directories - -```bash -sudo mkdir -p /var/lib/atcr/{blobs,hold,auth} -sudo chown -R $USER:$USER /var/lib/atcr -``` - -### 2. Build Binaries - -```bash -go build -o atcr-appview ./cmd/appview -go build -o atcr-hold ./cmd/hold -go build -o docker-credential-atcr ./cmd/credential-helper -``` - -### 3. Configure Environment - -Create a `.env` file in the project root: - -```bash -cp .env.example .env -``` - -Edit `.env` with your credentials: - -```env -# Your ATProto handle -ATPROTO_HANDLE=your-handle.bsky.social - -# Hold service public URL (hostname becomes the hold name) -HOLD_PUBLIC_URL=http://127.0.0.1:8080 - -# Enable OAuth registration on startup -HOLD_AUTO_REGISTER=true -``` - -**Notes:** -- Use your Bluesky handle (e.g., `alice.bsky.social`) -- For localhost, use `127.0.0.1` instead of `localhost` for OAuth -- The hostname from the URL becomes the hold name (e.g., `127.0.0.1` or `hold1.atcr.io`) - -**Load environment:** -```bash -export $(cat .env | xargs) -``` - -### 4. Start Services - -**Terminal 1 - AppView:** -```bash -./atcr-appview serve config/config.yml -``` - -**Terminal 2 - Hold:** -```bash -./atcr-hold config/hold.yml -``` - -### 5. Start Services and OAuth Registration - -**Terminal 1 - AppView:** -```bash -./atcr-appview serve config/config.yml -``` - -**Terminal 2 - Hold (OAuth registration):** -```bash -./atcr-hold config/hold.yml -``` - -The hold service will start an OAuth flow. You'll see output like: - -``` -================================================================================ -OAUTH AUTHORIZATION REQUIRED -================================================================================ - -Please visit this URL to authorize the hold service: - - https://bsky.social/oauth/authorize?... - -Waiting for authorization... -================================================================================ -``` - -**Steps:** -1. Copy the OAuth URL from the logs -2. Open it in your browser -3. Sign in to Bluesky and authorize -4. The callback will complete automatically -5. Hold service registers in your PDS - -After successful OAuth, you'll see: -``` -✓ Created hold record: at://did:plc:.../io.atcr.hold/127.0.0.1 -✓ Created crew record: at://did:plc:.../io.atcr.hold.crew/127.0.0.1-did:plc:... -================================================================================ -REGISTRATION COMPLETE -================================================================================ -Hold service is now registered and ready to use! -``` - -This creates two records in your PDS: -- `io.atcr.hold` - Defines the storage endpoint URL -- `io.atcr.hold.crew` - Grants you admin access - -### 6. Test Docker Push/Pull - -**Test 1: Basic Push** -```bash -# Tag an image -docker tag alpine:latest localhost:5000/alice/alpine:test - -# Push to local registry -docker push localhost:5000/alice/alpine:test -``` - -**Test 2: Pull** -```bash -# Remove local image -docker rmi localhost:5000/alice/alpine:test - -# Pull from registry -docker pull localhost:5000/alice/alpine:test -``` - -**Test 3: Verify Storage** -```bash -# Check manifests were stored in ATProto -# (Check your PDS for io.atcr.manifest records) - -# Check blobs were stored locally -ls -lh /var/lib/atcr/blobs/docker/registry/v2/ -``` - -## OAuth Testing (Optional) - -### Setup Credential Helper - -```bash -# Configure OAuth -./docker-credential-atcr configure - -# Follow the browser flow to authorize - -# Verify token was saved -ls -la ~/.atcr/oauth-token.json -``` - -### Configure Docker to Use Helper - -Edit `~/.docker/config.json`: -```json -{ - "credHelpers": { - "localhost:5000": "atcr" - } -} -``` - -### Test with OAuth - -```bash -# Push should now use OAuth automatically -docker push localhost:5000/alice/myapp:latest -``` - -## Troubleshooting - -### Registry won't start - -**Error:** `failed to create storage driver` -```bash -# Check directory permissions -ls -ld /var/lib/atcr/blobs -# Should be owned by your user - -# Fix permissions -sudo chown -R $USER:$USER /var/lib/atcr -``` - -**Error:** `address already in use` -```bash -# Check what's using port 5000 -lsof -i :5000 - -# Kill existing process -kill $(lsof -t -i :5000) -``` - -### Hold service won't start - -**Error:** `failed to create storage driver` -```bash -# Check hold directory -ls -ld /var/lib/atcr/hold -sudo chown -R $USER:$USER /var/lib/atcr/hold -``` - -**Error:** `address already in use` -```bash -# Check port 8080 -lsof -i :8080 -kill $(lsof -t -i :8080) -``` - -### Docker push fails - -**Error:** `unauthorized: authentication required` -- Check `ATPROTO_DID` and `ATPROTO_ACCESS_TOKEN` are set -- Verify token is valid (not expired) -- Check registry logs for auth errors - -**Error:** `denied: requested access to the resource is denied` -- Check the identity in the image name matches your DID -- Example: If your handle is `alice.bsky.social`, use: - ```bash - docker push localhost:5000/alice/myapp:test - # NOT localhost:5000/bob/myapp:test - ``` - -**Error:** `failed to resolve identity` -- Check internet connection (needs to resolve DIDs) -- Verify handle is correct -- Try using DID directly instead of handle - -### OAuth issues - -**Error:** `Failed to exchange token` -- Ensure registry is running and accessible -- Check `/auth/exchange` endpoint is responding -- Verify OAuth token hasn't expired - -**Error:** `Token validation failed` -- Token might be expired -- Run `./docker-credential-atcr configure` again -- Check PDS is accessible - -## Verifying the Flow - -### Check Registry is Running -```bash -curl http://localhost:5000/v2/ -# Should return: {} -``` - -### Check Hold is Running -```bash -curl http://localhost:8080/health -# Should return: {"status":"ok"} -``` - -### Check Auth Endpoint -```bash -curl -v http://localhost:5000/v2/ -# Should return 401 with WWW-Authenticate header -``` - -### Inspect Stored Data - -**Manifests (in ATProto):** -- Check your PDS web interface -- Look for `io.atcr.manifest` collection records - -**Blobs (local filesystem):** -```bash -# List blobs -find /var/lib/atcr/blobs -type f - -# Check blob content (should be binary) -ls -lh /var/lib/atcr/blobs/docker/registry/v2/blobs/sha256/ -``` - -## Clean Up - -### Stop Services -```bash -# If using test script -kill $(cat .atcr-pids) - -# Or manually -pkill atcr-appview -pkill atcr-hold -``` - -### Remove Test Data -```bash -# Remove all stored data -sudo rm -rf /var/lib/atcr/* - -# Remove OAuth tokens -rm -rf ~/.atcr/ -``` - -### Reset Docker Config -```bash -# Remove credential helper config -# Edit ~/.docker/config.json and remove "credHelpers" section -``` - -## Next Steps - -Once local testing works: - -1. **Deploy to production:** - - Use S3/Storj for blob storage - - Deploy registry and hold to separate hosts - - Configure DNS for `atcr.io` - -2. **Enable BYOS:** - - Users create `io.atcr.hold` records - - Deploy their own hold service - - AppView automatically routes to their storage - -3. **Add monitoring:** - - Registry metrics - - Hold service metrics - - Storage usage tracking