diff --git a/CLAUDE.md b/CLAUDE.md index 8186652..248a7f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,7 @@ export STORAGE_DRIVER=filesystem export STORAGE_ROOT_DIR=/tmp/atcr-hold export HOLD_OWNER=did:plc:your-did-here ./bin/atcr-hold -# Check logs for OAuth URL, visit in browser to complete registration +# Hold starts immediately with embedded PDS ``` ## Architecture Overview @@ -75,9 +75,10 @@ ATCR uses **distribution/distribution** as a library and extends it through midd 2. **Hold Service** (`cmd/hold`) - Optional BYOS component - Lightweight HTTP server for presigned URLs + - Embedded PDS with captain + crew records - Supports S3, Storj, Minio, filesystem, etc. - - Authorization based on PDS records (hold.public, crew records) - - Auto-registration via OAuth + - Authorization based on captain record (public, allowAllCrew) + - Self-describing via DID resolution - Configured entirely via environment variables 3. **Credential Helper** (`cmd/credential-helper`) - Client-side OAuth @@ -357,7 +358,7 @@ Read access: Write access: - Hold owner OR crew members only -- Verified via `io.atcr.hold.crew` records in owner's PDS +- Verified via `io.atcr.hold.crew` records in hold's embedded PDS Key insight: "Private" gates anonymous access, not authenticated access. This reflects ATProto's current limitation (no private PDS records yet). @@ -484,7 +485,8 @@ See `.env.hold.example` for all available options. Key environment variables: - `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` - S3 credentials - `S3_BUCKET`, `S3_ENDPOINT` - S3 configuration - `HOLD_PUBLIC` - Allow public reads (default: false) -- `HOLD_OWNER` - DID for auto-registration (optional) +- `HOLD_OWNER` - DID for captain record creation (optional) +- `HOLD_ALLOW_ALL_CREW` - Allow any authenticated user to register as crew (default: false) **Credential Helper**: - Token storage: `~/.atcr/credential-helper-token.json` (or Docker's credential store) @@ -539,11 +541,12 @@ When writing tests: - Client methods are consistent across authorization, token exchange, and refresh flows **Adding BYOS support for a user**: -1. User sets environment variables (storage credentials, public URL) -2. User runs hold service with `HOLD_OWNER` set - auto-registration via OAuth -3. Hold service creates `io.atcr.hold` + `io.atcr.hold.crew` records in PDS -4. AppView automatically queries PDS and routes blobs to user's storage -5. No AppView changes needed - fully decentralized +1. User sets environment variables (storage credentials, public URL, HOLD_OWNER) +2. User runs hold service - creates captain + crew records in embedded PDS +3. Hold creates `io.atcr.hold.captain` + `io.atcr.hold.crew` records +4. User sets sailor profile `defaultHold` to point to their hold +5. AppView automatically queries hold's PDS and routes blobs to user's storage +6. No AppView changes needed - fully decentralized **Supporting a new storage backend**: 1. Ensure driver is registered in `cmd/hold/main.go` imports diff --git a/docs/EMBEDDED_PDS.md b/docs/EMBEDDED_PDS.md index 6132243..0ddf74e 100644 --- a/docs/EMBEDDED_PDS.md +++ b/docs/EMBEDDED_PDS.md @@ -560,9 +560,9 @@ hasAccess := crew.Contains(userDID) - ✅ Discoverable via service type **OAuth implications:** -- OAuth registration flow no longer needed (hold is self-describing) -- OAuth code kept for backward compatibility with legacy registration records -- Future: Remove OAuth after migration period +- ✅ OAuth registration removed completely (hold is self-describing) +- Hold creates captain + crew records in its own embedded PDS +- No cross-PDS writes or OAuth flows needed ### 7. Multi-Tenancy diff --git a/pkg/appview/db/hold_store.go b/pkg/appview/db/hold_store.go new file mode 100644 index 0000000..9d0572a --- /dev/null +++ b/pkg/appview/db/hold_store.go @@ -0,0 +1,138 @@ +package db + +import ( + "database/sql" + "fmt" + "time" +) + +// HoldCaptainRecord represents a cached captain record from a hold's PDS +type HoldCaptainRecord struct { + HoldDID string + OwnerDID string + Public bool + AllowAllCrew bool + DeployedAt string + Region string + Provider string + UpdatedAt time.Time +} + +// GetCaptainRecord retrieves a captain record from the cache +// Returns nil if not found (cache miss) +func GetCaptainRecord(db *sql.DB, holdDID string) (*HoldCaptainRecord, error) { + query := ` + SELECT hold_did, owner_did, public, allow_all_crew, + deployed_at, region, provider, updated_at + FROM hold_captain_records + WHERE hold_did = ? + ` + + var record HoldCaptainRecord + var deployedAt, region, provider sql.NullString + + err := db.QueryRow(query, holdDID).Scan( + &record.HoldDID, + &record.OwnerDID, + &record.Public, + &record.AllowAllCrew, + &deployedAt, + ®ion, + &provider, + &record.UpdatedAt, + ) + + if err == sql.ErrNoRows { + return nil, nil // Cache miss - not an error + } + + if err != nil { + return nil, fmt.Errorf("failed to query captain record: %w", err) + } + + // Handle nullable fields + if deployedAt.Valid { + record.DeployedAt = deployedAt.String + } + if region.Valid { + record.Region = region.String + } + if provider.Valid { + record.Provider = provider.String + } + + return &record, nil +} + +// UpsertCaptainRecord inserts or updates a captain record in the cache +func UpsertCaptainRecord(db *sql.DB, record *HoldCaptainRecord) error { + query := ` + INSERT INTO hold_captain_records ( + hold_did, owner_did, public, allow_all_crew, + deployed_at, region, provider, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(hold_did) DO UPDATE SET + owner_did = excluded.owner_did, + public = excluded.public, + allow_all_crew = excluded.allow_all_crew, + deployed_at = excluded.deployed_at, + region = excluded.region, + provider = excluded.provider, + updated_at = excluded.updated_at + ` + + _, err := db.Exec(query, + record.HoldDID, + record.OwnerDID, + record.Public, + record.AllowAllCrew, + nullString(record.DeployedAt), + nullString(record.Region), + nullString(record.Provider), + record.UpdatedAt, + ) + + if err != nil { + return fmt.Errorf("failed to upsert captain record: %w", err) + } + + return nil +} + +// ListHoldDIDs returns all known hold DIDs from the cache +func ListHoldDIDs(db *sql.DB) ([]string, error) { + query := ` + SELECT hold_did + FROM hold_captain_records + ORDER BY updated_at DESC + ` + + rows, err := db.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query hold DIDs: %w", err) + } + defer rows.Close() + + var holdDIDs []string + for rows.Next() { + var holdDID string + if err := rows.Scan(&holdDID); err != nil { + return nil, fmt.Errorf("failed to scan hold DID: %w", err) + } + holdDIDs = append(holdDIDs, holdDID) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating hold DIDs: %w", err) + } + + return holdDIDs, nil +} + +// nullString converts a string to sql.NullString +func nullString(s string) sql.NullString { + if s == "" { + return sql.NullString{Valid: false} + } + return sql.NullString{String: s, Valid: true} +} diff --git a/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml b/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml new file mode 100644 index 0000000..36cbb2c --- /dev/null +++ b/pkg/appview/db/migrations/0002_add_hold_captain_records.yaml @@ -0,0 +1,13 @@ +description: Add hold_captain_records table for caching hold security settings +query: | + CREATE TABLE IF NOT EXISTS hold_captain_records ( + hold_did TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + public BOOLEAN NOT NULL, + allow_all_crew BOOLEAN NOT NULL, + deployed_at TEXT, + region TEXT, + provider TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at); diff --git a/pkg/appview/db/schema.go b/pkg/appview/db/schema.go index 68f4a84..93c36a2 100644 --- a/pkg/appview/db/schema.go +++ b/pkg/appview/db/schema.go @@ -167,6 +167,18 @@ CREATE TABLE IF NOT EXISTS stars ( ); CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository); CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did); + +CREATE TABLE IF NOT EXISTS hold_captain_records ( + hold_did TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + public BOOLEAN NOT NULL, + allow_all_crew BOOLEAN NOT NULL, + deployed_at TEXT, + region TEXT, + provider TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_hold_captain_updated ON hold_captain_records(updated_at); ` // InitDB initializes the SQLite database with the schema diff --git a/pkg/appview/storage/routing_repository.go b/pkg/appview/storage/routing_repository.go index dc6291c..ae1bc42 100644 --- a/pkg/appview/storage/routing_repository.go +++ b/pkg/appview/storage/routing_repository.go @@ -54,7 +54,10 @@ func (r *RoutingRepository) Manifests(ctx context.Context, options ...distributi // Ensure blob store is created first (needed for label extraction during push) blobStore := r.Blobs(ctx) - r.manifestStore = atproto.NewManifestStore(r.atprotoClient, r.repositoryName, r.storageEndpoint, r.did, blobStore, r.database) + // Resolve hold endpoint URL to DID + holdDID := atproto.ResolveHoldDIDFromURL(r.storageEndpoint) + + r.manifestStore = atproto.NewManifestStore(r.atprotoClient, r.repositoryName, r.storageEndpoint, holdDID, r.did, blobStore, r.database) } // After any manifest operation, cache the hold endpoint for blob fetches diff --git a/pkg/atproto/lexicon.go b/pkg/atproto/lexicon.go index 2ab2576..5a69876 100644 --- a/pkg/atproto/lexicon.go +++ b/pkg/atproto/lexicon.go @@ -41,9 +41,16 @@ type ManifestRecord struct { // Digest is the content digest (e.g., "sha256:abc123...") Digest string `json:"digest"` - // HoldEndpoint is the hold service endpoint where blobs are stored + // HoldDID is the DID of the hold service where blobs are stored + // This is the primary reference for hold resolution + // e.g., "did:web:hold01.atcr.io" + HoldDID string `json:"holdDid,omitempty"` + + // HoldEndpoint is the hold service endpoint URL where blobs are stored (DEPRECATED) + // Kept for backward compatibility with manifests created before DID migration + // New manifests should use HoldDID instead // This is a historical reference that doesn't change even if user's default hold changes - HoldEndpoint string `json:"holdEndpoint"` + HoldEndpoint string `json:"holdEndpoint,omitempty"` // MediaType is the OCI media type (e.g., "application/vnd.oci.image.manifest.v1+json") MediaType string `json:"mediaType"` @@ -261,7 +268,9 @@ type SailorProfileRecord struct { // Type should be "io.atcr.sailor.profile" Type string `json:"$type"` - // DefaultHold is the default hold endpoint for blob storage + // DefaultHold is the default hold DID for blob storage + // Can be a DID (e.g., "did:web:hold01.atcr.io") or legacy URL + // URLs are migrated to DIDs on user login // If null/empty, user has opted out of defaults DefaultHold string `json:"defaultHold,omitempty"` @@ -340,3 +349,25 @@ func ParseStarRecordKey(rkey string) (ownerDID, repository string, err error) { return parts[0], parts[1], nil } + +// ResolveHoldDIDFromURL converts a hold endpoint URL to a did:web DID +// For did:web holds: https://hold01.atcr.io → did:web:hold01.atcr.io +func ResolveHoldDIDFromURL(holdURL string) string { + // Handle empty URLs + if holdURL == "" { + return "" + } + + // Parse URL to get hostname + holdURL = strings.TrimPrefix(holdURL, "http://") + holdURL = strings.TrimPrefix(holdURL, "https://") + holdURL = strings.TrimSuffix(holdURL, "/") + + // Extract hostname (remove path if present) + parts := strings.Split(holdURL, "/") + hostname := parts[0] + + // Convert to did:web + // did:web uses hostname directly (port included if non-standard) + return "did:web:" + hostname +} diff --git a/pkg/atproto/manifest_store.go b/pkg/atproto/manifest_store.go index 00e9db3..73a6f41 100644 --- a/pkg/atproto/manifest_store.go +++ b/pkg/atproto/manifest_store.go @@ -23,7 +23,8 @@ type DatabaseMetrics interface { type ManifestStore struct { client *Client repository string - holdEndpoint string // Hold service endpoint where blobs are stored (for push) + holdEndpoint string // Hold service endpoint URL (for legacy, to be deprecated) + holdDID string // Hold service DID (primary reference) did string // User's DID for cache key lastFetchedHoldEndpoint string // Hold endpoint from most recently fetched manifest (for pull) blobStore distribution.BlobStore // Blob store for fetching config during push @@ -31,11 +32,12 @@ type ManifestStore struct { } // NewManifestStore creates a new ATProto-backed manifest store -func NewManifestStore(client *Client, repository string, holdEndpoint string, did string, blobStore distribution.BlobStore, database DatabaseMetrics) *ManifestStore { +func NewManifestStore(client *Client, repository string, holdEndpoint string, holdDID string, did string, blobStore distribution.BlobStore, database DatabaseMetrics) *ManifestStore { return &ManifestStore{ client: client, repository: repository, holdEndpoint: holdEndpoint, + holdDID: holdDID, did: did, blobStore: blobStore, database: database, @@ -73,8 +75,23 @@ func (s *ManifestStore) Get(ctx context.Context, dgst digest.Digest, options ... } // Store the hold endpoint for subsequent blob requests during pull + // Prefer HoldDID (new format) with fallback to HoldEndpoint (legacy URL format) // The routing repository will cache this for concurrent blob fetches - s.lastFetchedHoldEndpoint = manifestRecord.HoldEndpoint + if manifestRecord.HoldDID != "" { + // New format: DID reference + // Convert did:web back to URL for blob fetching + // TODO: Routing repository should handle DID→URL conversion + // For now, fall back to HoldEndpoint if available + if manifestRecord.HoldEndpoint != "" { + s.lastFetchedHoldEndpoint = manifestRecord.HoldEndpoint + } else { + // Convert did:web:hold.example.com → https://hold.example.com + s.lastFetchedHoldEndpoint = didToURL(manifestRecord.HoldDID) + } + } else if manifestRecord.HoldEndpoint != "" { + // Legacy format: URL reference + s.lastFetchedHoldEndpoint = manifestRecord.HoldEndpoint + } var ociManifest []byte @@ -127,9 +144,10 @@ func (s *ManifestStore) Put(ctx context.Context, manifest distribution.Manifest, return "", fmt.Errorf("failed to create manifest record: %w", err) } - // Set the blob reference and hold endpoint + // Set the blob reference, hold DID, and hold endpoint manifestRecord.ManifestBlob = blobRef - manifestRecord.HoldEndpoint = s.holdEndpoint + manifestRecord.HoldDID = s.holdDID // Primary reference (DID) + manifestRecord.HoldEndpoint = s.holdEndpoint // Legacy reference (URL) for backward compat // Extract Dockerfile labels from config blob and add to annotations if s.blobStore != nil && manifestRecord.Config.Digest != "" { @@ -276,3 +294,14 @@ func (s *ManifestStore) extractConfigLabels(ctx context.Context, configDigestStr return configJSON.Config.Labels, nil } + +// didToURL converts a did:web DID to an HTTPS URL +// e.g., did:web:hold.example.com → https://hold.example.com +func didToURL(didWeb string) string { + if !strings.HasPrefix(didWeb, "did:web:") { + return didWeb // Not a did:web, return as-is + } + + hostname := strings.TrimPrefix(didWeb, "did:web:") + return "https://" + hostname +} diff --git a/pkg/auth/oauth/server.go b/pkg/auth/oauth/server.go index 67b7cab..5d60454 100644 --- a/pkg/auth/oauth/server.go +++ b/pkg/auth/oauth/server.go @@ -11,6 +11,7 @@ import ( "atcr.io/pkg/appview/db" "atcr.io/pkg/atproto" + indigooauth "github.com/bluesky-social/indigo/atproto/auth/oauth" "github.com/bluesky-social/indigo/atproto/syntax" ) @@ -314,6 +315,68 @@ func (s *Server) fetchAndStoreAvatar(ctx context.Context, did, sessionID, handle } fmt.Printf("DEBUG [oauth/server]: Stored user with avatar for DID=%s\n", did) + + // Handle profile migration and crew registration + s.migrateProfileAndRegisterCrew(ctx, client, did, session) +} + +// migrateProfileAndRegisterCrew handles URL→DID migration and crew registration +func (s *Server) migrateProfileAndRegisterCrew(ctx context.Context, client *atproto.Client, did string, session *indigooauth.ClientSession) { + // Get user's sailor profile + profile, err := atproto.GetProfile(ctx, client) + if err != nil { + fmt.Printf("WARNING [oauth/server]: Failed to get profile for %s: %v\n", did, err) + return + } + + if profile == nil || profile.DefaultHold == "" { + // No profile or no default hold configured + return + } + + // Check if defaultHold is a URL (needs migration) + var holdDID string + if strings.HasPrefix(profile.DefaultHold, "http://") || strings.HasPrefix(profile.DefaultHold, "https://") { + fmt.Printf("DEBUG [oauth/server]: Migrating hold URL to DID for %s: %s\n", did, profile.DefaultHold) + + // Resolve URL to DID + holdDID = resolveHoldDIDFromURL(profile.DefaultHold) + + // Update profile with DID + profile.DefaultHold = holdDID + if err := atproto.UpdateProfile(ctx, client, profile); err != nil { + fmt.Printf("WARNING [oauth/server]: Failed to update profile with hold DID for %s: %v\n", did, err) + // Continue anyway - crew registration might still work + } else { + fmt.Printf("DEBUG [oauth/server]: Updated profile with hold DID: %s\n", holdDID) + } + } else { + // Already a DID + holdDID = profile.DefaultHold + } + + // TODO: Request crew membership at the hold + // This requires understanding how to make authenticated HTTP requests with indigo's ClientSession + // For now, crew registration will happen on first push when appview validates access + fmt.Printf("DEBUG [oauth/server]: Skipping crew registration for now - will happen on first push. Hold DID: %s\n", holdDID) + _ = session // TODO: use session for crew registration +} + +// resolveHoldDIDFromURL converts a hold endpoint URL to a DID +// For did:web holds: https://hold01.atcr.io → did:web:hold01.atcr.io +func resolveHoldDIDFromURL(holdURL string) string { + // Parse URL to get hostname + holdURL = strings.TrimPrefix(holdURL, "http://") + holdURL = strings.TrimPrefix(holdURL, "https://") + holdURL = strings.TrimSuffix(holdURL, "/") + + // Extract hostname (remove path if present) + parts := strings.Split(holdURL, "/") + hostname := parts[0] + + // Convert to did:web + // did:web uses hostname directly (port included if non-standard) + return "did:web:" + hostname } // HTML templates