From c6f9062979101b2e7dbd526805b4f9d4a0bc9797 Mon Sep 17 00:00:00 2001 From: lewis Date: Sun, 14 Dec 2025 23:55:04 +0200 Subject: [PATCH] Creating & posting records works. Also messed up newlines but will fix later. --- .config/nextest.toml | 2 - .env.example | 31 - .gitignore | 4 +- ...1473bafa06610e54f10d9b3cb2cc06469854.json} | 32 +- ...f0b093523c27e9527cd1f623c9a87b05b532.json} | 32 +- ...35083354c87330101ea0ea69d30f2a5b4b960.json | 23 + ...d01c27738d4677b7cae7ea6697f3f5135388.json} | 32 +- ...66a419c2f85244d2bb7188604c8f9c87b1b3.json} | 4 +- ...b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7.json | 23 + ...d8f751893039fb6bc2363d9760ca0d8a8a56.json} | 32 +- ...0ef4509aef9416a177dc6888a8e2648b173f.json} | 8 +- ...9ca026cc38e7839b6c567745fcd5186462cc3.json | 24 + Cargo.toml | 4 - Dockerfile | 11 - README.md | 17 - TODO.md | 34 - deploy/nginx/nginx-quadlet.conf | 10 - deploy/quadlets/bspds-app.container | 3 - deploy/quadlets/bspds-db.container | 3 - deploy/quadlets/bspds-minio.container | 3 - deploy/quadlets/bspds-nginx.container | 3 - deploy/quadlets/bspds-valkey.container | 3 - deploy/quadlets/bspds.pod | 1 - docker-compose.prod.yml | 8 - docker-compose.yaml | 5 - docs/install-alpine.md | 78 --- docs/install-containers.md | 120 ---- docs/install-debian.md | 70 -- docs/install-kubernetes.md | 97 --- docs/install-openbsd.md | 96 --- frontend/src/App.svelte | 11 - frontend/src/lib/api.ts | 43 -- frontend/src/lib/auth.svelte.ts | 20 - frontend/src/lib/router.svelte.ts | 3 - frontend/src/main.ts | 2 - frontend/src/routes/AppPasswords.svelte | 51 -- frontend/src/routes/Dashboard.svelte | 32 - frontend/src/routes/InviteCodes.svelte | 52 -- frontend/src/routes/Login.svelte | 45 -- frontend/src/routes/Notifications.svelte | 61 -- frontend/src/routes/Register.svelte | 63 -- frontend/src/routes/RepoExplorer.svelte | 118 ---- frontend/src/routes/Settings.svelte | 57 -- frontend/src/tests/AppPasswords.test.ts | 101 --- frontend/src/tests/Dashboard.test.ts | 27 - frontend/src/tests/Login.test.ts | 31 - frontend/src/tests/Notifications.test.ts | 97 --- frontend/src/tests/Settings.test.ts | 126 ---- frontend/src/tests/mocks.ts | 40 -- frontend/src/tests/setup.ts | 4 - frontend/src/tests/utils.ts | 8 - frontend/svelte.config.js | 2 - frontend/vite.config.ts | 1 - frontend/vitest.config.ts | 1 - justfile | 33 - migrations/20251211_initial_schema.sql | 48 -- migrations/20251213_performance_indexes.sql | 6 - migrations/20251214_add_prev_data_cid.sql | 1 + .../20251215_add_identity_account_fields.sql | 3 + nginx.prod.conf | 16 - ref_pds_downloader.sh | 3 - scripts/install-debian.sh | 95 --- scripts/run-tests.sh | 8 - scripts/test-infra.sh | 26 +- src/api/actor/mod.rs | 1 - src/api/actor/preferences.rs | 26 - src/api/actor/profile.rs | 29 - src/api/admin/account/delete.rs | 24 +- src/api/admin/account/email.rs | 12 - src/api/admin/account/info.rs | 15 - src/api/admin/account/mod.rs | 3 +- src/api/admin/account/profile.rs | 154 +++++ src/api/admin/account/update.rs | 23 - src/api/admin/invite.rs | 31 - src/api/admin/mod.rs | 5 +- src/api/admin/status.rs | 43 +- src/api/error.rs | 10 - src/api/feed/actor_likes.rs | 18 - src/api/feed/author_feed.rs | 19 - src/api/feed/custom_feed.rs | 16 - src/api/feed/mod.rs | 1 - src/api/feed/post_thread.rs | 37 -- src/api/feed/timeline.rs | 35 - src/api/identity/account.rs | 83 +-- src/api/identity/did.rs | 68 +- src/api/identity/mod.rs | 1 - src/api/identity/plc/mod.rs | 1 - src/api/identity/plc/request.rs | 11 - src/api/identity/plc/sign.rs | 22 - src/api/identity/plc/submit.rs | 25 - src/api/mod.rs | 1 - src/api/moderation/mod.rs | 10 - src/api/notification/mod.rs | 1 - src/api/notification/register_push.rs | 16 - src/api/notification_prefs.rs | 23 - src/api/proxy.rs | 15 - src/api/proxy_client.rs | 39 -- src/api/read_after_write.rs | 50 -- src/api/repo/blob.rs | 39 -- src/api/repo/import.rs | 34 +- src/api/repo/meta.rs | 22 +- src/api/repo/mod.rs | 1 - src/api/repo/record/batch.rs | 62 +- src/api/repo/record/delete.rs | 34 +- src/api/repo/record/mod.rs | 1 - src/api/repo/record/read.rs | 32 - src/api/repo/record/utils.rs | 302 +++++++-- src/api/repo/record/validation.rs | 1 - src/api/repo/record/write.rs | 82 +-- src/api/server/account_status.rs | 58 -- src/api/server/app_password.rs | 20 - src/api/server/email.rs | 48 -- src/api/server/invite.rs | 35 - src/api/server/meta.rs | 6 - src/api/server/mod.rs | 1 - src/api/server/password.rs | 32 - src/api/server/service_auth.rs | 8 - src/api/server/session.rs | 62 -- src/api/server/signing_key.rs | 12 - src/api/temp.rs | 4 - src/api/validation.rs | 19 - src/auth/extractor.rs | 27 - src/auth/mod.rs | 32 - src/auth/token.rs | 38 -- src/auth/verify.rs | 41 -- src/cache/mod.rs | 30 - src/circuit_breaker.rs | 45 -- src/config.rs | 30 - src/crawlers.rs | 17 - src/image/mod.rs | 51 -- src/lib.rs | 12 +- src/main.rs | 29 - src/metrics.rs | 29 - src/notifications/mod.rs | 1 - src/notifications/sender.rs | 60 -- src/notifications/service.rs | 57 -- src/notifications/types.rs | 7 - src/oauth/client.rs | 96 --- src/oauth/db/client.rs | 6 - src/oauth/db/device.rs | 16 - src/oauth/db/dpop.rs | 5 - src/oauth/db/helpers.rs | 3 - src/oauth/db/mod.rs | 1 - src/oauth/db/request.rs | 16 - src/oauth/db/token.rs | 27 - src/oauth/db/two_factor.rs | 18 - src/oauth/dpop.rs | 70 -- src/oauth/endpoints/authorize.rs | 96 --- src/oauth/endpoints/metadata.rs | 11 - src/oauth/endpoints/mod.rs | 1 - src/oauth/endpoints/par.rs | 29 - src/oauth/endpoints/token/grants.rs | 40 -- src/oauth/endpoints/token/helpers.rs | 31 - src/oauth/endpoints/token/introspect.rs | 15 - src/oauth/endpoints/token/mod.rs | 7 - src/oauth/endpoints/token/types.rs | 2 - src/oauth/error.rs | 6 - src/oauth/jwks.rs | 3 - src/oauth/mod.rs | 1 - src/oauth/templates.rs | 85 --- src/oauth/types.rs | 27 - src/oauth/verify.rs | 43 -- src/plc/mod.rs | 106 --- src/rate_limit.rs | 35 - src/repo/mod.rs | 21 - src/repo/tracking.rs | 45 +- src/state.rs | 13 - src/storage/mod.rs | 16 - src/sync/blob.rs | 20 - src/sync/car.rs | 4 - src/sync/commit.rs | 60 +- src/sync/crawl.rs | 4 - src/sync/deprecated.rs | 21 - src/sync/firehose.rs | 5 +- src/sync/frame.rs | 132 +++- src/sync/import.rs | 58 -- src/sync/listener.rs | 19 +- src/sync/mod.rs | 1 - src/sync/repo.rs | 177 ++--- src/sync/subscribe_repos.rs | 17 +- src/sync/util.rs | 282 ++++++-- src/sync/verify.rs | 48 -- src/sync/verify_tests.rs | 77 --- src/util.rs | 15 - src/validation/mod.rs | 77 --- tests/actor.rs | 56 -- tests/admin_email.rs | 31 - tests/admin_invite.rs | 51 -- tests/admin_moderation.rs | 41 -- tests/appview_integration.rs | 20 - tests/common/mod.rs | 82 --- tests/delete_account.rs | 56 -- tests/email_update.rs | 65 -- tests/feed.rs | 22 - tests/firehose.rs | 151 ++++- tests/firehose_validation.rs | 611 ++++++++++++++++++ tests/helpers/mod.rs | 24 - tests/identity.rs | 52 -- tests/image_processing.rs | 54 -- tests/import_verification.rs | 49 -- tests/import_with_verification.rs | 89 --- tests/invite.rs | 41 -- tests/jwt_security.rs | 193 ------ tests/lifecycle_record.rs | 162 ----- tests/lifecycle_session.rs | 67 -- tests/lifecycle_social.rs | 81 --- tests/moderation.rs | 10 - tests/notifications.rs | 22 - tests/oauth.rs | 250 ------- tests/oauth_lifecycle.rs | 147 ----- tests/oauth_security.rs | 312 --------- tests/password_reset.rs | 62 -- tests/plc_migration.rs | 182 +----- tests/plc_operations.rs | 62 -- tests/plc_validation.rs | 74 --- tests/proxy.rs | 30 - tests/rate_limit.rs | 33 - tests/record_validation.rs | 53 -- tests/repo_batch.rs | 29 - tests/repo_blob.rs | 14 - tests/security_fixes.rs | 80 --- tests/server.rs | 59 -- tests/signing_key.rs | 44 -- tests/sync_blob.rs | 15 - tests/sync_deprecated.rs | 42 -- tests/sync_repo.rs | 70 -- tests/verify_live_commit.rs | 131 ++++ 227 files changed, 2122 insertions(+), 7842 deletions(-) rename .sqlx/{query-a63aed47193f06cd11d87157799c17a591e0a0be4487f718250eaf7afd4b4b07.json => query-1bff90667ece9e1e44e20e3477df1473bafa06610e54f10d9b3cb2cc06469854.json} (62%) rename .sqlx/{query-8a7a8f0c4c0872c21c46d484219624215bdb14617b9f9a44974e394a28147f70.json => query-239555df14c147a09096beb28f2ff0b093523c27e9527cd1f623c9a87b05b532.json} (64%) create mode 100644 .sqlx/query-3792c455a955b8cf2c70c8aa76635083354c87330101ea0ea69d30f2a5b4b960.json rename .sqlx/{query-7b6ca5155c645f2011364594833effe41494d0c855b0c9314ab56ea8b0fb4e6d.json => query-44b78996f9799398f384d9aebb36d01c27738d4677b7cae7ea6697f3f5135388.json} (64%) rename .sqlx/{query-7ce9c5fb943b4217da17c041998263c0af7b77c8feecc654dde7a71fbab4e1ad.json => query-52196f20028eb03e8116abce050866a419c2f85244d2bb7188604c8f9c87b1b3.json} (65%) create mode 100644 .sqlx/query-5425a06bb6b83767fd334f70702b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7.json rename .sqlx/{query-b1c54d3f3e2d3031c0d926ccb0d39a0250320d41d08df65d6d9dcc640451527d.json => query-777386dcbf2aa2785a6c16abdccbd8f751893039fb6bc2363d9760ca0d8a8a56.json} (62%) rename .sqlx/{query-f68a05d2c78cc060b43c81b177a24f89c71e7e00dfa5af08ff4584bbe43b4155.json => query-d7d7e002dcdc663811303411c1200ef4509aef9416a177dc6888a8e2648b173f.json} (59%) create mode 100644 .sqlx/query-f2777a739a950b94f978e3196809ca026cc38e7839b6c567745fcd5186462cc3.json create mode 100644 migrations/20251214_add_prev_data_cid.sql create mode 100644 migrations/20251215_add_identity_account_fields.sql create mode 100644 src/api/admin/account/profile.rs create mode 100644 tests/firehose_validation.rs create mode 100644 tests/verify_live_commit.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 6df2ec9..2cfd181 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,11 +1,9 @@ [store] dir = "target/nextest" - [profile.default] retries = 0 fail-fast = true test-threads = "num-cpus" - [profile.ci] retries = 2 fail-fast = false diff --git a/.env.example b/.env.example index 328fb69..b44702a 100644 --- a/.env.example +++ b/.env.example @@ -3,20 +3,16 @@ # ============================================================================= SERVER_HOST=127.0.0.1 SERVER_PORT=3000 - # The public-facing hostname of the PDS (used in DID documents, JWTs, etc.) PDS_HOSTNAME=localhost:3000 - # ============================================================================= # Database # ============================================================================= DATABASE_URL=postgres://postgres:postgres@localhost:5432/pds - # Connection pool settings (defaults are good for most deployments) # DATABASE_MAX_CONNECTIONS=100 # DATABASE_MIN_CONNECTIONS=10 # DATABASE_ACQUIRE_TIMEOUT_SECS=30 - # ============================================================================= # Blob Storage (S3-compatible) # ============================================================================= @@ -25,120 +21,93 @@ AWS_REGION=us-east-1 S3_BUCKET=pds-blobs AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin - # ============================================================================= # Valkey (for caching and distributed rate limiting) # ============================================================================= # If not set, falls back to in-memory caching (single-node only) # VALKEY_URL=redis://localhost:6379 - # ============================================================================= # Security Secrets # ============================================================================= # These MUST be set in production (minimum 32 characters each) # In development, set BSPDS_ALLOW_INSECURE_SECRETS=1 to use defaults - # Server-wide secret for OAuth token signing (HS256) # JWT_SECRET=your-secure-random-string-at-least-32-chars - # Secret for DPoP proof validation # DPOP_SECRET=your-secure-random-string-at-least-32-chars - # Key for encrypting user signing keys at rest (AES-256-GCM) # MASTER_KEY=your-secure-random-string-at-least-32-chars - # Set this ONLY in development to allow default/weak secrets # BSPDS_ALLOW_INSECURE_SECRETS=1 - # ============================================================================= # PLC Directory # ============================================================================= # PLC_DIRECTORY_URL=https://plc.directory # PLC_TIMEOUT_SECS=10 # PLC_CONNECT_TIMEOUT_SECS=5 - # Optional: rotation key for PLC operations (defaults to user's key) # PLC_ROTATION_KEY=did:key:... - # ============================================================================= # Federation # ============================================================================= # Appview URL for proxying app.bsky.* requests # APPVIEW_URL=https://api.bsky.app - # Comma-separated list of relay URLs to notify via requestCrawl # CRAWLERS=https://bsky.network - # ============================================================================= # Firehose (subscribeRepos WebSocket) # ============================================================================= # Buffer size for firehose broadcast channel # FIREHOSE_BUFFER_SIZE=10000 - # Disconnect slow consumers after this many events of lag # FIREHOSE_MAX_LAG=5000 - # ============================================================================= # Notification Service # ============================================================================= # Queue processing settings # NOTIFICATION_BATCH_SIZE=100 # NOTIFICATION_POLL_INTERVAL_MS=1000 - # Email notifications (via sendmail/msmtp) # MAIL_FROM_ADDRESS=noreply@example.com # MAIL_FROM_NAME=My PDS # SENDMAIL_PATH=/usr/sbin/sendmail - # Discord notifications (via webhook) # DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... - # Telegram notifications (via bot) # TELEGRAM_BOT_TOKEN=your-bot-token - # Signal notifications (via signal-cli) # SIGNAL_CLI_PATH=/usr/local/bin/signal-cli # SIGNAL_SENDER_NUMBER=+1234567890 - # ============================================================================= # Repository Import # ============================================================================= # Set to "true" to accept repository imports # ACCEPTING_REPO_IMPORTS=false - # Maximum import size in bytes (default: 50MB) # MAX_IMPORT_SIZE=52428800 - # Maximum blocks per import (default: 100000) # MAX_IMPORT_BLOCKS=100000 - # Skip verification during import (testing only) # SKIP_IMPORT_VERIFICATION=false - # ============================================================================= # Account Registration # ============================================================================= # Require invite codes for registration # INVITE_CODE_REQUIRED=false - # Comma-separated list of available user domains # AVAILABLE_USER_DOMAINS=example.com - # ============================================================================= # Rate Limiting # ============================================================================= # Disable all rate limiting (testing only, NEVER in production) # DISABLE_RATE_LIMITING=1 - # ============================================================================= # Miscellaneous # ============================================================================= # Allow HTTP for proxy requests (development only) # ALLOW_HTTP_PROXY=1 - # Custom frontend directory (defaults to ./frontend/dist) # FRONTEND_DIR=/path/to/frontend/dist - CARGO_MOMMYS_LITTLE=mister CARGO_MOMMYS_PRONOUNS=his CARGO_MOMMYS_ROLES=daddy diff --git a/.gitignore b/.gitignore index 39ac4a5..03d3f1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,8 @@ /target - .env - reference-pds-hailey/ reference-pds-bsky/ - +reference-relay-indigo/ # Frontend build artifacts frontend/node_modules/ frontend/dist/ diff --git a/.sqlx/query-a63aed47193f06cd11d87157799c17a591e0a0be4487f718250eaf7afd4b4b07.json b/.sqlx/query-1bff90667ece9e1e44e20e3477df1473bafa06610e54f10d9b3cb2cc06469854.json similarity index 62% rename from .sqlx/query-a63aed47193f06cd11d87157799c17a591e0a0be4487f718250eaf7afd4b4b07.json rename to .sqlx/query-1bff90667ece9e1e44e20e3477df1473bafa06610e54f10d9b3cb2cc06469854.json index a8cbca7..cfcb8ac 100644 --- a/.sqlx/query-a63aed47193f06cd11d87157799c17a591e0a0be4487f718250eaf7afd4b4b07.json +++ b/.sqlx/query-1bff90667ece9e1e44e20e3477df1473bafa06610e54f10d9b3cb2cc06469854.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC\n ", + "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, prev_data_cid, ops, blobs, blocks_cids, handle, active, status\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2\n ", "describe": { "columns": [ { @@ -35,18 +35,38 @@ }, { "ordinal": 6, + "name": "prev_data_cid", + "type_info": "Text" + }, + { + "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { - "ordinal": 7, + "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { - "ordinal": 8, + "ordinal": 9, "name": "blocks_cids", "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "handle", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "active", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "status", + "type_info": "Text" } ], "parameters": { @@ -64,8 +84,12 @@ true, true, true, + true, + true, + true, + true, true ] }, - "hash": "a63aed47193f06cd11d87157799c17a591e0a0be4487f718250eaf7afd4b4b07" + "hash": "1bff90667ece9e1e44e20e3477df1473bafa06610e54f10d9b3cb2cc06469854" } diff --git a/.sqlx/query-8a7a8f0c4c0872c21c46d484219624215bdb14617b9f9a44974e394a28147f70.json b/.sqlx/query-239555df14c147a09096beb28f2ff0b093523c27e9527cd1f623c9a87b05b532.json similarity index 64% rename from .sqlx/query-8a7a8f0c4c0872c21c46d484219624215bdb14617b9f9a44974e394a28147f70.json rename to .sqlx/query-239555df14c147a09096beb28f2ff0b093523c27e9527cd1f623c9a87b05b532.json index d1f930d..503820c 100644 --- a/.sqlx/query-8a7a8f0c4c0872c21c46d484219624215bdb14617b9f9a44974e394a28147f70.json +++ b/.sqlx/query-239555df14c147a09096beb28f2ff0b093523c27e9527cd1f623c9a87b05b532.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n ", + "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, prev_data_cid, ops, blobs, blocks_cids, handle, active, status\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n ", "describe": { "columns": [ { @@ -35,18 +35,38 @@ }, { "ordinal": 6, + "name": "prev_data_cid", + "type_info": "Text" + }, + { + "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { - "ordinal": 7, + "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { - "ordinal": 8, + "ordinal": 9, "name": "blocks_cids", "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "handle", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "active", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "status", + "type_info": "Text" } ], "parameters": { @@ -63,8 +83,12 @@ true, true, true, + true, + true, + true, + true, true ] }, - "hash": "8a7a8f0c4c0872c21c46d484219624215bdb14617b9f9a44974e394a28147f70" + "hash": "239555df14c147a09096beb28f2ff0b093523c27e9527cd1f623c9a87b05b532" } diff --git a/.sqlx/query-3792c455a955b8cf2c70c8aa76635083354c87330101ea0ea69d30f2a5b4b960.json b/.sqlx/query-3792c455a955b8cf2c70c8aa76635083354c87330101ea0ea69d30f2a5b4b960.json new file mode 100644 index 0000000..1e90686 --- /dev/null +++ b/.sqlx/query-3792c455a955b8cf2c70c8aa76635083354c87330101ea0ea69d30f2a5b4b960.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid)\n VALUES ($1, 'sync', $2)\n RETURNING seq\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seq", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3792c455a955b8cf2c70c8aa76635083354c87330101ea0ea69d30f2a5b4b960" +} diff --git a/.sqlx/query-7b6ca5155c645f2011364594833effe41494d0c855b0c9314ab56ea8b0fb4e6d.json b/.sqlx/query-44b78996f9799398f384d9aebb36d01c27738d4677b7cae7ea6697f3f5135388.json similarity index 64% rename from .sqlx/query-7b6ca5155c645f2011364594833effe41494d0c855b0c9314ab56ea8b0fb4e6d.json rename to .sqlx/query-44b78996f9799398f384d9aebb36d01c27738d4677b7cae7ea6697f3f5135388.json index f95710d..5f566ab 100644 --- a/.sqlx/query-7b6ca5155c645f2011364594833effe41494d0c855b0c9314ab56ea8b0fb4e6d.json +++ b/.sqlx/query-44b78996f9799398f384d9aebb36d01c27738d4677b7cae7ea6697f3f5135388.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids\n FROM repo_seq\n WHERE seq = $1\n ", + "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, prev_data_cid, ops, blobs, blocks_cids, handle, active, status\n FROM repo_seq\n WHERE seq = $1\n ", "describe": { "columns": [ { @@ -35,18 +35,38 @@ }, { "ordinal": 6, + "name": "prev_data_cid", + "type_info": "Text" + }, + { + "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { - "ordinal": 7, + "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { - "ordinal": 8, + "ordinal": 9, "name": "blocks_cids", "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "handle", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "active", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "status", + "type_info": "Text" } ], "parameters": { @@ -63,8 +83,12 @@ true, true, true, + true, + true, + true, + true, true ] }, - "hash": "7b6ca5155c645f2011364594833effe41494d0c855b0c9314ab56ea8b0fb4e6d" + "hash": "44b78996f9799398f384d9aebb36d01c27738d4677b7cae7ea6697f3f5135388" } diff --git a/.sqlx/query-7ce9c5fb943b4217da17c041998263c0af7b77c8feecc654dde7a71fbab4e1ad.json b/.sqlx/query-52196f20028eb03e8116abce050866a419c2f85244d2bb7188604c8f9c87b1b3.json similarity index 65% rename from .sqlx/query-7ce9c5fb943b4217da17c041998263c0af7b77c8feecc654dde7a71fbab4e1ad.json rename to .sqlx/query-52196f20028eb03e8116abce050866a419c2f85244d2bb7188604c8f9c87b1b3.json index d40aeae..13d0bd8 100644 --- a/.sqlx/query-7ce9c5fb943b4217da17c041998263c0af7b77c8feecc654dde7a71fbab4e1ad.json +++ b/.sqlx/query-52196f20028eb03e8116abce050866a419c2f85244d2bb7188604c8f9c87b1b3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n RETURNING seq\n ", + "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, prev_data_cid, ops, blobs, blocks_cids)\n VALUES ($1, 'commit', $2, $3, $4, $5, $6, $7)\n RETURNING seq\n ", "describe": { "columns": [ { @@ -24,5 +24,5 @@ false ] }, - "hash": "7ce9c5fb943b4217da17c041998263c0af7b77c8feecc654dde7a71fbab4e1ad" + "hash": "52196f20028eb03e8116abce050866a419c2f85244d2bb7188604c8f9c87b1b3" } diff --git a/.sqlx/query-5425a06bb6b83767fd334f70702b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7.json b/.sqlx/query-5425a06bb6b83767fd334f70702b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7.json new file mode 100644 index 0000000..ee83964 --- /dev/null +++ b/.sqlx/query-5425a06bb6b83767fd334f70702b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repo_seq (did, event_type, handle)\n VALUES ($1, 'identity', $2)\n RETURNING seq\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seq", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5425a06bb6b83767fd334f70702b9c0ca99d63bf1a70fa1641b35cc0bfde1dc7" +} diff --git a/.sqlx/query-b1c54d3f3e2d3031c0d926ccb0d39a0250320d41d08df65d6d9dcc640451527d.json b/.sqlx/query-777386dcbf2aa2785a6c16abdccbd8f751893039fb6bc2363d9760ca0d8a8a56.json similarity index 62% rename from .sqlx/query-b1c54d3f3e2d3031c0d926ccb0d39a0250320d41d08df65d6d9dcc640451527d.json rename to .sqlx/query-777386dcbf2aa2785a6c16abdccbd8f751893039fb6bc2363d9760ca0d8a8a56.json index fa8c69c..e52bb8e 100644 --- a/.sqlx/query-b1c54d3f3e2d3031c0d926ccb0d39a0250320d41d08df65d6d9dcc640451527d.json +++ b/.sqlx/query-777386dcbf2aa2785a6c16abdccbd8f751893039fb6bc2363d9760ca0d8a8a56.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2\n ", + "query": "\n SELECT seq, did, created_at, event_type, commit_cid, prev_cid, prev_data_cid, ops, blobs, blocks_cids, handle, active, status\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC\n ", "describe": { "columns": [ { @@ -35,18 +35,38 @@ }, { "ordinal": 6, + "name": "prev_data_cid", + "type_info": "Text" + }, + { + "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { - "ordinal": 7, + "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { - "ordinal": 8, + "ordinal": 9, "name": "blocks_cids", "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "handle", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "active", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "status", + "type_info": "Text" } ], "parameters": { @@ -64,8 +84,12 @@ true, true, true, + true, + true, + true, + true, true ] }, - "hash": "b1c54d3f3e2d3031c0d926ccb0d39a0250320d41d08df65d6d9dcc640451527d" + "hash": "777386dcbf2aa2785a6c16abdccbd8f751893039fb6bc2363d9760ca0d8a8a56" } diff --git a/.sqlx/query-f68a05d2c78cc060b43c81b177a24f89c71e7e00dfa5af08ff4584bbe43b4155.json b/.sqlx/query-d7d7e002dcdc663811303411c1200ef4509aef9416a177dc6888a8e2648b173f.json similarity index 59% rename from .sqlx/query-f68a05d2c78cc060b43c81b177a24f89c71e7e00dfa5af08ff4584bbe43b4155.json rename to .sqlx/query-d7d7e002dcdc663811303411c1200ef4509aef9416a177dc6888a8e2648b173f.json index 3dc3c70..bcc3dc1 100644 --- a/.sqlx/query-f68a05d2c78cc060b43c81b177a24f89c71e7e00dfa5af08ff4584bbe43b4155.json +++ b/.sqlx/query-d7d7e002dcdc663811303411c1200ef4509aef9416a177dc6888a8e2648b173f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids)\n VALUES ($1, 'commit', $2, $3, $4, $5, $6)\n RETURNING seq\n ", + "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n RETURNING seq\n ", "describe": { "columns": [ { @@ -14,14 +14,16 @@ "Text", "Text", "Text", + "Text", "Jsonb", "TextArray", - "TextArray" + "TextArray", + "Text" ] }, "nullable": [ false ] }, - "hash": "f68a05d2c78cc060b43c81b177a24f89c71e7e00dfa5af08ff4584bbe43b4155" + "hash": "d7d7e002dcdc663811303411c1200ef4509aef9416a177dc6888a8e2648b173f" } diff --git a/.sqlx/query-f2777a739a950b94f978e3196809ca026cc38e7839b6c567745fcd5186462cc3.json b/.sqlx/query-f2777a739a950b94f978e3196809ca026cc38e7839b6c567745fcd5186462cc3.json new file mode 100644 index 0000000..4b7412b --- /dev/null +++ b/.sqlx/query-f2777a739a950b94f978e3196809ca026cc38e7839b6c567745fcd5186462cc3.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repo_seq (did, event_type, active, status)\n VALUES ($1, 'account', $2, $3)\n RETURNING seq\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seq", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f2777a739a950b94f978e3196809ca026cc38e7839b6c567745fcd5186462cc3" +} diff --git a/Cargo.toml b/Cargo.toml index eeed7fc..0a522ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ name = "bspds" version = "0.1.0" edition = "2024" - [dependencies] anyhow = "1.0.100" async-trait = "0.1.89" @@ -54,14 +53,11 @@ redis = { version = "0.27", features = ["tokio-comp", "connection-manager"] } tower-http = { version = "0.6", features = ["fs", "cors"] } metrics = "0.24" metrics-exporter-prometheus = { version = "0.16", default-features = false, features = ["http-listener"] } - [features] external-infra = [] - [dev-dependencies] ctor = "0.6.3" testcontainers = "0.26.0" testcontainers-modules = { version = "0.14.0", features = ["postgres"] } wiremock = "0.6.5" - # urlencoding is also in dependencies, but tests use it directly diff --git a/Dockerfile b/Dockerfile index 7aedc1c..ba6f0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,36 +3,25 @@ FROM denoland/deno:alpine AS frontend-builder WORKDIR /frontend COPY frontend/ ./ RUN deno task build - # Stage 2: Build Rust backend FROM rust:1.92-alpine AS builder - RUN apk add ca-certificates openssl openssl-dev pkgconfig - WORKDIR /app - COPY Cargo.toml Cargo.lock ./ RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build --release && rm -rf src - COPY src ./src COPY tests ./tests COPY migrations ./migrations COPY .sqlx ./.sqlx RUN touch src/main.rs && cargo build --release - # Stage 3: Final image FROM alpine:3.23 - COPY --from=builder /app/target/release/bspds /usr/local/bin/bspds COPY --from=builder /app/migrations /app/migrations COPY --from=frontend-builder /frontend/dist /app/frontend/dist - WORKDIR /app - ENV SERVER_HOST=0.0.0.0 ENV SERVER_PORT=3000 ENV FRONTEND_DIR=/app/frontend/dist - EXPOSE 3000 - CMD ["bspds"] diff --git a/README.md b/README.md index 7f9c4c9..fef910c 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,34 @@ # BSPDS - A production-grade Personal Data Server (PDS) for the AT Protocol. Drop-in replacement for Bluesky's reference PDS, using postgres and s3-compatible blob storage. - ## Features - - Full AT Protocol support (`com.atproto.*` endpoints) - OAuth 2.1 provider (PKCE, DPoP, PAR) - WebSocket firehose (`subscribeRepos`) - Multi-channel notifications (email, discord, telegram, signal) - Built-in web UI for account management - Per-IP rate limiting - ## Quick Start - ```bash cp .env.example .env podman compose up -d just run ``` - ## Configuration - See `.env.example` for all configuration options. - ## Development - Run `just` to see available commands. - ```bash just test # run tests just lint # clippy + fmt ``` - ## Production Deployment - ### Quick Deploy (Docker/Podman Compose) - ```bash cp .env.prod.example .env.prod # Edit .env.prod with your values (generate secrets with: openssl rand -base64 48) podman-compose -f docker-compose.prod.yml up -d ``` - ### Full Installation Guides - | Guide | Best For | |-------|----------| | **Native Installation** | Maximum performance, full control | @@ -54,7 +39,5 @@ podman-compose -f docker-compose.prod.yml up -d | [Containers](docs/install-containers.md) | Podman with quadlets (Debian) or OpenRC (Alpine) | | **Orchestrated** | High availability, auto-scaling | | [Kubernetes](docs/install-kubernetes.md) | Multi-node k8s cluster deployment | - ## License - TBD diff --git a/TODO.md b/TODO.md index ac997cb..3b7ac9a 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,5 @@ # PDS Implementation TODOs - Lewis' corrected big boy todofile - ## Server Infrastructure & Proxying - [x] Health Check - [x] Implement `GET /health` endpoint (returns "OK"). @@ -12,7 +10,6 @@ Lewis' corrected big boy todofile - [x] Implement strict forwarding for all `app.bsky.*` and `chat.bsky.*` requests to an appview. - [x] Forward auth headers correctly. - [x] Handle appview errors/timeouts gracefully. - ## Authentication & Account Management (`com.atproto.server`) - [x] Account Creation - [x] Implement `com.atproto.server.createAccount`. @@ -43,7 +40,6 @@ Lewis' corrected big boy todofile - [x] Implement `com.atproto.server.revokeAppPassword`. - [x] Implement `com.atproto.server.updateEmail`. - [x] Implement `com.atproto.server.confirmEmail`. - ## Repository Operations (`com.atproto.repo`) - [x] Record CRUD - [x] Implement `com.atproto.repo.createRecord`. @@ -62,7 +58,6 @@ Lewis' corrected big boy todofile - [x] Implement `com.atproto.repo.uploadBlob`. - [x] Store blob (S3). - [x] return `blob` ref (CID + MimeType). - ## Sync & Federation (`com.atproto.sync`) - [x] The Firehose (WebSocket) - [x] Implement `com.atproto.sync.subscribeRepos`. @@ -84,7 +79,6 @@ Lewis' corrected big boy todofile - [x] Deprecated Sync Endpoints (for compatibility) - [x] Implement `com.atproto.sync.getCheckout` (deprecated). - [x] Implement `com.atproto.sync.getHead` (deprecated). - ## Identity (`com.atproto.identity`) - [x] Resolution - [x] Implement `com.atproto.identity.resolveHandle` (Can be internal or proxy to PLC). @@ -92,7 +86,6 @@ Lewis' corrected big boy todofile - [x] Implement `com.atproto.identity.submitPlcOperation` / `signPlcOperation` / `requestPlcOperationSignature`. - [x] Implement `com.atproto.identity.getRecommendedDidCredentials`. - [x] Implement `/.well-known/did.json` (Depends on supporting did:web). - ## Admin Management (`com.atproto.admin`) - [x] Implement `com.atproto.admin.deleteAccount`. - [x] Implement `com.atproto.admin.disableAccountInvites`. @@ -106,16 +99,12 @@ Lewis' corrected big boy todofile - [x] Implement `com.atproto.admin.updateAccountHandle`. - [x] Implement `com.atproto.admin.updateAccountPassword`. - [x] Implement `com.atproto.admin.updateSubjectStatus`. - ## Moderation (`com.atproto.moderation`) - [x] Implement `com.atproto.moderation.createReport`. - ## Temp Namespace (`com.atproto.temp`) - [x] Implement `com.atproto.temp.checkSignupQueue` (signup queue status for gated signups). - ## Misc HTTP Endpoints - [x] Implement `/robots.txt` endpoint. - ## OAuth 2.1 Support Full OAuth 2.1 provider for ATProto native app authentication. - [x] OAuth Provider Core @@ -141,11 +130,8 @@ Full OAuth 2.1 provider for ATProto native app authentication. - [x] Authorization UI templates (HTML login form). - [x] Implement `private_key_jwt` signature verification with async JWKS fetching. - [x] HS256 JWT support (matches reference PDS). - ## OAuth Security Notes - Security measures implemented: - - Constant-time comparison for signature verification (prevents timing attacks) - HMAC-SHA256 for access token signing with configurable secret - Production secrets require 32+ character minimum @@ -159,21 +145,17 @@ Security measures implemented: - Deactivated/taken-down accounts blocked from OAuth authorization - Client ID validation on token exchange (defense-in-depth against cross-client attacks) - HTML escaping in OAuth templates (XSS prevention) - ### Auth Notes - Dual algorithm support: ES256K (secp256k1 ECDSA) with per-user keys AND HS256 (HMAC) for compatibility with reference PDS. - Token storage: Storing only token JTIs in session_tokens table (defense in depth against DB breaches). Refresh token family tracking enables detection of token reuse attacks. - Key encryption: User signing keys encrypted at rest using AES-256-GCM with keys derived via HKDF from KEY_ENCRYPTION_KEY environment variable. - ## PDS-Level App Endpoints These endpoints need to be implemented at the PDS level (not just proxied to appview). - ### Actor (`app.bsky.actor`) - [x] Implement `app.bsky.actor.getPreferences` (user preferences storage). - [x] Implement `app.bsky.actor.putPreferences` (update user preferences). - [x] Implement `app.bsky.actor.getProfile` (PDS-level with proxy fallback). - [x] Implement `app.bsky.actor.getProfiles` (PDS-level with proxy fallback). - ### Feed (`app.bsky.feed`) These are implemented at PDS level to enable local-first reads (read-after-write pattern): - [x] Implement `app.bsky.feed.getTimeline` (PDS-level with proxy + RAW). @@ -181,10 +163,8 @@ These are implemented at PDS level to enable local-first reads (read-after-write - [x] Implement `app.bsky.feed.getActorLikes` (PDS-level with proxy + RAW). - [x] Implement `app.bsky.feed.getPostThread` (PDS-level with proxy + RAW + NotFound handling). - [x] Implement `app.bsky.feed.getFeed` (proxy to feed generator). - ### Notification (`app.bsky.notification`) - [x] Implement `app.bsky.notification.registerPush` (push notification registration, proxied). - ## Infrastructure & Core Components - [x] Sequencer (Event Log) - [x] Implement a `Sequencer` (backed by `repo_seq` table). @@ -247,22 +227,18 @@ These are implemented at PDS level to enable local-first reads (read-after-write - [x] Constant-time signature comparison. - [x] SSRF protection for outbound requests. - [x] Timing attack protection (dummy bcrypt on user-not-found prevents account enumeration). - ## Lewis' fabulous mini-list of remaining TODOs - [x] The OAuth authorize POST endpoint has no rate limiting, allowing password brute-forcing. Fix this and audit all oauth and 2fa surface again. - [x] DID resolution caching (valkey). - [x] Record schema validation (generic validation framework). - [x] Fix any remaining TODOs in the code. - ## Future: Web Management UI A single-page web app for account management. The frontend (JS framework) calls existing ATProto XRPC endpoints - no server-side rendering or bespoke HTML form handlers. - ### Architecture - [x] Static SPA served from PDS (or separate static host) - [ ] Frontend authenticates via OAuth 2.1 flow (same as any ATProto client) - [x] All operations use standard XRPC endpoints (existing + new PDS-specific ones below) - [x] No server-side sessions or CSRF - pure API client - ### PDS-Specific XRPC Endpoints (new) Absolutely subject to change, "bspds" isn't even the real name of this pds thus far :D Anyway... endpoints for PDS settings not covered by standard ATProto: @@ -272,47 +248,37 @@ Anyway... endpoints for PDS settings not covered by standard ATProto: - [ ] `com.bspds.account.verifyChannel` - initiate verification for Discord/Telegram/Signal - [ ] `com.bspds.account.confirmChannelVerification` - confirm with code - [ ] `com.bspds.admin.getServerStats` - user count, storage usage, etc. - ### Frontend Views Uses existing ATProto endpoints where possible: - Authentication - [x] Login page (uses `com.atproto.server.createSession`) - [x] Registration page (uses `com.atproto.server.createAccount`) - [x] Signup verification flow (uses `com.atproto.server.confirmSignup`, `resendVerification`) - [ ] Password reset flow (uses `com.atproto.server.requestPasswordReset`, `resetPassword`) - User Dashboard - [x] Account overview (uses `com.atproto.server.getSession`, `com.atproto.admin.getAccountInfo`) - [ ] Active sessions view (needs new endpoint or extend existing) - [x] App passwords (uses `com.atproto.server.listAppPasswords`, `createAppPassword`, `revokeAppPassword`) - [x] Invite codes (uses `com.atproto.server.getAccountInviteCodes`, `createInviteCode`) - Notification Preferences - [x] Channel selector (uses `com.bspds.account.*` endpoints above) - [ ] Verification flows for Discord/Telegram/Signal - [ ] Notification history view - Account Settings - [x] Email change (uses `com.atproto.server.requestEmailUpdate`, `updateEmail`) - [ ] Password change while logged in (needs new endpoint - change password with current password) - [x] Handle change (uses `com.atproto.identity.updateHandle`) - [x] Account deletion (uses `com.atproto.server.requestAccountDelete`, `deleteAccount`) - Data Management - [x] Repo browser (browse collections, view/create/delete records via `com.atproto.repo.*`) - [ ] Data export/download (CAR file download via `com.atproto.sync.getRepo`) - Admin Dashboard (privileged users only) - [ ] User list (uses `com.atproto.admin.getAccountInfos` with pagination) - [ ] User detail/actions (uses `com.atproto.admin.*` endpoints) - [ ] Invite management (uses `com.atproto.admin.getInviteCodes`, `disableInviteCodes`) - [ ] Server stats (uses `com.bspds.admin.getServerStats`) - ## Future: private data I will see where the discourse about encrypted/privileged private data is at the current moment, and make an implementation that matches what the bsky team will likely do in their pds whenever they get around to it. Then when they come out with theirs, I can make adjustments to mine and be ready on day 1. Or 2. - We want records that only authorized parties can see and decrypt. This requires some sort of federation of keys and communication between PDSes? Gotta figure all of this out as a first step. - diff --git a/deploy/nginx/nginx-quadlet.conf b/deploy/nginx/nginx-quadlet.conf index f6c289e..7a91421 100644 --- a/deploy/nginx/nginx-quadlet.conf +++ b/deploy/nginx/nginx-quadlet.conf @@ -1,49 +1,39 @@ worker_processes auto; error_log /var/log/nginx/error.log warn; - events { worker_connections 4096; } - http { include /etc/nginx/mime.types; default_type application/octet-stream; access_log /var/log/nginx/access.log; sendfile on; keepalive_timeout 65; - gzip on; gzip_types text/plain text/css application/json application/javascript text/xml application/xml; - ssl_protocols TLSv1.2 TLSv1.3; ssl_prefer_server_ciphers off; ssl_session_cache shared:SSL:10m; ssl_stapling on; ssl_stapling_verify on; - server { listen 80; listen [::]:80; server_name _; - location /.well-known/acme-challenge/ { root /var/www/acme; } - location / { return 301 https://$host$request_uri; } } - server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name _; - ssl_certificate /etc/nginx/certs/fullchain.pem; ssl_certificate_key /etc/nginx/certs/privkey.pem; client_max_body_size 100M; - location / { proxy_pass http://127.0.0.1:3000; proxy_http_version 1.1; diff --git a/deploy/quadlets/bspds-app.container b/deploy/quadlets/bspds-app.container index baae42e..33492d9 100644 --- a/deploy/quadlets/bspds-app.container +++ b/deploy/quadlets/bspds-app.container @@ -1,7 +1,6 @@ [Unit] Description=BSPDS AT Protocol PDS After=bspds-db.service bspds-minio.service bspds-valkey.service - [Container] ContainerName=bspds-app Image=localhost/bspds:latest @@ -19,10 +18,8 @@ HealthInterval=30s HealthTimeout=10s HealthRetries=3 HealthStartPeriod=15s - [Service] Restart=always RestartSec=10 - [Install] WantedBy=default.target diff --git a/deploy/quadlets/bspds-db.container b/deploy/quadlets/bspds-db.container index 08ed6e1..d94a21c 100644 --- a/deploy/quadlets/bspds-db.container +++ b/deploy/quadlets/bspds-db.container @@ -1,6 +1,5 @@ [Unit] Description=BSPDS postgres database - [Container] ContainerName=bspds-db Image=docker.io/library/postgres:18-alpine @@ -14,10 +13,8 @@ HealthInterval=10s HealthTimeout=5s HealthRetries=5 HealthStartPeriod=10s - [Service] Restart=always RestartSec=10 - [Install] WantedBy=default.target diff --git a/deploy/quadlets/bspds-minio.container b/deploy/quadlets/bspds-minio.container index 9eec6d7..530b955 100644 --- a/deploy/quadlets/bspds-minio.container +++ b/deploy/quadlets/bspds-minio.container @@ -1,6 +1,5 @@ [Unit] Description=BSPDS minio object storage - [Container] ContainerName=bspds-minio Image=docker.io/minio/minio:RELEASE.2025-10-15T17-29-55Z @@ -14,10 +13,8 @@ HealthInterval=30s HealthTimeout=10s HealthRetries=3 HealthStartPeriod=10s - [Service] Restart=always RestartSec=10 - [Install] WantedBy=default.target diff --git a/deploy/quadlets/bspds-nginx.container b/deploy/quadlets/bspds-nginx.container index 8fb04be..3178970 100644 --- a/deploy/quadlets/bspds-nginx.container +++ b/deploy/quadlets/bspds-nginx.container @@ -1,7 +1,6 @@ [Unit] Description=BSPDS nginx reverse proxy After=bspds-app.service - [Container] ContainerName=bspds-nginx Image=docker.io/library/nginx:1.28-alpine @@ -9,10 +8,8 @@ Pod=bspds.pod Volume=/srv/bspds/config/nginx.conf:/etc/nginx/nginx.conf:ro,Z Volume=/srv/bspds/certs:/etc/nginx/certs:ro,Z Volume=/srv/bspds/acme:/var/www/acme:ro,Z - [Service] Restart=always RestartSec=10 - [Install] WantedBy=default.target diff --git a/deploy/quadlets/bspds-valkey.container b/deploy/quadlets/bspds-valkey.container index c81a894..a22e15f 100644 --- a/deploy/quadlets/bspds-valkey.container +++ b/deploy/quadlets/bspds-valkey.container @@ -1,6 +1,5 @@ [Unit] Description=BSPDS valkey cache - [Container] ContainerName=bspds-valkey Image=docker.io/valkey/valkey:9-alpine @@ -12,10 +11,8 @@ HealthInterval=10s HealthTimeout=5s HealthRetries=3 HealthStartPeriod=5s - [Service] Restart=always RestartSec=10 - [Install] WantedBy=default.target diff --git a/deploy/quadlets/bspds.pod b/deploy/quadlets/bspds.pod index c45b5a1..2b66599 100644 --- a/deploy/quadlets/bspds.pod +++ b/deploy/quadlets/bspds.pod @@ -2,6 +2,5 @@ PodName=bspds PublishPort=80:80 PublishPort=443:443 - [Install] WantedBy=default.target diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 78b9eb0..db861b5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -43,7 +43,6 @@ services: memory: 1G reservations: memory: 256M - db: image: postgres:18-alpine restart: unless-stopped @@ -65,7 +64,6 @@ services: memory: 512M reservations: memory: 128M - minio: image: minio/minio:RELEASE.2025-10-15T17-29-55Z restart: unless-stopped @@ -87,7 +85,6 @@ services: memory: 512M reservations: memory: 128M - minio-init: image: minio/mc:RELEASE.2025-07-16T15-35-03Z depends_on: @@ -103,7 +100,6 @@ services: environment: MINIO_ROOT_USER: "${MINIO_ROOT_USER:-minioadmin}" MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}" - valkey: image: valkey/valkey:9-alpine restart: unless-stopped @@ -122,7 +118,6 @@ services: memory: 300M reservations: memory: 64M - nginx: image: nginx:1.28-alpine restart: unless-stopped @@ -140,14 +135,12 @@ services: interval: 30s timeout: 10s retries: 3 - certbot: image: certbot/certbot:v5.2.2 volumes: - ./certs:/etc/letsencrypt - acme_challenge:/var/www/acme entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew --webroot -w /var/www/acme; sleep 12h & wait $${!}; done'" - prometheus: image: prom/prometheus:v3.8.0 restart: unless-stopped @@ -164,7 +157,6 @@ services: resources: limits: memory: 256M - volumes: postgres_data: minio_data: diff --git a/docker-compose.yaml b/docker-compose.yaml index e76d0a0..4c9c77c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -16,7 +16,6 @@ services: - db - objsto - cache - db: image: postgres:18-alpine environment: @@ -27,7 +26,6 @@ services: - "5432:5432" volumes: - postgres_data:/var/lib/postgresql - objsto: image: minio/minio ports: @@ -39,14 +37,12 @@ services: volumes: - minio_data:/data command: server /data --console-address ":9001" - cache: image: valkey/valkey:8-alpine ports: - "6379:6379" volumes: - valkey_data:/data - prometheus: image: prom/prometheus:v3.8.0 ports: @@ -59,7 +55,6 @@ services: - '--storage.tsdb.path=/prometheus' depends_on: - app - volumes: postgres_data: minio_data: diff --git a/docs/install-alpine.md b/docs/install-alpine.md index f67ac23..0c49d2c 100644 --- a/docs/install-alpine.md +++ b/docs/install-alpine.md @@ -1,85 +1,60 @@ # BSPDS Production Installation on Alpine Linux - > **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified. - This guide covers installing BSPDS on Alpine Linux 3.23 (current stable as of December 2025). - ## Choose Your Installation Method - | Method | Best For | |--------|----------| | **Native (this guide)** | Maximum performance, minimal footprint, full control | | **[Containerized](install-containers.md)** | Easier updates, isolation, reproducible deployments | | **[Kubernetes](install-kubernetes.md)** | Multi-node, high availability, auto-scaling | - This guide covers native installation. For containerized deployment with podman and systemd quadlets, see the [container guide](install-containers.md). - --- - ## Prerequisites - - A VPS with at least 2GB RAM and 20GB disk - A domain name pointing to your server's IP - Root access - ## 1. System Setup - ```sh apk update && apk upgrade apk add curl git build-base openssl-dev pkgconf ``` - ## 2. Install Rust - ```sh apk add rustup rustup-init -y source ~/.cargo/env rustup default stable ``` - This installs the latest stable Rust (1.92+ as of December 2025). Alpine 3.23 also ships Rust 1.91 via `apk add rust cargo` if you prefer system packages. - ## 3. Install postgres - Alpine 3.23 includes PostgreSQL 18: - ```sh apk add postgresql postgresql-contrib - rc-update add postgresql /etc/init.d/postgresql setup rc-service postgresql start - psql -U postgres -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';" psql -U postgres -c "CREATE DATABASE pds OWNER bspds;" psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;" ``` - ## 4. Install minio - ```sh curl -O https://dl.min.io/server/minio/release/linux-amd64/minio chmod +x minio mv minio /usr/local/bin/ - mkdir -p /var/lib/minio/data adduser -D -H -s /sbin/nologin minio-user chown -R minio-user:minio-user /var/lib/minio - cat > /etc/conf.d/minio << 'EOF' MINIO_ROOT_USER="minioadmin" MINIO_ROOT_PASSWORD="your-minio-password" MINIO_VOLUMES="/var/lib/minio/data" MINIO_OPTS="--console-address :9001" EOF - cat > /etc/init.d/minio << 'EOF' #!/sbin/openrc-run - name="minio" description="MinIO Object Storage" - command="/usr/local/bin/minio" command_args="server ${MINIO_VOLUMES} ${MINIO_OPTS}" command_user="minio-user" @@ -87,116 +62,85 @@ command_background=true pidfile="/run/${RC_SVCNAME}.pid" output_log="/var/log/minio.log" error_log="/var/log/minio.log" - depend() { need net } - start_pre() { . /etc/conf.d/minio export MINIO_ROOT_USER MINIO_ROOT_PASSWORD } EOF - chmod +x /etc/init.d/minio rc-update add minio rc-service minio start ``` - Create the blob bucket (wait a few seconds for minio to start): - ```sh curl -O https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc mv mc /usr/local/bin/ - mc alias set local http://localhost:9000 minioadmin your-minio-password mc mb local/pds-blobs ``` - ## 5. Install valkey - Alpine 3.23 includes Valkey 9: - ```sh apk add valkey - rc-update add valkey rc-service valkey start ``` - ## 6. Install deno (for frontend build) - ```sh curl -fsSL https://deno.land/install.sh | sh export PATH="$HOME/.deno/bin:$PATH" echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.profile ``` - ## 7. Clone and Build BSPDS - ```sh mkdir -p /opt && cd /opt git clone https://tangled.org/lewis.moe/bspds-sandbox bspds cd bspds - cd frontend deno task build cd .. - cargo build --release ``` - ## 8. Install sqlx-cli and Run Migrations - ```sh cargo install sqlx-cli --no-default-features --features postgres - export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run ``` - ## 9. Configure BSPDS - ```sh mkdir -p /etc/bspds cp /opt/bspds/.env.example /etc/bspds/bspds.env chmod 600 /etc/bspds/bspds.env ``` - Edit `/etc/bspds/bspds.env` and fill in your values. Generate secrets with: - ```sh openssl rand -base64 48 ``` - ## 10. Create OpenRC Service - ```sh adduser -D -H -s /sbin/nologin bspds - cp /opt/bspds/target/release/bspds /usr/local/bin/ mkdir -p /var/lib/bspds cp -r /opt/bspds/frontend/dist /var/lib/bspds/frontend chown -R bspds:bspds /var/lib/bspds - cat > /etc/init.d/bspds << 'EOF' #!/sbin/openrc-run - name="bspds" description="BSPDS - AT Protocol PDS" - command="/usr/local/bin/bspds" command_user="bspds" command_background=true pidfile="/run/${RC_SVCNAME}.pid" output_log="/var/log/bspds.log" error_log="/var/log/bspds.log" - depend() { need net postgresql minio } - start_pre() { export FRONTEND_DIR=/var/lib/bspds/frontend . /etc/bspds/bspds.env @@ -205,25 +149,19 @@ start_pre() { export VALKEY_URL JWT_SECRET DPOP_SECRET MASTER_KEY APPVIEW_URL CRAWLERS } EOF - chmod +x /etc/init.d/bspds rc-update add bspds rc-service bspds start ``` - ## 11. Install and Configure nginx - Alpine 3.23 includes nginx 1.28: - ```sh apk add nginx certbot certbot-nginx - cat > /etc/nginx/http.d/bspds.conf << 'EOF' server { listen 80; listen [::]:80; server_name pds.example.com; - location / { proxy_pass http://127.0.0.1:3000; proxy_http_version 1.1; @@ -237,63 +175,48 @@ server { } } EOF - rc-update add nginx rc-service nginx start ``` - ## 12. Obtain SSL Certificate - ```sh certbot --nginx -d pds.example.com ``` - Set up auto-renewal: - ```sh echo "0 0 * * * certbot renew --quiet" | crontab - ``` - ## 13. Configure Firewall - ```sh apk add iptables ip6tables - iptables -A INPUT -p tcp --dport 22 -j ACCEPT iptables -A INPUT -p tcp --dport 80 -j ACCEPT iptables -A INPUT -p tcp --dport 443 -j ACCEPT iptables -A INPUT -i lo -j ACCEPT iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT iptables -P INPUT DROP - ip6tables -A INPUT -p tcp --dport 22 -j ACCEPT ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT ip6tables -A INPUT -i lo -j ACCEPT ip6tables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT ip6tables -P INPUT DROP - rc-update add iptables rc-update add ip6tables /etc/init.d/iptables save /etc/init.d/ip6tables save ``` - ## 14. Verify Installation - ```sh rc-service bspds status curl -s https://pds.example.com/xrpc/_health curl -s https://pds.example.com/.well-known/atproto-did ``` - ## Maintenance - View logs: ```sh tail -f /var/log/bspds.log ``` - Update BSPDS: ```sh cd /opt/bspds @@ -306,7 +229,6 @@ cp -r frontend/dist /var/lib/bspds/frontend DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run rc-service bspds start ``` - Backup database: ```sh pg_dump -U postgres pds > /var/backups/pds-$(date +%Y%m%d).sql diff --git a/docs/install-containers.md b/docs/install-containers.md index 74ee45a..d61a7fb 100644 --- a/docs/install-containers.md +++ b/docs/install-containers.md @@ -1,164 +1,113 @@ # BSPDS Containerized Production Deployment - > **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified. - This guide covers deploying BSPDS using containers with podman. - - **Debian 13+**: Uses systemd quadlets (modern, declarative container management) - **Alpine 3.23+**: Uses OpenRC service script with podman-compose - ## Prerequisites - - A VPS with at least 2GB RAM and 20GB disk - A domain name pointing to your server's IP - Root or sudo access - ## Quick Start (Docker/Podman Compose) - If you just want to get running quickly: - ```sh cp .env.example .env - # Edit .env with your values # Generate secrets: openssl rand -base64 48 - # Build and start podman-compose -f docker-compose.prod.yml up -d - # Get initial certificate (after DNS is configured) podman-compose -f docker-compose.prod.yml run --rm certbot certonly \ --webroot -w /var/www/acme -d pds.example.com - # Restart nginx to load certificate podman-compose -f docker-compose.prod.yml restart nginx ``` - For production setups with proper service management, continue to either the Debian or Alpine section below. - --- - # Debian 13+ with Systemd Quadlets - Quadlets are the modern way to run podman containers under systemd. - ## 1. Install Podman - ```bash apt update apt install -y podman ``` - ## 2. Create Directory Structure - ```bash mkdir -p /etc/containers/systemd mkdir -p /srv/bspds/{postgres,minio,valkey,certs,acme,config} ``` - ## 3. Create Environment File - ```bash cp /opt/bspds/.env.example /srv/bspds/config/bspds.env chmod 600 /srv/bspds/config/bspds.env ``` - Edit `/srv/bspds/config/bspds.env` and fill in your values. Generate secrets with: - ```bash openssl rand -base64 48 ``` - For quadlets, also add `DATABASE_URL` with the full connection string (systemd doesn't support variable expansion). - ## 4. Install Quadlet Definitions - Copy the quadlet files from the repository: - ```bash cp /opt/bspds/deploy/quadlets/*.pod /etc/containers/systemd/ cp /opt/bspds/deploy/quadlets/*.container /etc/containers/systemd/ ``` - Note: Systemd doesn't support shell-style variable expansion in `Environment=` lines. The quadlet files expect DATABASE_URL to be set in the environment file. - ## 5. Create nginx Configuration - ```bash cp /opt/bspds/deploy/nginx/nginx-quadlet.conf /srv/bspds/config/nginx.conf ``` - ## 6. Build BSPDS Image - ```bash cd /opt git clone https://tangled.org/lewis.moe/bspds-sandbox bspds cd bspds podman build -t bspds:latest . ``` - ## 7. Create Podman Secrets - ```bash source /srv/bspds/config/bspds.env echo "$DB_PASSWORD" | podman secret create bspds-db-password - echo "$MINIO_ROOT_PASSWORD" | podman secret create bspds-minio-password - ``` - ## 8. Start Services and Initialize - ```bash systemctl daemon-reload systemctl start bspds-db bspds-minio bspds-valkey - sleep 10 - # Create MinIO bucket podman run --rm --pod bspds \ -e MINIO_ROOT_USER=minioadmin \ -e MINIO_ROOT_PASSWORD=your-minio-password \ docker.io/minio/mc:RELEASE.2025-07-16T15-35-03Z \ sh -c "mc alias set local http://localhost:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD && mc mb --ignore-existing local/pds-blobs" - # Run migrations cargo install sqlx-cli --no-default-features --features postgres DATABASE_URL="postgres://bspds:your-db-password@localhost:5432/pds" sqlx migrate run --source /opt/bspds/migrations ``` - ## 9. Obtain SSL Certificate - Create temporary self-signed cert: - ```bash openssl req -x509 -nodes -days 1 -newkey rsa:2048 \ -keyout /srv/bspds/certs/privkey.pem \ -out /srv/bspds/certs/fullchain.pem \ -subj "/CN=pds.example.com" - systemctl start bspds-app bspds-nginx - # Get real certificate podman run --rm \ -v /srv/bspds/certs:/etc/letsencrypt:Z \ -v /srv/bspds/acme:/var/www/acme:Z \ docker.io/certbot/certbot:v5.2.2 certonly \ --webroot -w /var/www/acme -d pds.example.com --agree-tos --email you@example.com - # Link certificates ln -sf /srv/bspds/certs/live/pds.example.com/fullchain.pem /srv/bspds/certs/fullchain.pem ln -sf /srv/bspds/certs/live/pds.example.com/privkey.pem /srv/bspds/certs/privkey.pem - systemctl restart bspds-nginx ``` - ## 10. Enable All Services - ```bash systemctl enable bspds-db bspds-minio bspds-valkey bspds-app bspds-nginx ``` - ## 11. Configure Firewall - ```bash apt install -y ufw ufw allow ssh @@ -166,109 +115,78 @@ ufw allow 80/tcp ufw allow 443/tcp ufw enable ``` - ## 12. Certificate Renewal - Add to root's crontab (`crontab -e`): - ``` 0 0 * * * podman run --rm -v /srv/bspds/certs:/etc/letsencrypt:Z -v /srv/bspds/acme:/var/www/acme:Z docker.io/certbot/certbot:v5.2.2 renew --quiet && systemctl reload bspds-nginx ``` - --- - # Alpine 3.23+ with OpenRC - Alpine uses OpenRC, not systemd. We'll use podman-compose with an OpenRC service wrapper. - ## 1. Install Podman - ```sh apk update apk add podman podman-compose fuse-overlayfs cni-plugins rc-update add cgroups rc-service cgroups start ``` - Enable podman socket for compose: - ```sh rc-update add podman rc-service podman start ``` - ## 2. Create Directory Structure - ```sh mkdir -p /srv/bspds/{data,config} mkdir -p /srv/bspds/data/{postgres,minio,valkey,certs,acme} ``` - ## 3. Clone Repository and Build - ```sh cd /opt git clone https://tangled.org/lewis.moe/bspds-sandbox bspds cd bspds podman build -t bspds:latest . ``` - ## 4. Create Environment File - ```sh cp /opt/bspds/.env.example /srv/bspds/config/bspds.env chmod 600 /srv/bspds/config/bspds.env ``` - Edit `/srv/bspds/config/bspds.env` and fill in your values. Generate secrets with: - ```sh openssl rand -base64 48 ``` - ## 5. Set Up Compose and nginx - Copy the production compose and nginx configs: - ```sh cp /opt/bspds/docker-compose.prod.yml /srv/bspds/docker-compose.yml cp /opt/bspds/nginx.prod.conf /srv/bspds/config/nginx.conf ``` - Edit `/srv/bspds/docker-compose.yml` to adjust paths if needed: - Update volume mounts to use `/srv/bspds/data/` paths - Update nginx cert paths to match `/srv/bspds/data/certs/` - Edit `/srv/bspds/config/nginx.conf` to update cert paths: - Change `/etc/nginx/certs/live/${PDS_HOSTNAME}/` to `/etc/nginx/certs/` - ## 6. Create OpenRC Service - ```sh cat > /etc/init.d/bspds << 'EOF' #!/sbin/openrc-run - name="bspds" description="BSPDS AT Protocol PDS (containerized)" - command="/usr/bin/podman-compose" command_args="-f /srv/bspds/docker-compose.yml up" command_background=true pidfile="/run/${RC_SVCNAME}.pid" - directory="/srv/bspds" - depend() { need net podman after firewall } - start_pre() { set -a . /srv/bspds/config/bspds.env set +a } - stop() { ebegin "Stopping ${name}" cd /srv/bspds @@ -279,18 +197,13 @@ stop() { eend $? } EOF - chmod +x /etc/init.d/bspds ``` - ## 7. Initialize Services - ```sh # Start services rc-service bspds start - sleep 15 - # Create MinIO bucket source /srv/bspds/config/bspds.env podman run --rm --network bspds_default \ @@ -298,30 +211,23 @@ podman run --rm --network bspds_default \ -e MINIO_ROOT_PASSWORD="$MINIO_ROOT_PASSWORD" \ docker.io/minio/mc:RELEASE.2025-07-16T15-35-03Z \ sh -c 'mc alias set local http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD && mc mb --ignore-existing local/pds-blobs' - # Run migrations apk add rustup rustup-init -y source ~/.cargo/env cargo install sqlx-cli --no-default-features --features postgres - # Get database container IP DB_IP=$(podman inspect bspds-db-1 --format '{{.NetworkSettings.Networks.bspds_default.IPAddress}}') DATABASE_URL="postgres://bspds:$DB_PASSWORD@$DB_IP:5432/pds" sqlx migrate run --source /opt/bspds/migrations ``` - ## 8. Obtain SSL Certificate - Create temporary self-signed cert: - ```sh openssl req -x509 -nodes -days 1 -newkey rsa:2048 \ -keyout /srv/bspds/data/certs/privkey.pem \ -out /srv/bspds/data/certs/fullchain.pem \ -subj "/CN=pds.example.com" - rc-service bspds restart - # Get real certificate podman run --rm \ -v /srv/bspds/data/certs:/etc/letsencrypt \ @@ -329,99 +235,73 @@ podman run --rm \ --network bspds_default \ docker.io/certbot/certbot:v5.2.2 certonly \ --webroot -w /var/www/acme -d pds.example.com --agree-tos --email you@example.com - # Link certificates ln -sf /srv/bspds/data/certs/live/pds.example.com/fullchain.pem /srv/bspds/data/certs/fullchain.pem ln -sf /srv/bspds/data/certs/live/pds.example.com/privkey.pem /srv/bspds/data/certs/privkey.pem - rc-service bspds restart ``` - ## 9. Enable Service at Boot - ```sh rc-update add bspds ``` - ## 10. Configure Firewall - ```sh apk add iptables ip6tables - iptables -A INPUT -p tcp --dport 22 -j ACCEPT iptables -A INPUT -p tcp --dport 80 -j ACCEPT iptables -A INPUT -p tcp --dport 443 -j ACCEPT iptables -A INPUT -i lo -j ACCEPT iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT iptables -P INPUT DROP - ip6tables -A INPUT -p tcp --dport 22 -j ACCEPT ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT ip6tables -A INPUT -i lo -j ACCEPT ip6tables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT ip6tables -P INPUT DROP - rc-update add iptables rc-update add ip6tables /etc/init.d/iptables save /etc/init.d/ip6tables save ``` - ## 11. Certificate Renewal - Add to root's crontab (`crontab -e`): - ``` 0 0 * * * podman run --rm -v /srv/bspds/data/certs:/etc/letsencrypt -v /srv/bspds/data/acme:/var/www/acme docker.io/certbot/certbot:v5.2.2 renew --quiet && rc-service bspds restart ``` - --- - # Verification and Maintenance - ## Verify Installation - ```sh curl -s https://pds.example.com/xrpc/_health | jq curl -s https://pds.example.com/.well-known/atproto-did ``` - ## View Logs - **Debian:** ```bash journalctl -u bspds-app -f podman logs -f bspds-app ``` - **Alpine:** ```sh podman-compose -f /srv/bspds/docker-compose.yml logs -f podman logs -f bspds-bspds-1 ``` - ## Update BSPDS - ```sh cd /opt/bspds git pull podman build -t bspds:latest . - # Debian: systemctl restart bspds-app - # Alpine: rc-service bspds restart ``` - ## Backup Database - **Debian:** ```bash podman exec bspds-db pg_dump -U bspds pds > /var/backups/pds-$(date +%Y%m%d).sql ``` - **Alpine:** ```sh podman exec bspds-db-1 pg_dump -U bspds pds > /var/backups/pds-$(date +%Y%m%d).sql diff --git a/docs/install-debian.md b/docs/install-debian.md index 4fe6249..c5c79de 100644 --- a/docs/install-debian.md +++ b/docs/install-debian.md @@ -1,82 +1,58 @@ # BSPDS Production Installation on Debian - > **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified. - This guide covers installing BSPDS on Debian 13 "Trixie" (current stable as of December 2025). - ## Choose Your Installation Method - | Method | Best For | |--------|----------| | **Native (this guide)** | Maximum performance, full control, simpler debugging | | **[Containerized](install-containers.md)** | Easier updates, isolation, reproducible deployments | | **[Kubernetes](install-kubernetes.md)** | Multi-node, high availability, auto-scaling | - This guide covers native installation. For containerized deployment with podman and systemd quadlets, see the [container guide](install-containers.md). - --- - ## Prerequisites - - A VPS with at least 2GB RAM and 20GB disk - A domain name pointing to your server's IP - Root or sudo access - ## 1. System Setup - ```bash apt update && apt upgrade -y apt install -y curl git build-essential pkg-config libssl-dev ``` - ## 2. Install Rust - ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source ~/.cargo/env rustup default stable ``` - This installs the latest stable Rust (1.92+ as of December 2025). - ## 3. Install postgres - Debian 13 includes PostgreSQL 17: - ```bash apt install -y postgresql postgresql-contrib - systemctl enable postgresql systemctl start postgresql - sudo -u postgres psql -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';" sudo -u postgres psql -c "CREATE DATABASE pds OWNER bspds;" sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;" ``` - ## 4. Install minio - ```bash curl -O https://dl.min.io/server/minio/release/linux-amd64/minio chmod +x minio mv minio /usr/local/bin/ - mkdir -p /var/lib/minio/data useradd -r -s /sbin/nologin minio-user chown -R minio-user:minio-user /var/lib/minio - cat > /etc/default/minio << 'EOF' MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=your-minio-password MINIO_VOLUMES="/var/lib/minio/data" MINIO_OPTS="--console-address :9001" EOF - cat > /etc/systemd/system/minio.service << 'EOF' [Unit] Description=MinIO Object Storage After=network.target - [Service] User=minio-user Group=minio-user @@ -84,98 +60,71 @@ EnvironmentFile=/etc/default/minio ExecStart=/usr/local/bin/minio server $MINIO_VOLUMES $MINIO_OPTS Restart=always LimitNOFILE=65536 - [Install] WantedBy=multi-user.target EOF - systemctl daemon-reload systemctl enable minio systemctl start minio ``` - Create the blob bucket (wait a few seconds for minio to start): - ```bash curl -O https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc mv mc /usr/local/bin/ - mc alias set local http://localhost:9000 minioadmin your-minio-password mc mb local/pds-blobs ``` - ## 5. Install valkey - Debian 13 includes Valkey 8: - ```bash apt install -y valkey - systemctl enable valkey-server systemctl start valkey-server ``` - ## 6. Install deno (for frontend build) - ```bash curl -fsSL https://deno.land/install.sh | sh export PATH="$HOME/.deno/bin:$PATH" echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.bashrc ``` - ## 7. Clone and Build BSPDS - ```bash cd /opt git clone https://tangled.org/lewis.moe/bspds-sandbox bspds cd bspds - cd frontend deno task build cd .. - cargo build --release ``` - ## 8. Install sqlx-cli and Run Migrations - ```bash cargo install sqlx-cli --no-default-features --features postgres - export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run ``` - ## 9. Configure BSPDS - ```bash mkdir -p /etc/bspds cp /opt/bspds/.env.example /etc/bspds/bspds.env chmod 600 /etc/bspds/bspds.env ``` - Edit `/etc/bspds/bspds.env` and fill in your values. Generate secrets with: - ```bash openssl rand -base64 48 ``` - ## 10. Create Systemd Service - ```bash useradd -r -s /sbin/nologin bspds - cp /opt/bspds/target/release/bspds /usr/local/bin/ mkdir -p /var/lib/bspds cp -r /opt/bspds/frontend/dist /var/lib/bspds/frontend chown -R bspds:bspds /var/lib/bspds - cat > /etc/systemd/system/bspds.service << 'EOF' [Unit] Description=BSPDS - AT Protocol PDS After=network.target postgresql.service minio.service - [Service] Type=simple User=bspds @@ -185,29 +134,22 @@ Environment=FRONTEND_DIR=/var/lib/bspds/frontend ExecStart=/usr/local/bin/bspds Restart=always RestartSec=5 - [Install] WantedBy=multi-user.target EOF - systemctl daemon-reload systemctl enable bspds systemctl start bspds ``` - ## 11. Install and Configure nginx - Debian 13 includes nginx 1.26: - ```bash apt install -y nginx certbot python3-certbot-nginx - cat > /etc/nginx/sites-available/bspds << 'EOF' server { listen 80; listen [::]:80; server_name pds.example.com; - location / { proxy_pass http://127.0.0.1:3000; proxy_http_version 1.1; @@ -221,23 +163,17 @@ server { } } EOF - ln -s /etc/nginx/sites-available/bspds /etc/nginx/sites-enabled/ rm -f /etc/nginx/sites-enabled/default nginx -t systemctl reload nginx ``` - ## 12. Obtain SSL Certificate - ```bash certbot --nginx -d pds.example.com ``` - Certbot automatically configures nginx for HTTP/2 and sets up auto-renewal. - ## 13. Configure Firewall - ```bash apt install -y ufw ufw allow ssh @@ -245,22 +181,17 @@ ufw allow 80/tcp ufw allow 443/tcp ufw enable ``` - ## 14. Verify Installation - ```bash systemctl status bspds curl -s https://pds.example.com/xrpc/_health | jq curl -s https://pds.example.com/.well-known/atproto-did ``` - ## Maintenance - View logs: ```bash journalctl -u bspds -f ``` - Update BSPDS: ```bash cd /opt/bspds @@ -273,7 +204,6 @@ cp -r frontend/dist /var/lib/bspds/frontend DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run systemctl start bspds ``` - Backup database: ```bash sudo -u postgres pg_dump pds > /var/backups/pds-$(date +%Y%m%d).sql diff --git a/docs/install-kubernetes.md b/docs/install-kubernetes.md index 5e291fb..d312741 100644 --- a/docs/install-kubernetes.md +++ b/docs/install-kubernetes.md @@ -1,11 +1,7 @@ # BSPDS Production Kubernetes Deployment - > **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified. - This guide covers deploying BSPDS on a production multi-node Kubernetes cluster with high availability, auto-scaling, and proper secrets management. - ## Architecture Overview - ``` ┌─────────────────────────────────────────────────┐ │ Kubernetes Cluster │ @@ -30,20 +26,15 @@ This guide covers deploying BSPDS on a production multi-node Kubernetes cluster │ └──────────────────────────────────────┘ │ └─────────────────────────────────────────────────┘ ``` - ## Prerequisites - - Kubernetes cluster (1.30+) with at least 3 nodes (1.34 is current stable) - `kubectl` configured to access your cluster - `helm` 3.x installed - Storage class that supports `ReadWriteOnce` (for databases) - Ingress controller installed (nginx-ingress or traefik) - cert-manager installed for TLS certificates - ### Quick Prerequisites Setup - If you need to install prerequisites: - ```bash # Install nginx-ingress (chart v4.14.1 - December 2025) helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx @@ -51,7 +42,6 @@ helm repo update helm install ingress-nginx ingress-nginx/ingress-nginx \ --namespace ingress-nginx --create-namespace \ --version 4.14.1 - # Install cert-manager (v1.19.2 - December 2025) helm repo add jetstack https://charts.jetstack.io helm repo update @@ -60,20 +50,14 @@ helm install cert-manager jetstack/cert-manager \ --version v1.19.2 \ --set installCRDs=true ``` - --- - ## 1. Create Namespace - ```bash kubectl create namespace bspds kubectl config set-context --current --namespace=bspds ``` - ## 2. Create Secrets - Generate secure passwords and secrets: - ```bash # Generate secrets DB_PASSWORD=$(openssl rand -base64 32) @@ -81,21 +65,17 @@ MINIO_PASSWORD=$(openssl rand -base64 32) JWT_SECRET=$(openssl rand -base64 48) DPOP_SECRET=$(openssl rand -base64 48) MASTER_KEY=$(openssl rand -base64 48) - # Create Kubernetes secrets kubectl create secret generic bspds-db-credentials \ --from-literal=username=bspds \ --from-literal=password="$DB_PASSWORD" - kubectl create secret generic bspds-minio-credentials \ --from-literal=root-user=minioadmin \ --from-literal=root-password="$MINIO_PASSWORD" - kubectl create secret generic bspds-secrets \ --from-literal=jwt-secret="$JWT_SECRET" \ --from-literal=dpop-secret="$DPOP_SECRET" \ --from-literal=master-key="$MASTER_KEY" - # Save secrets locally (KEEP SECURE!) echo "DB_PASSWORD=$DB_PASSWORD" > secrets.txt echo "MINIO_PASSWORD=$MINIO_PASSWORD" >> secrets.txt @@ -104,21 +84,16 @@ echo "DPOP_SECRET=$DPOP_SECRET" >> secrets.txt echo "MASTER_KEY=$MASTER_KEY" >> secrets.txt chmod 600 secrets.txt ``` - ## 3. Deploy PostgreSQL - ### Option A: CloudNativePG Operator (Recommended for HA) - ```bash # Install CloudNativePG operator (v1.28.0 - December 2025) kubectl apply --server-side -f \ https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.28/releases/cnpg-1.28.0.yaml - # Wait for operator kubectl wait --for=condition=available --timeout=120s \ deployment/cnpg-controller-manager -n cnpg-system ``` - ```bash cat < **Note**: If your BSPDS image doesn't have a `--migrate-only` flag, you can skip this step. The app will run migrations on first startup. Alternatively, build a separate migration image with `sqlx-cli` installed. - ## 8. Deploy BSPDS Application - ```bash cat < backup-$(date +%Y%m%d).sql ``` - ### Run Migrations - If you have a migration Job defined, you can re-run it: - ```bash # Delete old job first (if exists) kubectl delete job bspds-migrate -n bspds --ignore-not-found - # Re-apply the migration job from step 7 # Or simply restart the deployment - BSPDS runs migrations on startup kubectl rollout restart deployment/bspds -n bspds ``` - --- - ## Troubleshooting - ### Pod Won't Start - ```bash kubectl describe pod -l app=bspds -n bspds kubectl logs -l app=bspds -n bspds --previous ``` - ### Database Connection Issues - ```bash # Test connectivity from a debug pod kubectl run debug --rm -it --restart=Never --image=postgres:18-alpine -- \ psql "postgres://bspds:PASSWORD@bspds-db-rw:5432/pds" -c "SELECT 1" ``` - ### Certificate Issues - ```bash kubectl describe certificate bspds-tls -n bspds kubectl describe certificaterequest -n bspds kubectl logs -l app.kubernetes.io/name=cert-manager -n cert-manager ``` - ### View Resource Usage - ```bash kubectl top pods -n bspds kubectl top nodes diff --git a/docs/install-openbsd.md b/docs/install-openbsd.md index 2053f97..0518ce0 100644 --- a/docs/install-openbsd.md +++ b/docs/install-openbsd.md @@ -1,196 +1,136 @@ # BSPDS Production Installation on OpenBSD - > **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified. - This guide covers installing BSPDS on OpenBSD 7.8 (current release as of December 2025). - ## Prerequisites - - A VPS with at least 2GB RAM and 20GB disk - A domain name pointing to your server's IP - Root access (or doas configured) - ## Why nginx over relayd? - OpenBSD's native `relayd` supports WebSockets but does **not** support HTTP/2. For a modern PDS deployment, we recommend nginx which provides HTTP/2, WebSocket support, and automatic OCSP stapling. - ## 1. System Setup - ```sh pkg_add curl git ``` - ## 2. Install Rust - ```sh pkg_add rust ``` - OpenBSD 7.8 ships Rust 1.82+. For the latest stable (1.92+), use rustup: - ```sh pkg_add rustup rustup-init -y source ~/.cargo/env rustup default stable ``` - ## 3. Install postgres - OpenBSD 7.8 includes PostgreSQL 17 (PostgreSQL 18 may not yet be in ports): - ```sh pkg_add postgresql-server postgresql-client - mkdir -p /var/postgresql/data chown _postgresql:_postgresql /var/postgresql/data su - _postgresql -c "initdb -D /var/postgresql/data -U postgres -A scram-sha-256" - rcctl enable postgresql rcctl start postgresql - psql -U postgres -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';" psql -U postgres -c "CREATE DATABASE pds OWNER bspds;" psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;" ``` - ## 4. Install minio - OpenBSD doesn't have a minio package. Options: - **Option A: Use an external S3-compatible service (recommended for production)** - aws s3, backblaze b2, or upcloud managed object storage. Skip to step 5 and configure the S3 credentials in step 9. - **Option B: Build minio from source** - ```sh pkg_add go - mkdir -p /tmp/minio-build && cd /tmp/minio-build ftp -o minio.tar.gz https://github.com/minio/minio/archive/refs/tags/RELEASE.2025-10-15T17-29-55Z.tar.gz tar xzf minio.tar.gz cd minio-* go build -o minio . cp minio /usr/local/bin/ - mkdir -p /var/minio/data useradd -d /var/minio -s /sbin/nologin _minio chown -R _minio:_minio /var/minio - cat > /etc/minio.conf << 'EOF' MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=your-minio-password EOF chmod 600 /etc/minio.conf - cat > /etc/rc.d/minio << 'EOF' #!/bin/ksh - daemon="/usr/local/bin/minio" daemon_user="_minio" daemon_flags="server /var/minio/data --console-address :9001" - . /etc/rc.d/rc.subr - rc_pre() { . /etc/minio.conf export MINIO_ROOT_USER MINIO_ROOT_PASSWORD } - rc_cmd $1 EOF - chmod +x /etc/rc.d/minio rcctl enable minio rcctl start minio ``` - Create the blob bucket: - ```sh ftp -o /usr/local/bin/mc https://dl.min.io/client/mc/release/openbsd-amd64/mc chmod +x /usr/local/bin/mc - mc alias set local http://localhost:9000 minioadmin your-minio-password mc mb local/pds-blobs ``` - ## 5. Install redis - OpenBSD has redis in ports (valkey may not be available yet): - ```sh pkg_add redis - rcctl enable redis rcctl start redis ``` - ## 6. Install deno (for frontend build) - ```sh curl -fsSL https://deno.land/install.sh | sh export PATH="$HOME/.deno/bin:$PATH" echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.profile ``` - ## 7. Clone and Build BSPDS - ```sh mkdir -p /opt && cd /opt git clone https://tangled.org/lewis.moe/bspds-sandbox bspds cd bspds - cd frontend deno task build cd .. - cargo build --release ``` - ## 8. Install sqlx-cli and Run Migrations - ```sh cargo install sqlx-cli --no-default-features --features postgres - export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run ``` - ## 9. Configure BSPDS - ```sh mkdir -p /etc/bspds cp /opt/bspds/.env.example /etc/bspds/bspds.conf chmod 600 /etc/bspds/bspds.conf ``` - Edit `/etc/bspds/bspds.conf` and fill in your values. Generate secrets with: - ```sh openssl rand -base64 48 ``` - ## 10. Create rc.d Service - ```sh useradd -d /var/empty -s /sbin/nologin _bspds - cp /opt/bspds/target/release/bspds /usr/local/bin/ mkdir -p /var/bspds cp -r /opt/bspds/frontend/dist /var/bspds/frontend chown -R _bspds:_bspds /var/bspds - cat > /etc/rc.d/bspds << 'EOF' #!/bin/ksh - daemon="/usr/local/bin/bspds" daemon_user="_bspds" daemon_logger="daemon.info" - . /etc/rc.d/rc.subr - rc_pre() { export FRONTEND_DIR=/var/bspds/frontend while IFS='=' read -r key value; do @@ -200,56 +140,43 @@ rc_pre() { export "$key=$value" done < /etc/bspds/bspds.conf } - rc_cmd $1 EOF - chmod +x /etc/rc.d/bspds rcctl enable bspds rcctl start bspds ``` - ## 11. Install and Configure nginx - ```sh pkg_add nginx - cat > /etc/nginx/nginx.conf << 'EOF' worker_processes 1; - events { worker_connections 1024; } - http { include mime.types; - server { listen 80; listen [::]:80; server_name pds.example.com; - location /.well-known/acme-challenge/ { root /var/www/acme; } - location / { return 301 https://$host$request_uri; } } - server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name pds.example.com; - ssl_certificate /etc/ssl/pds.example.com.fullchain.pem; ssl_certificate_key /etc/ssl/private/pds.example.com.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:10m; - location / { proxy_pass http://127.0.0.1:3000; proxy_http_version 1.1; @@ -264,77 +191,55 @@ http { } } EOF - mkdir -p /var/www/acme rcctl enable nginx ``` - ## 12. Obtain SSL Certificate with acme-client - OpenBSD's native acme-client works well: - ```sh cat >> /etc/acme-client.conf << 'EOF' - authority letsencrypt { api url "https://acme-v02.api.letsencrypt.org/directory" account key "/etc/acme/letsencrypt-privkey.pem" } - domain pds.example.com { domain key "/etc/ssl/private/pds.example.com.key" domain full chain certificate "/etc/ssl/pds.example.com.fullchain.pem" sign with letsencrypt } EOF - mkdir -p /etc/acme - rcctl start nginx - acme-client -v pds.example.com - rcctl restart nginx ``` - Set up auto-renewal in root's crontab: - ```sh crontab -e ``` - Add: ``` 0 0 * * * acme-client pds.example.com && rcctl reload nginx ``` - ## 13. Configure Packet Filter (pf) - ```sh cat >> /etc/pf.conf << 'EOF' - # BSPDS rules pass in on egress proto tcp from any to any port { 22, 80, 443 } EOF - pfctl -f /etc/pf.conf ``` - ## 14. Verify Installation - ```sh rcctl check bspds ftp -o - https://pds.example.com/xrpc/_health ftp -o - https://pds.example.com/.well-known/atproto-did ``` - ## Maintenance - View logs: ```sh tail -f /var/log/daemon ``` - Update BSPDS: ```sh cd /opt/bspds @@ -347,7 +252,6 @@ cp -r frontend/dist /var/bspds/frontend DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run rcctl start bspds ``` - Backup database: ```sh pg_dump -U postgres pds > /var/backups/pds-$(date +%Y%m%d).sql diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 2691ebb..200871e 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -9,13 +9,10 @@ import Settings from './routes/Settings.svelte' import Notifications from './routes/Notifications.svelte' import RepoExplorer from './routes/RepoExplorer.svelte' - const auth = getAuthState() - $effect(() => { initAuth() }) - function getComponent(path: string) { switch (path) { case '/login': @@ -38,11 +35,9 @@ return auth.session ? Dashboard : Login } } - let currentPath = $derived(getCurrentPath()) let CurrentComponent = $derived(getComponent(currentPath)) -
{#if auth.loading}
@@ -52,7 +47,6 @@ {/if}
-