From bf0a57cd47d210061dd45abefe5cb78cd3e49027 Mon Sep 17 00:00:00 2001
From: lewis <lu5a@proton.me>
Date: Sat, 13 Dec 2025 17:03:14 +0200
Subject: [PATCH] Some theoretical installation instructions, for me to verify

---
 README.md                              |  23 +
 deploy/nginx/nginx-quadlet.conf        |  61 ++
 deploy/quadlets/bspds-app.container    |  28 +
 deploy/quadlets/bspds-db.container     |  23 +
 deploy/quadlets/bspds-minio.container  |  23 +
 deploy/quadlets/bspds-nginx.container  |  18 +
 deploy/quadlets/bspds-valkey.container |  21 +
 deploy/quadlets/bspds.pod              |   7 +
 docker-compose.prod.yml                | 173 +++++
 docker-compose.yaml                    |   4 +-
 docs/install-alpine.md                 | 313 ++++++++
 docs/install-containers.md             | 428 +++++++++++
 docs/install-debian.md                 | 280 ++++++++
 docs/install-kubernetes.md             | 956 +++++++++++++++++++++++++
 docs/install-openbsd.md                | 354 +++++++++
 nginx.prod.conf                        | 103 +++
 scripts/test-infra.sh                  |   4 +-
 17 files changed, 2815 insertions(+), 4 deletions(-)
 create mode 100644 deploy/nginx/nginx-quadlet.conf
 create mode 100644 deploy/quadlets/bspds-app.container
 create mode 100644 deploy/quadlets/bspds-db.container
 create mode 100644 deploy/quadlets/bspds-minio.container
 create mode 100644 deploy/quadlets/bspds-nginx.container
 create mode 100644 deploy/quadlets/bspds-valkey.container
 create mode 100644 deploy/quadlets/bspds.pod
 create mode 100644 docker-compose.prod.yml
 create mode 100644 docs/install-alpine.md
 create mode 100644 docs/install-containers.md
 create mode 100644 docs/install-debian.md
 create mode 100644 docs/install-kubernetes.md
 create mode 100644 docs/install-openbsd.md
 create mode 100644 nginx.prod.conf

diff --git a/README.md b/README.md
index be8ed97..7f9c4c9 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,29 @@ just test      # run tests
 just lint      # clippy + fmt
 ```
 
+## Production Deployment
+
+### Quick Deploy (Docker/Podman Compose)
+
+```bash
+cp .env.prod.example .env.prod
+# Edit .env.prod with your values (generate secrets with: openssl rand -base64 48)
+podman-compose -f docker-compose.prod.yml up -d
+```
+
+### Full Installation Guides
+
+| Guide | Best For |
+|-------|----------|
+| **Native Installation** | Maximum performance, full control |
+| [Debian](docs/install-debian.md) | Debian 13+ with systemd |
+| [Alpine](docs/install-alpine.md) | Alpine 3.23+ with OpenRC |
+| [OpenBSD](docs/install-openbsd.md) | OpenBSD 7.8+ with rc.d |
+| **Containerized** | Easier updates, isolation |
+| [Containers](docs/install-containers.md) | Podman with quadlets (Debian) or OpenRC (Alpine) |
+| **Orchestrated** | High availability, auto-scaling |
+| [Kubernetes](docs/install-kubernetes.md) | Multi-node k8s cluster deployment |
+
 ## License
 
 TBD
diff --git a/deploy/nginx/nginx-quadlet.conf b/deploy/nginx/nginx-quadlet.conf
new file mode 100644
index 0000000..f6c289e
--- /dev/null
+++ b/deploy/nginx/nginx-quadlet.conf
@@ -0,0 +1,61 @@
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+
+events {
+    worker_connections 4096;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    access_log /var/log/nginx/access.log;
+    sendfile on;
+    keepalive_timeout 65;
+
+    gzip on;
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers off;
+    ssl_session_cache shared:SSL:10m;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+
+    server {
+        listen 80;
+        listen [::]:80;
+        server_name _;
+
+        location /.well-known/acme-challenge/ {
+            root /var/www/acme;
+        }
+
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    server {
+        listen 443 ssl http2;
+        listen [::]:443 ssl http2;
+        server_name _;
+
+        ssl_certificate /etc/nginx/certs/fullchain.pem;
+        ssl_certificate_key /etc/nginx/certs/privkey.pem;
+        client_max_body_size 100M;
+
+        location / {
+            proxy_pass http://127.0.0.1:3000;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+            proxy_buffering off;
+        }
+    }
+}
diff --git a/deploy/quadlets/bspds-app.container b/deploy/quadlets/bspds-app.container
new file mode 100644
index 0000000..baae42e
--- /dev/null
+++ b/deploy/quadlets/bspds-app.container
@@ -0,0 +1,28 @@
+[Unit]
+Description=BSPDS AT Protocol PDS
+After=bspds-db.service bspds-minio.service bspds-valkey.service
+
+[Container]
+ContainerName=bspds-app
+Image=localhost/bspds:latest
+Pod=bspds.pod
+EnvironmentFile=/srv/bspds/config/bspds.env
+Environment=SERVER_HOST=0.0.0.0
+Environment=SERVER_PORT=3000
+Environment=S3_ENDPOINT=http://localhost:9000
+Environment=AWS_REGION=us-east-1
+Environment=S3_BUCKET=pds-blobs
+Environment=VALKEY_URL=redis://localhost:6379
+Environment=FRONTEND_DIR=/app/frontend/dist
+HealthCmd=wget -q --spider http://localhost:3000/xrpc/_health
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=15s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target
diff --git a/deploy/quadlets/bspds-db.container b/deploy/quadlets/bspds-db.container
new file mode 100644
index 0000000..08ed6e1
--- /dev/null
+++ b/deploy/quadlets/bspds-db.container
@@ -0,0 +1,23 @@
+[Unit]
+Description=BSPDS postgres database
+
+[Container]
+ContainerName=bspds-db
+Image=docker.io/library/postgres:18-alpine
+Pod=bspds.pod
+Environment=POSTGRES_USER=bspds
+Environment=POSTGRES_DB=pds
+Secret=bspds-db-password,type=env,target=POSTGRES_PASSWORD
+Volume=/srv/bspds/postgres:/var/lib/postgresql/data:Z
+HealthCmd=pg_isready -U bspds -d pds
+HealthInterval=10s
+HealthTimeout=5s
+HealthRetries=5
+HealthStartPeriod=10s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target
diff --git a/deploy/quadlets/bspds-minio.container b/deploy/quadlets/bspds-minio.container
new file mode 100644
index 0000000..9eec6d7
--- /dev/null
+++ b/deploy/quadlets/bspds-minio.container
@@ -0,0 +1,23 @@
+[Unit]
+Description=BSPDS minio object storage
+
+[Container]
+ContainerName=bspds-minio
+Image=docker.io/minio/minio:RELEASE.2025-10-15T17-29-55Z
+Pod=bspds.pod
+Environment=MINIO_ROOT_USER=minioadmin
+Secret=bspds-minio-password,type=env,target=MINIO_ROOT_PASSWORD
+Volume=/srv/bspds/minio:/data:Z
+Exec=server /data --console-address :9001
+HealthCmd=curl -f http://localhost:9000/minio/health/live || exit 1
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=10s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target
diff --git a/deploy/quadlets/bspds-nginx.container b/deploy/quadlets/bspds-nginx.container
new file mode 100644
index 0000000..8fb04be
--- /dev/null
+++ b/deploy/quadlets/bspds-nginx.container
@@ -0,0 +1,18 @@
+[Unit]
+Description=BSPDS nginx reverse proxy
+After=bspds-app.service
+
+[Container]
+ContainerName=bspds-nginx
+Image=docker.io/library/nginx:1.28-alpine
+Pod=bspds.pod
+Volume=/srv/bspds/config/nginx.conf:/etc/nginx/nginx.conf:ro,Z
+Volume=/srv/bspds/certs:/etc/nginx/certs:ro,Z
+Volume=/srv/bspds/acme:/var/www/acme:ro,Z
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target
diff --git a/deploy/quadlets/bspds-valkey.container b/deploy/quadlets/bspds-valkey.container
new file mode 100644
index 0000000..c81a894
--- /dev/null
+++ b/deploy/quadlets/bspds-valkey.container
@@ -0,0 +1,21 @@
+[Unit]
+Description=BSPDS valkey cache
+
+[Container]
+ContainerName=bspds-valkey
+Image=docker.io/valkey/valkey:9-alpine
+Pod=bspds.pod
+Volume=/srv/bspds/valkey:/data:Z
+Exec=valkey-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
+HealthCmd=valkey-cli ping
+HealthInterval=10s
+HealthTimeout=5s
+HealthRetries=3
+HealthStartPeriod=5s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target
diff --git a/deploy/quadlets/bspds.pod b/deploy/quadlets/bspds.pod
new file mode 100644
index 0000000..c45b5a1
--- /dev/null
+++ b/deploy/quadlets/bspds.pod
@@ -0,0 +1,7 @@
+[Pod]
+PodName=bspds
+PublishPort=80:80
+PublishPort=443:443
+
+[Install]
+WantedBy=default.target
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
new file mode 100644
index 0000000..78b9eb0
--- /dev/null
+++ b/docker-compose.prod.yml
@@ -0,0 +1,173 @@
+services:
+  bspds:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: bspds:latest
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:3000:3000"
+    environment:
+      SERVER_HOST: "0.0.0.0"
+      SERVER_PORT: "3000"
+      PDS_HOSTNAME: "${PDS_HOSTNAME:?PDS_HOSTNAME is required}"
+      DATABASE_URL: "postgres://bspds:${DB_PASSWORD:?DB_PASSWORD is required}@db:5432/pds"
+      S3_ENDPOINT: "http://minio:9000"
+      AWS_REGION: "us-east-1"
+      S3_BUCKET: "pds-blobs"
+      AWS_ACCESS_KEY_ID: "${MINIO_ROOT_USER:-minioadmin}"
+      AWS_SECRET_ACCESS_KEY: "${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}"
+      VALKEY_URL: "redis://valkey:6379"
+      JWT_SECRET: "${JWT_SECRET:?JWT_SECRET is required (min 32 chars)}"
+      DPOP_SECRET: "${DPOP_SECRET:?DPOP_SECRET is required (min 32 chars)}"
+      MASTER_KEY: "${MASTER_KEY:?MASTER_KEY is required (min 32 chars)}"
+      APPVIEW_URL: "${APPVIEW_URL:-https://api.bsky.app}"
+      CRAWLERS: "${CRAWLERS:-https://bsky.network}"
+      FRONTEND_DIR: "/app/frontend/dist"
+    depends_on:
+      db:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      valkey:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/xrpc/_health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+        reservations:
+          memory: 256M
+
+  db:
+    image: postgres:18-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: bspds
+      POSTGRES_PASSWORD: "${DB_PASSWORD:?DB_PASSWORD is required}"
+      POSTGRES_DB: pds
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U bspds -d pds"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+        reservations:
+          memory: 128M
+
+  minio:
+    image: minio/minio:RELEASE.2025-10-15T17-29-55Z
+    restart: unless-stopped
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: "${MINIO_ROOT_USER:-minioadmin}"
+      MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}"
+    volumes:
+      - minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+        reservations:
+          memory: 128M
+
+  minio-init:
+    image: minio/mc:RELEASE.2025-07-16T15-35-03Z
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: >
+      /bin/sh -c "
+      mc alias set local http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD};
+      mc mb --ignore-existing local/pds-blobs;
+      mc anonymous set none local/pds-blobs;
+      exit 0;
+      "
+    environment:
+      MINIO_ROOT_USER: "${MINIO_ROOT_USER:-minioadmin}"
+      MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}"
+
+  valkey:
+    image: valkey/valkey:9-alpine
+    restart: unless-stopped
+    command: valkey-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
+    volumes:
+      - valkey_data:/data
+    healthcheck:
+      test: ["CMD", "valkey-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    deploy:
+      resources:
+        limits:
+          memory: 300M
+        reservations:
+          memory: 64M
+
+  nginx:
+    image: nginx:1.28-alpine
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx.prod.conf:/etc/nginx/nginx.conf:ro
+      - ./certs:/etc/nginx/certs:ro
+      - acme_challenge:/var/www/acme:ro
+    depends_on:
+      - bspds
+    healthcheck:
+      test: ["CMD", "nginx", "-t"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  certbot:
+    image: certbot/certbot:v5.2.2
+    volumes:
+      - ./certs:/etc/letsencrypt
+      - acme_challenge:/var/www/acme
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew --webroot -w /var/www/acme; sleep 12h & wait $${!}; done'"
+
+  prometheus:
+    image: prom/prometheus:v3.8.0
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:9090:9090"
+    volumes:
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=30d'
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+volumes:
+  postgres_data:
+  minio_data:
+  valkey_data:
+  prometheus_data:
+  acme_challenge:
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 1973c8c..e76d0a0 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -18,7 +18,7 @@ services:
       - cache
 
   db:
-    image: postgres:latest
+    image: postgres:18-alpine
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
@@ -48,7 +48,7 @@ services:
       - valkey_data:/data
 
   prometheus:
-    image: prom/prometheus:latest
+    image: prom/prometheus:v3.8.0
     ports:
       - "9090:9090"
     volumes:
diff --git a/docs/install-alpine.md b/docs/install-alpine.md
new file mode 100644
index 0000000..b85f403
--- /dev/null
+++ b/docs/install-alpine.md
@@ -0,0 +1,313 @@
+# BSPDS Production Installation on Alpine Linux
+
+> **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified.
+
+This guide covers installing BSPDS on Alpine Linux 3.23 (current stable as of December 2025).
+
+## Choose Your Installation Method
+
+| Method | Best For |
+|--------|----------|
+| **Native (this guide)** | Maximum performance, minimal footprint, full control |
+| **[Containerized](install-containers.md)** | Easier updates, isolation, reproducible deployments |
+| **[Kubernetes](install-kubernetes.md)** | Multi-node, high availability, auto-scaling |
+
+This guide covers native installation. For containerized deployment with podman and systemd quadlets, see the [container guide](install-containers.md).
+
+---
+
+## Prerequisites
+
+- A VPS with at least 2GB RAM and 20GB disk
+- A domain name pointing to your server's IP
+- Root access
+
+## 1. System Setup
+
+```sh
+apk update && apk upgrade
+apk add curl git build-base openssl-dev pkgconf
+```
+
+## 2. Install Rust
+
+```sh
+apk add rustup
+rustup-init -y
+source ~/.cargo/env
+rustup default stable
+```
+
+This installs the latest stable Rust (1.92+ as of December 2025). Alpine 3.23 also ships Rust 1.91 via `apk add rust cargo` if you prefer system packages.
+
+## 3. Install postgres
+
+Alpine 3.23 includes PostgreSQL 18:
+
+```sh
+apk add postgresql postgresql-contrib
+
+rc-update add postgresql
+/etc/init.d/postgresql setup
+rc-service postgresql start
+
+psql -U postgres -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';"
+psql -U postgres -c "CREATE DATABASE pds OWNER bspds;"
+psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;"
+```
+
+## 4. Install minio
+
+```sh
+curl -O https://dl.min.io/server/minio/release/linux-amd64/minio
+chmod +x minio
+mv minio /usr/local/bin/
+
+mkdir -p /var/lib/minio/data
+adduser -D -H -s /sbin/nologin minio-user
+chown -R minio-user:minio-user /var/lib/minio
+
+cat > /etc/conf.d/minio << 'EOF'
+MINIO_ROOT_USER="minioadmin"
+MINIO_ROOT_PASSWORD="your-minio-password"
+MINIO_VOLUMES="/var/lib/minio/data"
+MINIO_OPTS="--console-address :9001"
+EOF
+
+cat > /etc/init.d/minio << 'EOF'
+#!/sbin/openrc-run
+
+name="minio"
+description="MinIO Object Storage"
+
+command="/usr/local/bin/minio"
+command_args="server ${MINIO_VOLUMES} ${MINIO_OPTS}"
+command_user="minio-user"
+command_background=true
+pidfile="/run/${RC_SVCNAME}.pid"
+output_log="/var/log/minio.log"
+error_log="/var/log/minio.log"
+
+depend() {
+    need net
+}
+
+start_pre() {
+    . /etc/conf.d/minio
+    export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+}
+EOF
+
+chmod +x /etc/init.d/minio
+rc-update add minio
+rc-service minio start
+```
+
+Create the blob bucket (wait a few seconds for minio to start):
+
+```sh
+curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
+chmod +x mc
+mv mc /usr/local/bin/
+
+mc alias set local http://localhost:9000 minioadmin your-minio-password
+mc mb local/pds-blobs
+```
+
+## 5. Install valkey
+
+Alpine 3.23 includes Valkey 9:
+
+```sh
+apk add valkey
+
+rc-update add valkey
+rc-service valkey start
+```
+
+## 6. Install deno (for frontend build)
+
+```sh
+curl -fsSL https://deno.land/install.sh | sh
+export PATH="$HOME/.deno/bin:$PATH"
+echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.profile
+```
+
+## 7. Clone and Build BSPDS
+
+```sh
+mkdir -p /opt && cd /opt
+git clone https://tangled.org/lewis.moe/bspds.git
+cd bspds
+
+cd frontend
+deno task build
+cd ..
+
+cargo build --release
+```
+
+## 8. Install sqlx-cli and Run Migrations
+
+```sh
+cargo install sqlx-cli --no-default-features --features postgres
+
+export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds"
+sqlx migrate run
+```
+
+## 9. Configure BSPDS
+
+```sh
+mkdir -p /etc/bspds
+cp /opt/bspds/.env.example /etc/bspds/bspds.env
+chmod 600 /etc/bspds/bspds.env
+```
+
+Edit `/etc/bspds/bspds.env` and fill in your values. Generate secrets with:
+
+```sh
+openssl rand -base64 48
+```
+
+## 10. Create OpenRC Service
+
+```sh
+adduser -D -H -s /sbin/nologin bspds
+
+cp /opt/bspds/target/release/bspds /usr/local/bin/
+mkdir -p /var/lib/bspds
+cp -r /opt/bspds/frontend/dist /var/lib/bspds/frontend
+chown -R bspds:bspds /var/lib/bspds
+
+cat > /etc/init.d/bspds << 'EOF'
+#!/sbin/openrc-run
+
+name="bspds"
+description="BSPDS - AT Protocol PDS"
+
+command="/usr/local/bin/bspds"
+command_user="bspds"
+command_background=true
+pidfile="/run/${RC_SVCNAME}.pid"
+output_log="/var/log/bspds.log"
+error_log="/var/log/bspds.log"
+
+depend() {
+    need net postgresql minio
+}
+
+start_pre() {
+    export FRONTEND_DIR=/var/lib/bspds/frontend
+    . /etc/bspds/bspds.env
+    export SERVER_HOST SERVER_PORT PDS_HOSTNAME DATABASE_URL
+    export S3_ENDPOINT AWS_REGION S3_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+    export VALKEY_URL JWT_SECRET DPOP_SECRET MASTER_KEY APPVIEW_URL CRAWLERS
+}
+EOF
+
+chmod +x /etc/init.d/bspds
+rc-update add bspds
+rc-service bspds start
+```
+
+## 11. Install and Configure nginx
+
+Alpine 3.23 includes nginx 1.28:
+
+```sh
+apk add nginx certbot certbot-nginx
+
+cat > /etc/nginx/http.d/bspds.conf << 'EOF'
+server {
+    listen 80;
+    listen [::]:80;
+    server_name pds.example.com;
+
+    location / {
+        proxy_pass http://127.0.0.1:3000;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 86400;
+    }
+}
+EOF
+
+rc-update add nginx
+rc-service nginx start
+```
+
+## 12. Obtain SSL Certificate
+
+```sh
+certbot --nginx -d pds.example.com
+```
+
+Set up auto-renewal:
+
+```sh
+echo "0 0 * * * certbot renew --quiet" | crontab -
+```
+
+## 13. Configure Firewall
+
+```sh
+apk add iptables ip6tables
+
+iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+iptables -A INPUT -p tcp --dport 80 -j ACCEPT
+iptables -A INPUT -p tcp --dport 443 -j ACCEPT
+iptables -A INPUT -i lo -j ACCEPT
+iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+iptables -P INPUT DROP
+
+ip6tables -A INPUT -p tcp --dport 22 -j ACCEPT
+ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT
+ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT
+ip6tables -A INPUT -i lo -j ACCEPT
+ip6tables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+ip6tables -P INPUT DROP
+
+rc-update add iptables
+rc-update add ip6tables
+/etc/init.d/iptables save
+/etc/init.d/ip6tables save
+```
+
+## 14. Verify Installation
+
+```sh
+rc-service bspds status
+curl -s https://pds.example.com/xrpc/_health
+curl -s https://pds.example.com/.well-known/atproto-did
+```
+
+## Maintenance
+
+View logs:
+```sh
+tail -f /var/log/bspds.log
+```
+
+Update BSPDS:
+```sh
+cd /opt/bspds
+git pull
+cd frontend && deno task build && cd ..
+cargo build --release
+rc-service bspds stop
+cp target/release/bspds /usr/local/bin/
+cp -r frontend/dist /var/lib/bspds/frontend
+DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run
+rc-service bspds start
+```
+
+Backup database:
+```sh
+pg_dump -U postgres pds > /var/backups/pds-$(date +%Y%m%d).sql
+```
diff --git a/docs/install-containers.md b/docs/install-containers.md
new file mode 100644
index 0000000..46bd50f
--- /dev/null
+++ b/docs/install-containers.md
@@ -0,0 +1,428 @@
+# BSPDS Containerized Production Deployment
+
+> **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified.
+
+This guide covers deploying BSPDS using containers with podman.
+
+- **Debian 13+**: Uses systemd quadlets (modern, declarative container management)
+- **Alpine 3.23+**: Uses OpenRC service script with podman-compose
+
+## Prerequisites
+
+- A VPS with at least 2GB RAM and 20GB disk
+- A domain name pointing to your server's IP
+- Root or sudo access
+
+## Quick Start (Docker/Podman Compose)
+
+If you just want to get running quickly:
+
+```sh
+cp .env.example .env
+
+# Edit .env with your values
+# Generate secrets: openssl rand -base64 48
+
+# Build and start
+podman-compose -f docker-compose.prod.yml up -d
+
+# Get initial certificate (after DNS is configured)
+podman-compose -f docker-compose.prod.yml run --rm certbot certonly \
+  --webroot -w /var/www/acme -d pds.example.com
+
+# Restart nginx to load certificate
+podman-compose -f docker-compose.prod.yml restart nginx
+```
+
+For production setups with proper service management, continue to either the Debian or Alpine section below.
+
+---
+
+# Debian 13+ with Systemd Quadlets
+
+Quadlets are the modern way to run podman containers under systemd.
+
+## 1. Install Podman
+
+```bash
+apt update
+apt install -y podman
+```
+
+## 2. Create Directory Structure
+
+```bash
+mkdir -p /etc/containers/systemd
+mkdir -p /srv/bspds/{postgres,minio,valkey,certs,acme,config}
+```
+
+## 3. Create Environment File
+
+```bash
+cp /opt/bspds/.env.example /srv/bspds/config/bspds.env
+chmod 600 /srv/bspds/config/bspds.env
+```
+
+Edit `/srv/bspds/config/bspds.env` and fill in your values. Generate secrets with:
+
+```bash
+openssl rand -base64 48
+```
+
+For quadlets, also add `DATABASE_URL` with the full connection string (systemd doesn't support variable expansion).
+
+## 4. Install Quadlet Definitions
+
+Copy the quadlet files from the repository:
+
+```bash
+cp /opt/bspds/deploy/quadlets/*.pod /etc/containers/systemd/
+cp /opt/bspds/deploy/quadlets/*.container /etc/containers/systemd/
+```
+
+Note: Systemd doesn't support shell-style variable expansion in `Environment=` lines. The quadlet files expect DATABASE_URL to be set in the environment file.
+
+## 5. Create nginx Configuration
+
+```bash
+cp /opt/bspds/deploy/nginx/nginx-quadlet.conf /srv/bspds/config/nginx.conf
+```
+
+## 6. Build BSPDS Image
+
+```bash
+cd /opt
+git clone https://tangled.org/lewis.moe/bspds.git
+cd bspds
+podman build -t bspds:latest .
+```
+
+## 7. Create Podman Secrets
+
+```bash
+source /srv/bspds/config/bspds.env
+echo "$DB_PASSWORD" | podman secret create bspds-db-password -
+echo "$MINIO_ROOT_PASSWORD" | podman secret create bspds-minio-password -
+```
+
+## 8. Start Services and Initialize
+
+```bash
+systemctl daemon-reload
+systemctl start bspds-db bspds-minio bspds-valkey
+
+sleep 10
+
+# Create MinIO bucket
+podman run --rm --pod bspds \
+  -e MINIO_ROOT_USER=minioadmin \
+  -e MINIO_ROOT_PASSWORD=your-minio-password \
+  docker.io/minio/mc:RELEASE.2025-07-16T15-35-03Z \
+  sh -c "mc alias set local http://localhost:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD && mc mb --ignore-existing local/pds-blobs"
+
+# Run migrations
+cargo install sqlx-cli --no-default-features --features postgres
+DATABASE_URL="postgres://bspds:your-db-password@localhost:5432/pds" sqlx migrate run --source /opt/bspds/migrations
+```
+
+## 9. Obtain SSL Certificate
+
+Create temporary self-signed cert:
+
+```bash
+openssl req -x509 -nodes -days 1 -newkey rsa:2048 \
+  -keyout /srv/bspds/certs/privkey.pem \
+  -out /srv/bspds/certs/fullchain.pem \
+  -subj "/CN=pds.example.com"
+
+systemctl start bspds-app bspds-nginx
+
+# Get real certificate
+podman run --rm \
+  -v /srv/bspds/certs:/etc/letsencrypt:Z \
+  -v /srv/bspds/acme:/var/www/acme:Z \
+  docker.io/certbot/certbot:v5.2.2 certonly \
+  --webroot -w /var/www/acme -d pds.example.com --agree-tos --email you@example.com
+
+# Link certificates
+ln -sf /srv/bspds/certs/live/pds.example.com/fullchain.pem /srv/bspds/certs/fullchain.pem
+ln -sf /srv/bspds/certs/live/pds.example.com/privkey.pem /srv/bspds/certs/privkey.pem
+
+systemctl restart bspds-nginx
+```
+
+## 10. Enable All Services
+
+```bash
+systemctl enable bspds-db bspds-minio bspds-valkey bspds-app bspds-nginx
+```
+
+## 11. Configure Firewall
+
+```bash
+apt install -y ufw
+ufw allow ssh
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw enable
+```
+
+## 12. Certificate Renewal
+
+Add to root's crontab (`crontab -e`):
+
+```
+0 0 * * * podman run --rm -v /srv/bspds/certs:/etc/letsencrypt:Z -v /srv/bspds/acme:/var/www/acme:Z docker.io/certbot/certbot:v5.2.2 renew --quiet && systemctl reload bspds-nginx
+```
+
+---
+
+# Alpine 3.23+ with OpenRC
+
+Alpine uses OpenRC, not systemd. We'll use podman-compose with an OpenRC service wrapper.
+
+## 1. Install Podman
+
+```sh
+apk update
+apk add podman podman-compose fuse-overlayfs cni-plugins
+rc-update add cgroups
+rc-service cgroups start
+```
+
+Enable podman socket for compose:
+
+```sh
+rc-update add podman
+rc-service podman start
+```
+
+## 2. Create Directory Structure
+
+```sh
+mkdir -p /srv/bspds/{data,config}
+mkdir -p /srv/bspds/data/{postgres,minio,valkey,certs,acme}
+```
+
+## 3. Clone Repository and Build
+
+```sh
+cd /opt
+git clone https://tangled.org/lewis.moe/bspds.git
+cd bspds
+podman build -t bspds:latest .
+```
+
+## 4. Create Environment File
+
+```sh
+cp /opt/bspds/.env.example /srv/bspds/config/bspds.env
+chmod 600 /srv/bspds/config/bspds.env
+```
+
+Edit `/srv/bspds/config/bspds.env` and fill in your values. Generate secrets with:
+
+```sh
+openssl rand -base64 48
+```
+
+## 5. Set Up Compose and nginx
+
+Copy the production compose and nginx configs:
+
+```sh
+cp /opt/bspds/docker-compose.prod.yml /srv/bspds/docker-compose.yml
+cp /opt/bspds/nginx.prod.conf /srv/bspds/config/nginx.conf
+```
+
+Edit `/srv/bspds/docker-compose.yml` to adjust paths if needed:
+- Update volume mounts to use `/srv/bspds/data/` paths
+- Update nginx cert paths to match `/srv/bspds/data/certs/`
+
+Edit `/srv/bspds/config/nginx.conf` to update cert paths:
+- Change `/etc/nginx/certs/live/${PDS_HOSTNAME}/` to `/etc/nginx/certs/`
+
+## 6. Create OpenRC Service
+
+```sh
+cat > /etc/init.d/bspds << 'EOF'
+#!/sbin/openrc-run
+
+name="bspds"
+description="BSPDS AT Protocol PDS (containerized)"
+
+command="/usr/bin/podman-compose"
+command_args="-f /srv/bspds/docker-compose.yml up"
+command_background=true
+pidfile="/run/${RC_SVCNAME}.pid"
+
+directory="/srv/bspds"
+
+depend() {
+    need net podman
+    after firewall
+}
+
+start_pre() {
+    set -a
+    . /srv/bspds/config/bspds.env
+    set +a
+}
+
+stop() {
+    ebegin "Stopping ${name}"
+    cd /srv/bspds
+    set -a
+    . /srv/bspds/config/bspds.env
+    set +a
+    podman-compose -f /srv/bspds/docker-compose.yml down
+    eend $?
+}
+EOF
+
+chmod +x /etc/init.d/bspds
+```
+
+## 7. Initialize Services
+
+```sh
+# Start services
+rc-service bspds start
+
+sleep 15
+
+# Create MinIO bucket
+source /srv/bspds/config/bspds.env
+podman run --rm --network bspds_default \
+  -e MINIO_ROOT_USER="$MINIO_ROOT_USER" \
+  -e MINIO_ROOT_PASSWORD="$MINIO_ROOT_PASSWORD" \
+  docker.io/minio/mc:RELEASE.2025-07-16T15-35-03Z \
+  sh -c 'mc alias set local http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD && mc mb --ignore-existing local/pds-blobs'
+
+# Run migrations
+apk add rustup
+rustup-init -y
+source ~/.cargo/env
+cargo install sqlx-cli --no-default-features --features postgres
+
+# Get database container IP
+DB_IP=$(podman inspect bspds-db-1 --format '{{.NetworkSettings.Networks.bspds_default.IPAddress}}')
+DATABASE_URL="postgres://bspds:$DB_PASSWORD@$DB_IP:5432/pds" sqlx migrate run --source /opt/bspds/migrations
+```
+
+## 8. Obtain SSL Certificate
+
+Create temporary self-signed cert:
+
+```sh
+openssl req -x509 -nodes -days 1 -newkey rsa:2048 \
+  -keyout /srv/bspds/data/certs/privkey.pem \
+  -out /srv/bspds/data/certs/fullchain.pem \
+  -subj "/CN=pds.example.com"
+
+rc-service bspds restart
+
+# Get real certificate
+podman run --rm \
+  -v /srv/bspds/data/certs:/etc/letsencrypt \
+  -v /srv/bspds/data/acme:/var/www/acme \
+  --network bspds_default \
+  docker.io/certbot/certbot:v5.2.2 certonly \
+  --webroot -w /var/www/acme -d pds.example.com --agree-tos --email you@example.com
+
+# Link certificates
+ln -sf /srv/bspds/data/certs/live/pds.example.com/fullchain.pem /srv/bspds/data/certs/fullchain.pem
+ln -sf /srv/bspds/data/certs/live/pds.example.com/privkey.pem /srv/bspds/data/certs/privkey.pem
+
+rc-service bspds restart
+```
+
+## 9. Enable Service at Boot
+
+```sh
+rc-update add bspds
+```
+
+## 10. Configure Firewall
+
+```sh
+apk add iptables ip6tables
+
+iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+iptables -A INPUT -p tcp --dport 80 -j ACCEPT
+iptables -A INPUT -p tcp --dport 443 -j ACCEPT
+iptables -A INPUT -i lo -j ACCEPT
+iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+iptables -P INPUT DROP
+
+ip6tables -A INPUT -p tcp --dport 22 -j ACCEPT
+ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT
+ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT
+ip6tables -A INPUT -i lo -j ACCEPT
+ip6tables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+ip6tables -P INPUT DROP
+
+rc-update add iptables
+rc-update add ip6tables
+/etc/init.d/iptables save
+/etc/init.d/ip6tables save
+```
+
+## 11. Certificate Renewal
+
+Add to root's crontab (`crontab -e`):
+
+```
+0 0 * * * podman run --rm -v /srv/bspds/data/certs:/etc/letsencrypt -v /srv/bspds/data/acme:/var/www/acme docker.io/certbot/certbot:v5.2.2 renew --quiet && rc-service bspds restart
+```
+
+---
+
+# Verification and Maintenance
+
+## Verify Installation
+
+```sh
+curl -s https://pds.example.com/xrpc/_health | jq
+curl -s https://pds.example.com/.well-known/atproto-did
+```
+
+## View Logs
+
+**Debian:**
+```bash
+journalctl -u bspds-app -f
+podman logs -f bspds-app
+```
+
+**Alpine:**
+```sh
+podman-compose -f /srv/bspds/docker-compose.yml logs -f
+podman logs -f bspds-bspds-1
+```
+
+## Update BSPDS
+
+```sh
+cd /opt/bspds
+git pull
+podman build -t bspds:latest .
+
+# Debian:
+systemctl restart bspds-app
+
+# Alpine:
+rc-service bspds restart
+```
+
+## Backup Database
+
+**Debian:**
+```bash
+podman exec bspds-db pg_dump -U bspds pds > /var/backups/pds-$(date +%Y%m%d).sql
+```
+
+**Alpine:**
+```sh
+podman exec bspds-db-1 pg_dump -U bspds pds > /var/backups/pds-$(date +%Y%m%d).sql
+```
diff --git a/docs/install-debian.md b/docs/install-debian.md
new file mode 100644
index 0000000..f359f06
--- /dev/null
+++ b/docs/install-debian.md
@@ -0,0 +1,280 @@
+# BSPDS Production Installation on Debian
+
+> **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified.
+
+This guide covers installing BSPDS on Debian 13 "Trixie" (current stable as of December 2025).
+
+## Choose Your Installation Method
+
+| Method | Best For |
+|--------|----------|
+| **Native (this guide)** | Maximum performance, full control, simpler debugging |
+| **[Containerized](install-containers.md)** | Easier updates, isolation, reproducible deployments |
+| **[Kubernetes](install-kubernetes.md)** | Multi-node, high availability, auto-scaling |
+
+This guide covers native installation. For containerized deployment with podman and systemd quadlets, see the [container guide](install-containers.md).
+
+---
+
+## Prerequisites
+
+- A VPS with at least 2GB RAM and 20GB disk
+- A domain name pointing to your server's IP
+- Root or sudo access
+
+## 1. System Setup
+
+```bash
+apt update && apt upgrade -y
+apt install -y curl git build-essential pkg-config libssl-dev
+```
+
+## 2. Install Rust
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+source ~/.cargo/env
+rustup default stable
+```
+
+This installs the latest stable Rust (1.92+ as of December 2025).
+
+## 3. Install postgres
+
+Debian 13 includes PostgreSQL 17:
+
+```bash
+apt install -y postgresql postgresql-contrib
+
+systemctl enable postgresql
+systemctl start postgresql
+
+sudo -u postgres psql -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';"
+sudo -u postgres psql -c "CREATE DATABASE pds OWNER bspds;"
+sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;"
+```
+
+## 4. Install minio
+
+```bash
+curl -O https://dl.min.io/server/minio/release/linux-amd64/minio
+chmod +x minio
+mv minio /usr/local/bin/
+
+mkdir -p /var/lib/minio/data
+useradd -r -s /sbin/nologin minio-user
+chown -R minio-user:minio-user /var/lib/minio
+
+cat > /etc/default/minio << 'EOF'
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=your-minio-password
+MINIO_VOLUMES="/var/lib/minio/data"
+MINIO_OPTS="--console-address :9001"
+EOF
+
+cat > /etc/systemd/system/minio.service << 'EOF'
+[Unit]
+Description=MinIO Object Storage
+After=network.target
+
+[Service]
+User=minio-user
+Group=minio-user
+EnvironmentFile=/etc/default/minio
+ExecStart=/usr/local/bin/minio server $MINIO_VOLUMES $MINIO_OPTS
+Restart=always
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable minio
+systemctl start minio
+```
+
+Create the blob bucket (wait a few seconds for minio to start):
+
+```bash
+curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
+chmod +x mc
+mv mc /usr/local/bin/
+
+mc alias set local http://localhost:9000 minioadmin your-minio-password
+mc mb local/pds-blobs
+```
+
+## 5. Install valkey
+
+Debian 13 includes Valkey 8:
+
+```bash
+apt install -y valkey
+
+systemctl enable valkey-server
+systemctl start valkey-server
+```
+
+## 6. Install deno (for frontend build)
+
+```bash
+curl -fsSL https://deno.land/install.sh | sh
+export PATH="$HOME/.deno/bin:$PATH"
+echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.bashrc
+```
+
+## 7. Clone and Build BSPDS
+
+```bash
+cd /opt
+git clone https://tangled.org/lewis.moe/bspds.git
+cd bspds
+
+cd frontend
+deno task build
+cd ..
+
+cargo build --release
+```
+
+## 8. Install sqlx-cli and Run Migrations
+
+```bash
+cargo install sqlx-cli --no-default-features --features postgres
+
+export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds"
+sqlx migrate run
+```
+
+## 9. Configure BSPDS
+
+```bash
+mkdir -p /etc/bspds
+cp /opt/bspds/.env.example /etc/bspds/bspds.env
+chmod 600 /etc/bspds/bspds.env
+```
+
+Edit `/etc/bspds/bspds.env` and fill in your values. Generate secrets with:
+
+```bash
+openssl rand -base64 48
+```
+
+## 10. Create Systemd Service
+
+```bash
+useradd -r -s /sbin/nologin bspds
+
+cp /opt/bspds/target/release/bspds /usr/local/bin/
+mkdir -p /var/lib/bspds
+cp -r /opt/bspds/frontend/dist /var/lib/bspds/frontend
+chown -R bspds:bspds /var/lib/bspds
+
+cat > /etc/systemd/system/bspds.service << 'EOF'
+[Unit]
+Description=BSPDS - AT Protocol PDS
+After=network.target postgresql.service minio.service
+
+[Service]
+Type=simple
+User=bspds
+Group=bspds
+EnvironmentFile=/etc/bspds/bspds.env
+Environment=FRONTEND_DIR=/var/lib/bspds/frontend
+ExecStart=/usr/local/bin/bspds
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable bspds
+systemctl start bspds
+```
+
+## 11. Install and Configure nginx
+
+Debian 13 includes nginx 1.26:
+
+```bash
+apt install -y nginx certbot python3-certbot-nginx
+
+cat > /etc/nginx/sites-available/bspds << 'EOF'
+server {
+    listen 80;
+    listen [::]:80;
+    server_name pds.example.com;
+
+    location / {
+        proxy_pass http://127.0.0.1:3000;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 86400;
+    }
+}
+EOF
+
+ln -s /etc/nginx/sites-available/bspds /etc/nginx/sites-enabled/
+rm -f /etc/nginx/sites-enabled/default
+nginx -t
+systemctl reload nginx
+```
+
+## 12. Obtain SSL Certificate
+
+```bash
+certbot --nginx -d pds.example.com
+```
+
+Certbot automatically configures nginx for HTTP/2 and sets up auto-renewal.
+
+## 13. Configure Firewall
+
+```bash
+apt install -y ufw
+ufw allow ssh
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw enable
+```
+
+## 14. Verify Installation
+
+```bash
+systemctl status bspds
+curl -s https://pds.example.com/xrpc/_health | jq
+curl -s https://pds.example.com/.well-known/atproto-did
+```
+
+## Maintenance
+
+View logs:
+```bash
+journalctl -u bspds -f
+```
+
+Update BSPDS:
+```bash
+cd /opt/bspds
+git pull
+cd frontend && deno task build && cd ..
+cargo build --release
+systemctl stop bspds
+cp target/release/bspds /usr/local/bin/
+cp -r frontend/dist /var/lib/bspds/frontend
+DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run
+systemctl start bspds
+```
+
+Backup database:
+```bash
+sudo -u postgres pg_dump pds > /var/backups/pds-$(date +%Y%m%d).sql
+```
diff --git a/docs/install-kubernetes.md b/docs/install-kubernetes.md
new file mode 100644
index 0000000..5e291fb
--- /dev/null
+++ b/docs/install-kubernetes.md
@@ -0,0 +1,956 @@
+# BSPDS Production Kubernetes Deployment
+
+> **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified.
+
+This guide covers deploying BSPDS on a production multi-node Kubernetes cluster with high availability, auto-scaling, and proper secrets management.
+
+## Architecture Overview
+
+```
+                    ┌─────────────────────────────────────────────────┐
+                    │              Kubernetes Cluster                 │
+                    │                                                 │
+    Internet ──────►│  Ingress Controller (nginx/traefik)             │
+                    │         │                                       │
+                    │         ▼                                       │
+                    │  ┌─────────────┐                                │
+                    │  │   Service   │◄── HPA (2-10 replicas)         │
+                    │  └──────┬──────┘                                │
+                    │         │                                       │
+                    │    ┌────┴────┐                                  │
+                    │    ▼         ▼                                  │
+                    │ ┌─────┐  ┌─────┐                                │
+                    │ │BSPDS│  │BSPDS│  ... (pods)                    │
+                    │ └──┬──┘  └──┬──┘                                │
+                    │    │        │                                   │
+                    │    ▼        ▼                                   │
+                    │ ┌──────────────────────────────────────┐        │
+                    │ │  PostgreSQL  │  MinIO  │  Valkey     │        │
+                    │ │  (HA/Operator)│ (StatefulSet) │ (Sentinel)    │
+                    │ └──────────────────────────────────────┘        │
+                    └─────────────────────────────────────────────────┘
+```
+
+## Prerequisites
+
+- Kubernetes cluster (1.30+) with at least 3 nodes (1.34 is current stable)
+- `kubectl` configured to access your cluster
+- `helm` 3.x installed
+- Storage class that supports `ReadWriteOnce` (for databases)
+- Ingress controller installed (nginx-ingress or traefik)
+- cert-manager installed for TLS certificates
+
+### Quick Prerequisites Setup
+
+If you need to install prerequisites:
+
+```bash
+# Install nginx-ingress (chart v4.14.1 - December 2025)
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+helm repo update
+helm install ingress-nginx ingress-nginx/ingress-nginx \
+  --namespace ingress-nginx --create-namespace \
+  --version 4.14.1
+
+# Install cert-manager (v1.19.2 - December 2025)
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install cert-manager jetstack/cert-manager \
+  --namespace cert-manager --create-namespace \
+  --version v1.19.2 \
+  --set installCRDs=true
+```
+
+---
+
+## 1. Create Namespace
+
+```bash
+kubectl create namespace bspds
+kubectl config set-context --current --namespace=bspds
+```
+
+## 2. Create Secrets
+
+Generate secure passwords and secrets:
+
+```bash
+# Generate secrets
+DB_PASSWORD=$(openssl rand -base64 32)
+MINIO_PASSWORD=$(openssl rand -base64 32)
+JWT_SECRET=$(openssl rand -base64 48)
+DPOP_SECRET=$(openssl rand -base64 48)
+MASTER_KEY=$(openssl rand -base64 48)
+
+# Create Kubernetes secrets
+kubectl create secret generic bspds-db-credentials \
+  --from-literal=username=bspds \
+  --from-literal=password="$DB_PASSWORD"
+
+kubectl create secret generic bspds-minio-credentials \
+  --from-literal=root-user=minioadmin \
+  --from-literal=root-password="$MINIO_PASSWORD"
+
+kubectl create secret generic bspds-secrets \
+  --from-literal=jwt-secret="$JWT_SECRET" \
+  --from-literal=dpop-secret="$DPOP_SECRET" \
+  --from-literal=master-key="$MASTER_KEY"
+
+# Save secrets locally (KEEP SECURE!)
+echo "DB_PASSWORD=$DB_PASSWORD" > secrets.txt
+echo "MINIO_PASSWORD=$MINIO_PASSWORD" >> secrets.txt
+echo "JWT_SECRET=$JWT_SECRET" >> secrets.txt
+echo "DPOP_SECRET=$DPOP_SECRET" >> secrets.txt
+echo "MASTER_KEY=$MASTER_KEY" >> secrets.txt
+chmod 600 secrets.txt
+```
+
+## 3. Deploy PostgreSQL
+
+### Option A: CloudNativePG Operator (Recommended for HA)
+
+```bash
+# Install CloudNativePG operator (v1.28.0 - December 2025)
+kubectl apply --server-side -f \
+  https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.28/releases/cnpg-1.28.0.yaml
+
+# Wait for operator
+kubectl wait --for=condition=available --timeout=120s \
+  deployment/cnpg-controller-manager -n cnpg-system
+```
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: bspds-db
+  namespace: bspds
+spec:
+  instances: 3
+
+  postgresql:
+    parameters:
+      max_connections: "200"
+      shared_buffers: "256MB"
+
+  bootstrap:
+    initdb:
+      database: pds
+      owner: bspds
+      secret:
+        name: bspds-db-credentials
+
+  storage:
+    size: 20Gi
+    storageClass: standard  # adjust for your cluster
+
+  resources:
+    requests:
+      memory: "512Mi"
+      cpu: "250m"
+    limits:
+      memory: "1Gi"
+      cpu: "1000m"
+
+  affinity:
+    podAntiAffinityType: required
+EOF
+```
+
+### Option B: Simple StatefulSet (Single Instance)
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: bspds-db-pvc
+  namespace: bspds
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: bspds-db
+  namespace: bspds
+spec:
+  serviceName: bspds-db
+  replicas: 1
+  selector:
+    matchLabels:
+      app: bspds-db
+  template:
+    metadata:
+      labels:
+        app: bspds-db
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:18-alpine
+          ports:
+            - containerPort: 5432
+          env:
+            - name: POSTGRES_DB
+              value: pds
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-db-credentials
+                  key: username
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-db-credentials
+                  key: password
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+              cpu: "500m"
+          livenessProbe:
+            exec:
+              command: ["pg_isready", "-U", "bspds", "-d", "pds"]
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command: ["pg_isready", "-U", "bspds", "-d", "pds"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: bspds-db-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bspds-db-rw
+  namespace: bspds
+spec:
+  selector:
+    app: bspds-db
+  ports:
+    - port: 5432
+      targetPort: 5432
+EOF
+```
+
+## 4. Deploy MinIO
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: bspds-minio-pvc
+  namespace: bspds
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: bspds-minio
+  namespace: bspds
+spec:
+  serviceName: bspds-minio
+  replicas: 1
+  selector:
+    matchLabels:
+      app: bspds-minio
+  template:
+    metadata:
+      labels:
+        app: bspds-minio
+    spec:
+      containers:
+        - name: minio
+          image: minio/minio:RELEASE.2025-10-15T17-29-55Z
+          args:
+            - server
+            - /data
+            - --console-address
+            - ":9001"
+          ports:
+            - containerPort: 9000
+              name: api
+            - containerPort: 9001
+              name: console
+          env:
+            - name: MINIO_ROOT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-minio-credentials
+                  key: root-user
+            - name: MINIO_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-minio-credentials
+                  key: root-password
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "512Mi"
+              cpu: "500m"
+          livenessProbe:
+            httpGet:
+              path: /minio/health/live
+              port: 9000
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /minio/health/ready
+              port: 9000
+            initialDelaySeconds: 10
+            periodSeconds: 5
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: bspds-minio-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bspds-minio
+  namespace: bspds
+spec:
+  selector:
+    app: bspds-minio
+  ports:
+    - port: 9000
+      targetPort: 9000
+      name: api
+    - port: 9001
+      targetPort: 9001
+      name: console
+EOF
+```
+
+### Initialize MinIO Bucket
+
+```bash
+kubectl run minio-init --rm -it --restart=Never \
+  --image=minio/mc:RELEASE.2025-07-16T15-35-03Z \
+  --env="MINIO_ROOT_USER=minioadmin" \
+  --env="MINIO_ROOT_PASSWORD=$(kubectl get secret bspds-minio-credentials -o jsonpath='{.data.root-password}' | base64 -d)" \
+  --command -- sh -c "
+    mc alias set local http://bspds-minio:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD &&
+    mc mb --ignore-existing local/pds-blobs
+  "
+```
+
+## 5. Deploy Valkey
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: bspds-valkey-pvc
+  namespace: bspds
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: bspds-valkey
+  namespace: bspds
+spec:
+  serviceName: bspds-valkey
+  replicas: 1
+  selector:
+    matchLabels:
+      app: bspds-valkey
+  template:
+    metadata:
+      labels:
+        app: bspds-valkey
+    spec:
+      containers:
+        - name: valkey
+          image: valkey/valkey:9-alpine
+          args:
+            - valkey-server
+            - --appendonly
+            - "yes"
+            - --maxmemory
+            - "256mb"
+            - --maxmemory-policy
+            - allkeys-lru
+          ports:
+            - containerPort: 6379
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "50m"
+            limits:
+              memory: "300Mi"
+              cpu: "200m"
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "ping"]
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          readinessProbe:
+            exec:
+              command: ["valkey-cli", "ping"]
+            initialDelaySeconds: 5
+            periodSeconds: 3
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: bspds-valkey-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bspds-valkey
+  namespace: bspds
+spec:
+  selector:
+    app: bspds-valkey
+  ports:
+    - port: 6379
+      targetPort: 6379
+EOF
+```
+
+## 6. Build and Push BSPDS Image
+
+```bash
+# Build image
+cd /path/to/bspds
+docker build -t your-registry.com/bspds:latest .
+docker push your-registry.com/bspds:latest
+```
+
+If using a private registry, create an image pull secret:
+
+```bash
+kubectl create secret docker-registry regcred \
+  --docker-server=your-registry.com \
+  --docker-username=your-username \
+  --docker-password=your-password \
+  --docker-email=your-email
+```
+
+## 7. Run Database Migrations
+
+BSPDS runs migrations automatically on startup. However, if you want to run migrations separately (recommended for zero-downtime deployments), you can use a Job:
+
+```bash
+cat <<'EOF' | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: bspds-migrate
+  namespace: bspds
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: migrate
+          image: your-registry.com/bspds:latest
+          command: ["/usr/local/bin/bspds"]
+          args: ["--migrate-only"]  # Add this flag to your app, or remove this Job
+          env:
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-db-credentials
+                  key: password
+            - name: DATABASE_URL
+              value: "postgres://bspds:$(DB_PASSWORD)@bspds-db-rw:5432/pds"
+EOF
+
+kubectl wait --for=condition=complete --timeout=120s job/bspds-migrate
+```
+
+> **Note**: If your BSPDS image doesn't have a `--migrate-only` flag, you can skip this step. The app will run migrations on first startup. Alternatively, build a separate migration image with `sqlx-cli` installed.
+
+## 8. Deploy BSPDS Application
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bspds-config
+  namespace: bspds
+data:
+  PDS_HOSTNAME: "pds.example.com"
+  SERVER_HOST: "0.0.0.0"
+  SERVER_PORT: "3000"
+  S3_ENDPOINT: "http://bspds-minio:9000"
+  AWS_REGION: "us-east-1"
+  S3_BUCKET: "pds-blobs"
+  VALKEY_URL: "redis://bspds-valkey:6379"
+  APPVIEW_URL: "https://api.bsky.app"
+  CRAWLERS: "https://bsky.network"
+  FRONTEND_DIR: "/app/frontend/dist"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bspds
+  namespace: bspds
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: bspds
+  template:
+    metadata:
+      labels:
+        app: bspds
+    spec:
+      imagePullSecrets:
+        - name: regcred  # Remove if using public registry
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: bspds
+                topologyKey: kubernetes.io/hostname
+      containers:
+        - name: bspds
+          image: your-registry.com/bspds:latest
+          ports:
+            - containerPort: 3000
+              name: http
+          envFrom:
+            - configMapRef:
+                name: bspds-config
+          env:
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-db-credentials
+                  key: password
+            - name: DATABASE_URL
+              value: "postgres://bspds:$(DB_PASSWORD)@bspds-db-rw:5432/pds"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-minio-credentials
+                  key: root-user
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-minio-credentials
+                  key: root-password
+            - name: JWT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-secrets
+                  key: jwt-secret
+            - name: DPOP_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-secrets
+                  key: dpop-secret
+            - name: MASTER_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: bspds-secrets
+                  key: master-key
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+              cpu: "1000m"
+          livenessProbe:
+            httpGet:
+              path: /xrpc/_health
+              port: 3000
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /xrpc/_health
+              port: 3000
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            allowPrivilegeEscalation: false
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bspds
+  namespace: bspds
+spec:
+  selector:
+    app: bspds
+  ports:
+    - port: 80
+      targetPort: 3000
+      name: http
+EOF
+```
+
+## 9. Configure Horizontal Pod Autoscaler
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: bspds
+  namespace: bspds
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: bspds
+  minReplicas: 2
+  maxReplicas: 10
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 80
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 300
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 60
+    scaleUp:
+      stabilizationWindowSeconds: 0
+      policies:
+        - type: Percent
+          value: 100
+          periodSeconds: 15
+        - type: Pods
+          value: 4
+          periodSeconds: 15
+      selectPolicy: Max
+EOF
+```
+
+## 10. Configure Pod Disruption Budget
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: bspds
+  namespace: bspds
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: bspds
+EOF
+```
+
+## 11. Configure TLS with cert-manager
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: your-email@example.com
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
+EOF
+```
+
+## 12. Configure Ingress
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: bspds
+  namespace: bspds
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "86400"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "86400"
+    nginx.ingress.kubernetes.io/proxy-body-size: "100m"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/websocket-services: "bspds"
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - pds.example.com
+      secretName: bspds-tls
+  rules:
+    - host: pds.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: bspds
+                port:
+                  number: 80
+EOF
+```
+
+## 13. Configure Network Policies (Optional but Recommended)
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: bspds-network-policy
+  namespace: bspds
+spec:
+  podSelector:
+    matchLabels:
+      app: bspds
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: ingress-nginx
+      ports:
+        - protocol: TCP
+          port: 3000
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app: bspds-db
+      ports:
+        - protocol: TCP
+          port: 5432
+    - to:
+        - podSelector:
+            matchLabels:
+              app: bspds-minio
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - podSelector:
+            matchLabels:
+              app: bspds-valkey
+      ports:
+        - protocol: TCP
+          port: 6379
+    - to:  # Allow DNS
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+    - to:  # Allow external HTTPS (for federation)
+        - ipBlock:
+            cidr: 0.0.0.0/0
+      ports:
+        - protocol: TCP
+          port: 443
+EOF
+```
+
+## 14. Deploy Prometheus Monitoring (Optional)
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: bspds
+  namespace: bspds
+  labels:
+    release: prometheus
+spec:
+  selector:
+    matchLabels:
+      app: bspds
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+EOF
+```
+
+---
+
+## Verification
+
+```bash
+# Check all pods are running
+kubectl get pods -n bspds
+
+# Check services
+kubectl get svc -n bspds
+
+# Check ingress
+kubectl get ingress -n bspds
+
+# Check certificate
+kubectl get certificate -n bspds
+
+# Test health endpoint
+curl -s https://pds.example.com/xrpc/_health | jq
+
+# Test DID endpoint
+curl -s https://pds.example.com/.well-known/atproto-did
+```
+
+---
+
+## Maintenance
+
+### View Logs
+
+```bash
+# All BSPDS pods
+kubectl logs -l app=bspds -n bspds -f
+
+# Specific pod
+kubectl logs -f deployment/bspds -n bspds
+```
+
+### Scale Manually
+
+```bash
+kubectl scale deployment bspds --replicas=5 -n bspds
+```
+
+### Update BSPDS
+
+```bash
+# Build and push new image
+docker build -t your-registry.com/bspds:v1.2.3 .
+docker push your-registry.com/bspds:v1.2.3
+
+# Update deployment
+kubectl set image deployment/bspds bspds=your-registry.com/bspds:v1.2.3 -n bspds
+
+# Watch rollout
+kubectl rollout status deployment/bspds -n bspds
+```
+
+### Backup Database
+
+```bash
+# For CloudNativePG
+kubectl cnpg backup bspds-db -n bspds
+
+# For StatefulSet
+kubectl exec -it bspds-db-0 -n bspds -- pg_dump -U bspds pds > backup-$(date +%Y%m%d).sql
+```
+
+### Run Migrations
+
+If you have a migration Job defined, you can re-run it:
+
+```bash
+# Delete old job first (if exists)
+kubectl delete job bspds-migrate -n bspds --ignore-not-found
+
+# Re-apply the migration job from step 7
+# Or simply restart the deployment - BSPDS runs migrations on startup
+kubectl rollout restart deployment/bspds -n bspds
+```
+
+---
+
+## Troubleshooting
+
+### Pod Won't Start
+
+```bash
+kubectl describe pod -l app=bspds -n bspds
+kubectl logs -l app=bspds -n bspds --previous
+```
+
+### Database Connection Issues
+
+```bash
+# Test connectivity from a debug pod
+kubectl run debug --rm -it --restart=Never --image=postgres:18-alpine -- \
+  psql "postgres://bspds:PASSWORD@bspds-db-rw:5432/pds" -c "SELECT 1"
+```
+
+### Certificate Issues
+
+```bash
+kubectl describe certificate bspds-tls -n bspds
+kubectl describe certificaterequest -n bspds
+kubectl logs -l app.kubernetes.io/name=cert-manager -n cert-manager
+```
+
+### View Resource Usage
+
+```bash
+kubectl top pods -n bspds
+kubectl top nodes
+```
diff --git a/docs/install-openbsd.md b/docs/install-openbsd.md
new file mode 100644
index 0000000..869819f
--- /dev/null
+++ b/docs/install-openbsd.md
@@ -0,0 +1,354 @@
+# BSPDS Production Installation on OpenBSD
+
+> **Warning**: These instructions are untested and theoretical, written from the top of Lewis' head. They may contain errors or omissions. This warning will be removed once the guide has been verified.
+
+This guide covers installing BSPDS on OpenBSD 7.8 (current release as of December 2025).
+
+## Prerequisites
+
+- A VPS with at least 2GB RAM and 20GB disk
+- A domain name pointing to your server's IP
+- Root access (or doas configured)
+
+## Why nginx over relayd?
+
+OpenBSD's native `relayd` supports WebSockets but does **not** support HTTP/2. For a modern PDS deployment, we recommend nginx which provides HTTP/2, WebSocket support, and automatic OCSP stapling.
+
+## 1. System Setup
+
+```sh
+pkg_add curl git
+```
+
+## 2. Install Rust
+
+```sh
+pkg_add rust
+```
+
+OpenBSD 7.8 ships Rust 1.82+. For the latest stable (1.92+), use rustup:
+
+```sh
+pkg_add rustup
+rustup-init -y
+source ~/.cargo/env
+rustup default stable
+```
+
+## 3. Install postgres
+
+OpenBSD 7.8 includes PostgreSQL 17 (PostgreSQL 18 may not yet be in ports):
+
+```sh
+pkg_add postgresql-server postgresql-client
+
+mkdir -p /var/postgresql/data
+chown _postgresql:_postgresql /var/postgresql/data
+su - _postgresql -c "initdb -D /var/postgresql/data -U postgres -A scram-sha-256"
+
+rcctl enable postgresql
+rcctl start postgresql
+
+psql -U postgres -c "CREATE USER bspds WITH PASSWORD 'your-secure-password';"
+psql -U postgres -c "CREATE DATABASE pds OWNER bspds;"
+psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE pds TO bspds;"
+```
+
+## 4. Install minio
+
+OpenBSD doesn't have a minio package. Options:
+
+**Option A: Use an external S3-compatible service (recommended for production)**
+
+aws s3, backblaze b2, or upcloud managed object storage. Skip to step 5 and configure the S3 credentials in step 9.
+
+**Option B: Build minio from source**
+
+```sh
+pkg_add go
+
+mkdir -p /tmp/minio-build && cd /tmp/minio-build
+ftp -o minio.tar.gz https://github.com/minio/minio/archive/refs/tags/RELEASE.2025-10-15T17-29-55Z.tar.gz
+tar xzf minio.tar.gz
+cd minio-*
+go build -o minio .
+cp minio /usr/local/bin/
+
+mkdir -p /var/minio/data
+useradd -d /var/minio -s /sbin/nologin _minio
+chown -R _minio:_minio /var/minio
+
+cat > /etc/minio.conf << 'EOF'
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=your-minio-password
+EOF
+chmod 600 /etc/minio.conf
+
+cat > /etc/rc.d/minio << 'EOF'
+#!/bin/ksh
+
+daemon="/usr/local/bin/minio"
+daemon_user="_minio"
+daemon_flags="server /var/minio/data --console-address :9001"
+
+. /etc/rc.d/rc.subr
+
+rc_pre() {
+    . /etc/minio.conf
+    export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+}
+
+rc_cmd $1
+EOF
+
+chmod +x /etc/rc.d/minio
+rcctl enable minio
+rcctl start minio
+```
+
+Create the blob bucket:
+
+```sh
+ftp -o /usr/local/bin/mc https://dl.min.io/client/mc/release/openbsd-amd64/mc
+chmod +x /usr/local/bin/mc
+
+mc alias set local http://localhost:9000 minioadmin your-minio-password
+mc mb local/pds-blobs
+```
+
+## 5. Install redis
+
+OpenBSD has redis in ports (valkey may not be available yet):
+
+```sh
+pkg_add redis
+
+rcctl enable redis
+rcctl start redis
+```
+
+## 6. Install deno (for frontend build)
+
+```sh
+curl -fsSL https://deno.land/install.sh | sh
+export PATH="$HOME/.deno/bin:$PATH"
+echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.profile
+```
+
+## 7. Clone and Build BSPDS
+
+```sh
+mkdir -p /opt && cd /opt
+git clone https://tangled.org/lewis.moe/bspds.git
+cd bspds
+
+cd frontend
+deno task build
+cd ..
+
+cargo build --release
+```
+
+## 8. Install sqlx-cli and Run Migrations
+
+```sh
+cargo install sqlx-cli --no-default-features --features postgres
+
+export DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds"
+sqlx migrate run
+```
+
+## 9. Configure BSPDS
+
+```sh
+mkdir -p /etc/bspds
+cp /opt/bspds/.env.example /etc/bspds/bspds.conf
+chmod 600 /etc/bspds/bspds.conf
+```
+
+Edit `/etc/bspds/bspds.conf` and fill in your values. Generate secrets with:
+
+```sh
+openssl rand -base64 48
+```
+
+## 10. Create rc.d Service
+
+```sh
+useradd -d /var/empty -s /sbin/nologin _bspds
+
+cp /opt/bspds/target/release/bspds /usr/local/bin/
+mkdir -p /var/bspds
+cp -r /opt/bspds/frontend/dist /var/bspds/frontend
+chown -R _bspds:_bspds /var/bspds
+
+cat > /etc/rc.d/bspds << 'EOF'
+#!/bin/ksh
+
+daemon="/usr/local/bin/bspds"
+daemon_user="_bspds"
+daemon_logger="daemon.info"
+
+. /etc/rc.d/rc.subr
+
+rc_pre() {
+    export FRONTEND_DIR=/var/bspds/frontend
+    while IFS='=' read -r key value; do
+        case "$key" in
+            \#*|"") continue ;;
+        esac
+        export "$key=$value"
+    done < /etc/bspds/bspds.conf
+}
+
+rc_cmd $1
+EOF
+
+chmod +x /etc/rc.d/bspds
+rcctl enable bspds
+rcctl start bspds
+```
+
+## 11. Install and Configure nginx
+
+```sh
+pkg_add nginx
+
+cat > /etc/nginx/nginx.conf << 'EOF'
+worker_processes 1;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include mime.types;
+
+    server {
+        listen 80;
+        listen [::]:80;
+        server_name pds.example.com;
+
+        location /.well-known/acme-challenge/ {
+            root /var/www/acme;
+        }
+
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    server {
+        listen 443 ssl http2;
+        listen [::]:443 ssl http2;
+        server_name pds.example.com;
+
+        ssl_certificate /etc/ssl/pds.example.com.fullchain.pem;
+        ssl_certificate_key /etc/ssl/private/pds.example.com.key;
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_ciphers HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers on;
+        ssl_session_cache shared:SSL:10m;
+
+        location / {
+            proxy_pass http://127.0.0.1:3000;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 86400;
+        }
+    }
+}
+EOF
+
+mkdir -p /var/www/acme
+rcctl enable nginx
+```
+
+## 12. Obtain SSL Certificate with acme-client
+
+OpenBSD's native acme-client works well:
+
+```sh
+cat >> /etc/acme-client.conf << 'EOF'
+
+authority letsencrypt {
+    api url "https://acme-v02.api.letsencrypt.org/directory"
+    account key "/etc/acme/letsencrypt-privkey.pem"
+}
+
+domain pds.example.com {
+    domain key "/etc/ssl/private/pds.example.com.key"
+    domain full chain certificate "/etc/ssl/pds.example.com.fullchain.pem"
+    sign with letsencrypt
+}
+EOF
+
+mkdir -p /etc/acme
+
+rcctl start nginx
+
+acme-client -v pds.example.com
+
+rcctl restart nginx
+```
+
+Set up auto-renewal in root's crontab:
+
+```sh
+crontab -e
+```
+
+Add:
+```
+0 0 * * * acme-client pds.example.com && rcctl reload nginx
+```
+
+## 13. Configure Packet Filter (pf)
+
+```sh
+cat >> /etc/pf.conf << 'EOF'
+
+# BSPDS rules
+pass in on egress proto tcp from any to any port { 22, 80, 443 }
+EOF
+
+pfctl -f /etc/pf.conf
+```
+
+## 14. Verify Installation
+
+```sh
+rcctl check bspds
+ftp -o - https://pds.example.com/xrpc/_health
+ftp -o - https://pds.example.com/.well-known/atproto-did
+```
+
+## Maintenance
+
+View logs:
+```sh
+tail -f /var/log/daemon
+```
+
+Update BSPDS:
+```sh
+cd /opt/bspds
+git pull
+cd frontend && deno task build && cd ..
+cargo build --release
+rcctl stop bspds
+cp target/release/bspds /usr/local/bin/
+cp -r frontend/dist /var/bspds/frontend
+DATABASE_URL="postgres://bspds:your-secure-password@localhost:5432/pds" sqlx migrate run
+rcctl start bspds
+```
+
+Backup database:
+```sh
+pg_dump -U postgres pds > /var/backups/pds-$(date +%Y%m%d).sql
+```
diff --git a/nginx.prod.conf b/nginx.prod.conf
new file mode 100644
index 0000000..9869fd4
--- /dev/null
+++ b/nginx.prod.conf
@@ -0,0 +1,103 @@
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 4096;
+    use epoll;
+    multi_accept on;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for" '
+                    'rt=$request_time uct="$upstream_connect_time" '
+                    'uht="$upstream_header_time" urt="$upstream_response_time"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript
+               application/xml application/xml+rss text/javascript application/activity+json;
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
+    ssl_prefer_server_ciphers off;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_timeout 1d;
+    ssl_session_tickets off;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+
+    upstream bspds {
+        server bspds:3000;
+        keepalive 32;
+    }
+
+    server {
+        listen 80;
+        listen [::]:80;
+        server_name _;
+
+        location /.well-known/acme-challenge/ {
+            root /var/www/acme;
+        }
+
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    server {
+        listen 443 ssl http2;
+        listen [::]:443 ssl http2;
+        server_name _;
+
+        ssl_certificate /etc/nginx/certs/live/${PDS_HOSTNAME}/fullchain.pem;
+        ssl_certificate_key /etc/nginx/certs/live/${PDS_HOSTNAME}/privkey.pem;
+
+        client_max_body_size 100M;
+
+        location / {
+            proxy_pass http://bspds;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+            proxy_buffering off;
+            proxy_request_buffering off;
+        }
+
+        location /xrpc/com.atproto.sync.subscribeRepos {
+            proxy_pass http://bspds;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+            proxy_buffering off;
+        }
+    }
+}
diff --git a/scripts/test-infra.sh b/scripts/test-infra.sh
index 6ef044d..37ef455 100755
--- a/scripts/test-infra.sh
+++ b/scripts/test-infra.sh
@@ -57,7 +57,7 @@ start_infra() {
         -e MINIO_ROOT_PASSWORD=minioadmin \
         -P \
         --label bspds_test=true \
-        minio/minio:latest server /data >/dev/null
+        minio/minio:RELEASE.2025-10-15T17-29-55Z server /data >/dev/null
 
     echo "Starting Valkey..."
     $CONTAINER_CMD run -d \
@@ -100,7 +100,7 @@ start_infra() {
     echo "Creating MinIO bucket..."
     $CONTAINER_CMD run --rm --network host \
         -e MC_HOST_minio="http://minioadmin:minioadmin@127.0.0.1:${MINIO_PORT}" \
-        minio/mc:latest mb minio/test-bucket --ignore-existing >/dev/null 2>&1 || true
+        minio/mc:RELEASE.2025-07-16T15-35-03Z mb minio/test-bucket --ignore-existing >/dev/null 2>&1 || true
 
     cat > "$INFRA_FILE" << EOF
 export DATABASE_URL="postgres://postgres:postgres@127.0.0.1:${PG_PORT}/postgres"