diff --git a/deployments/ai-stack/.env.example b/deployments/ai-stack/.env.example index d81ae83..969c3a3 100644 --- a/deployments/ai-stack/.env.example +++ b/deployments/ai-stack/.env.example @@ -27,3 +27,11 @@ COMFYUI_IMAGE_TAG=latest # gated repos (Flux-dev, SD3, etc.). Generate a read token at # https://huggingface.co/settings/tokens. Leave empty for public-only. HF_TOKEN= + +# HTTPS base URL of an S3 bucket / CDN that hosts mirrored Ollama model +# tarballs (created by mirror-ollama-model.sh). Files under this base are +# fetched by init-models.sh's s3_pull instead of registry.ollama.ai — +# faster and immune to upstream rate-limiting / removal. Example: +# S3_OLLAMA_BASE=https://your-bucket.s3.amazonaws.com/ollama-models +# Leave empty to fall back to plain `ollama pull` for everything. +S3_OLLAMA_BASE= diff --git a/deployments/ai-stack/README.md b/deployments/ai-stack/README.md index 7b799e4..0f5119c 100644 --- a/deployments/ai-stack/README.md +++ b/deployments/ai-stack/README.md @@ -15,6 +15,7 @@ production `srvno.de` deployment. | `docker-compose.yml` | Service definitions, volumes, GPU reservations | | `Caddyfile` | TLS + reverse proxy config (one site block per hostname) | | `init-models.sh` | LLMs to preseed into Ollama on first boot | +| `mirror-ollama-model.sh` | Helper — mirror an Ollama model into a tarball you can host on S3 | | `comfyui-init-models.sh` | Checkpoints/VAEs/LoRAs to preseed into ComfyUI on first boot | | `openwebui-tools/smart_image_gen.py` | Tool that auto-routes image generation AND editing to the right SDXL checkpoint | | `openwebui-models/image_studio.md` | Dedicated chat-model preset — manual setup walkthrough | @@ -64,7 +65,14 @@ Then edit: ``` - **`init-models.sh`** — keep the LLMs you want preseeded, drop the rest. Check sizes at first; the host needs disk - for everything listed. + for everything listed. Two pull paths are available: + - `pull ""` — standard registry pull from + `registry.ollama.ai`. + - `s3_pull "" ""` — fetches from your own + mirror set via `S3_OLLAMA_BASE` in `.env`. Falls back to + `ollama pull` if the env var isn't set, so this is safe to enable + incrementally. Create the tarballs once with + `mirror-ollama-model.sh` (see [Mirroring models to S3](#mirroring-models-to-s3)). - **`comfyui-init-models.sh`** — checkpoints/VAEs/LoRAs to preseed into ComfyUI. Ships empty (no active fetches) — uncomment the SDXL/Flux/ upscaler examples or add your own. Whatever filename you pick should @@ -217,6 +225,54 @@ To extend (new checkpoint, new style): auto-detect path. - Re-paste the Tool source in Workspace -> Tools. +## Mirroring models to S3 + +For models you want to pin against upstream changes (or pull faster +from your own infra), mirror them to S3 once and have the +deployment fetch from there. + +### Create the mirror tarball + +Run [`mirror-ollama-model.sh`](mirror-ollama-model.sh) on any machine +that has the model pulled locally. It reads `~/.ollama/models/`, +pulls the manifest's referenced blobs, and tars everything together: + +```sh +./mirror-ollama-model.sh huihui_ai/qwen3.5-abliterated:9b qwen3.5-abliterated-9b.tgz +``` + +### Upload to S3 + +Whatever fits — `aws s3 cp`, `mc`, `rclone`, etc. The bucket needs +to expose the file over HTTPS (public-read ACL on the object, a +CloudFront distribution, R2 with public URLs, etc.): + +```sh +aws s3 cp qwen3.5-abliterated-9b.tgz s3://your-bucket/ollama-models/ --acl public-read +``` + +### Wire the deployment to fetch from there + +In `.env`: + +``` +S3_OLLAMA_BASE=https://your-bucket.s3.amazonaws.com/ollama-models +``` + +In `init-models.sh`, switch the affected models from `pull` to +`s3_pull`: + +```sh +s3_pull "huihui_ai/qwen3.5-abliterated:9b" "qwen3.5-abliterated-9b.tgz" +``` + +`docker compose up -d model-init` re-runs the init container; the +script downloads the tarball, extracts into the `ollama-data` volume, +and the running Ollama daemon picks it up on its next manifest scan. + +If `S3_OLLAMA_BASE` isn't set, `s3_pull` transparently falls back to +`ollama pull` — safe to commit `s3_pull` lines without S3 ready yet. + ## Enabling Anubis (later) The `anubis-owui` service is defined in compose but no Caddy site block diff --git a/deployments/ai-stack/docker-compose.yml b/deployments/ai-stack/docker-compose.yml index 33c38c9..b13e471 100644 --- a/deployments/ai-stack/docker-compose.yml +++ b/deployments/ai-stack/docker-compose.yml @@ -79,6 +79,10 @@ services: # One-shot model puller. Runs after ollama is healthy, pulls whatever # init-models.sh lists, exits. `restart: "no"` keeps it from looping. + # + # Models can come from registry.ollama.ai (default) or your own S3 + # mirror (set S3_OLLAMA_BASE in .env; create tarballs with + # mirror-ollama-model.sh). model-init: image: ollama/ollama:latest container_name: ollama-model-init @@ -90,6 +94,7 @@ services: - ./init-models.sh:/init-models.sh:ro environment: - OLLAMA_HOST=ollama:11434 + - S3_OLLAMA_BASE=${S3_OLLAMA_BASE:-} entrypoint: ["/bin/sh", "/init-models.sh"] restart: "no" diff --git a/deployments/ai-stack/init-models.sh b/deployments/ai-stack/init-models.sh index dfc9714..374973d 100644 --- a/deployments/ai-stack/init-models.sh +++ b/deployments/ai-stack/init-models.sh @@ -3,20 +3,61 @@ # Runs once via the model-init service (see docker-compose.yml). Safe to # re-run — already-present models are skipped. # -# Add or remove tags to taste. The host needs enough disk for everything -# listed; check sizes at https://ollama.com/library before adding. +# Two pull paths: +# - s3_pull — fetches a tarball from $S3_OLLAMA_BASE (your own mirror, +# created by mirror-ollama-model.sh) and extracts into +# Ollama's data dir. Faster + immune to upstream changes. +# Falls back to ollama pull if S3_OLLAMA_BASE is unset. +# - pull — standard `ollama pull` against registry.ollama.ai. set -e -MODELS="dolphin3:8b llama3.1:8b ministral-3:8b mistral-nemo:12b qwen3.6:latest" +# Make sure curl is available — ollama/ollama:latest doesn't always include +# it, and s3_pull needs it. tar is in the base image. +if ! command -v curl >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null +fi -for model in $MODELS; do - if ollama list | awk 'NR>1 {print $1}' | grep -qx "$model"; then - echo "✓ $model already present" - else - echo "→ Pulling $model…" - ollama pull "$model" - fi +S3_OLLAMA_BASE="${S3_OLLAMA_BASE:-}" +OLLAMA_DATA="/root/.ollama" + +s3_pull() { + name="$1"; archive="$2" + if ollama list 2>/dev/null | awk 'NR>1 {print $1}' | grep -qx "$name"; then + echo "✓ $name already present" + return + fi + if [ -z "$S3_OLLAMA_BASE" ]; then + echo "→ $name: S3_OLLAMA_BASE unset, falling back to ollama pull" + ollama pull "$name" + return + fi + url="${S3_OLLAMA_BASE%/}/$archive" + echo "→ Downloading $name from $url…" + curl -fL -C - --retry 3 -o "/tmp/$archive" "$url" + tar -xzf "/tmp/$archive" -C "$OLLAMA_DATA/models/" + rm -f "/tmp/$archive" + echo "✓ $name installed (mirror)" +} + +pull() { + name="$1" + if ollama list 2>/dev/null | awk 'NR>1 {print $1}' | grep -qx "$name"; then + echo "✓ $name already present" + else + echo "→ Pulling $name from registry.ollama.ai…" + ollama pull "$name" + fi +} + +# ─── S3-mirrored models ───────────────────────────────────────────────────── +# These live in your own bucket. Create the tarballs once with +# mirror-ollama-model.sh, upload to S3, then list them here. +s3_pull "huihui_ai/qwen3.5-abliterated:9b" "qwen3.5-abliterated-9b.tgz" + +# ─── Direct registry pulls ────────────────────────────────────────────────── +for model in dolphin3:8b llama3.1:8b ministral-3:8b mistral-nemo:12b qwen3.6:latest; do + pull "$model" done echo "Done." diff --git a/deployments/ai-stack/mirror-ollama-model.sh b/deployments/ai-stack/mirror-ollama-model.sh new file mode 100644 index 0000000..53b571c --- /dev/null +++ b/deployments/ai-stack/mirror-ollama-model.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Mirror an Ollama model into a portable tarball you can upload to S3 +# (or any HTTPS host) and re-fetch via init-models.sh's s3_pull. +# +# Run on any machine that already has the model pulled locally — the +# script reads ~/.ollama/models/, parses the manifest to find the +# referenced blobs, and tars them together. +# +# Usage: ./mirror-ollama-model.sh +# Example: ./mirror-ollama-model.sh huihui_ai/qwen3.5-abliterated:9b qwen3.5-abliterated-9b.tgz +# +# Upload the tarball to S3, then add to init-models.sh: +# s3_pull "huihui_ai/qwen3.5-abliterated:9b" "qwen3.5-abliterated-9b.tgz" +# and set S3_OLLAMA_BASE in .env to your bucket's HTTPS base URL. + +set -euo pipefail + +MODEL="${1:?Usage: $0 }" +OUT="${2:?Usage: $0 }" + +OLLAMA_HOME="${OLLAMA_HOME:-$HOME/.ollama}" +MODELS="$OLLAMA_HOME/models" + +if ! ollama list | awk 'NR>1 {print $1}' | grep -qx "$MODEL"; then + echo "Model $MODEL not found locally; pulling first..." + ollama pull "$MODEL" +fi + +# huihui_ai/qwen3.5-abliterated:9b → manifests/registry.ollama.ai/huihui_ai/qwen3.5-abliterated/9b +ns_and_name="${MODEL%:*}" +tag="${MODEL##*:}" +manifest_rel="manifests/registry.ollama.ai/$ns_and_name/$tag" +manifest_abs="$MODELS/$manifest_rel" + +if [ ! -f "$manifest_abs" ]; then + echo "ERROR: manifest not found at $manifest_abs" >&2 + exit 1 +fi + +# Pull every sha256:* digest out of the manifest JSON. Each maps to +# blobs/sha256-. +blob_files="" +for digest in $(grep -oE 'sha256:[a-f0-9]+' "$manifest_abs" | sort -u); do + blob_rel="blobs/${digest/:/-}" + if [ ! -f "$MODELS/$blob_rel" ]; then + echo "WARNING: missing blob $blob_rel — skipping" >&2 + continue + fi + blob_files="$blob_files $blob_rel" +done + +count=$(echo "$blob_files" | wc -w | tr -d ' ') +echo "Archiving manifest + $count blob(s)..." +tar -czf "$OUT" -C "$MODELS" "$manifest_rel" $blob_files + +size=$(du -h "$OUT" | cut -f1) +echo "Done: $OUT ($size)" +echo +echo "Next:" +echo " 1. Upload to your bucket, e.g." +echo " aws s3 cp $OUT s3://YOUR-BUCKET/ollama-models/ --acl public-read" +echo " (or whatever exposes it over HTTPS)" +echo " 2. Set S3_OLLAMA_BASE in .env to the bucket's HTTPS base, e.g." +echo " S3_OLLAMA_BASE=https://YOUR-BUCKET.s3.amazonaws.com/ollama-models" +echo " 3. Add to init-models.sh:" +echo " s3_pull \"$MODEL\" \"$(basename "$OUT")\""