From 02a4bece5dc60448b39075053e5b73df167c8411 Mon Sep 17 00:00:00 2001 From: William Gill Date: Sun, 19 Apr 2026 19:25:42 -0500 Subject: [PATCH] Ollama: keep loaded models resident until evicted (KEEP_ALIVE=-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Was 30m, which evicts after 30 minutes of inactivity and forces a reload penalty on the next request. Setting -1 holds models in VRAM indefinitely; MAX_LOADED_MODELS=3 caps how many can stay resident simultaneously (vs the previous 2). Tune MAX higher if you're rotating between more than three models AND your GPU has the VRAM for it — comment in the compose explains the trade-off. For the live srvno.de stack: OLLAMA_KEEP_ALIVE=-1 takes effect on the next `docker compose up -d ollama`. Loaded models survive the restart only if they're re-requested before swap-out anyway. Co-Authored-By: Claude Opus 4.7 (1M context) --- deployments/ai-stack/docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deployments/ai-stack/docker-compose.yml b/deployments/ai-stack/docker-compose.yml index 311f3dc..109d9d2 100644 --- a/deployments/ai-stack/docker-compose.yml +++ b/deployments/ai-stack/docker-compose.yml @@ -60,8 +60,12 @@ services: - ollama-data:/root/.ollama environment: - OLLAMA_HOST=0.0.0.0:11434 - - OLLAMA_KEEP_ALIVE=30m - - OLLAMA_MAX_LOADED_MODELS=2 + # KEEP_ALIVE=-1 holds loaded models in VRAM until evicted by another + # load (vs the default 5m / our previous 30m which forces a reload + # penalty on every cold use). Pair with MAX_LOADED_MODELS sized to + # whatever fits in your GPU's VRAM — see README "VRAM sizing". + - OLLAMA_KEEP_ALIVE=-1 + - OLLAMA_MAX_LOADED_MODELS=3 - OLLAMA_FLASH_ATTENTION=1 deploy: resources: