diff --git a/deployments/ai-stack/docker-compose.yml b/deployments/ai-stack/docker-compose.yml index 311f3dc..109d9d2 100644 --- a/deployments/ai-stack/docker-compose.yml +++ b/deployments/ai-stack/docker-compose.yml @@ -60,8 +60,12 @@ services: - ollama-data:/root/.ollama environment: - OLLAMA_HOST=0.0.0.0:11434 - - OLLAMA_KEEP_ALIVE=30m - - OLLAMA_MAX_LOADED_MODELS=2 + # KEEP_ALIVE=-1 holds loaded models in VRAM until evicted by another + # load (vs the default 5m / our previous 30m which forces a reload + # penalty on every cold use). Pair with MAX_LOADED_MODELS sized to + # whatever fits in your GPU's VRAM — see README "VRAM sizing". + - OLLAMA_KEEP_ALIVE=-1 + - OLLAMA_MAX_LOADED_MODELS=3 - OLLAMA_FLASH_ATTENTION=1 deploy: resources: