From a73aa4a38ac970f53c76fe9d85a5bc894c7f3c64 Mon Sep 17 00:00:00 2001 From: William Gill Date: Sun, 19 Apr 2026 10:21:43 -0500 Subject: [PATCH] =?UTF-8?q?Initial=20commit=20=E2=80=94=20ComfyUI=20on=20N?= =?UTF-8?q?VIDIA=20fronted=20by=20Open=20WebUI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the figment/segment/Forge stack with a single ComfyUI backend fronted by Open WebUI's native ComfyUI integration. ComfyUI is built from the official manual install for NVIDIA. Ships txt2img and img2img workflow templates plus matching node-mapping JSONs that paste into Open WebUI's admin panel. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 9 +++ .gitignore | 5 ++ Dockerfile | 59 +++++++++++++++ README.md | 36 ++++++++++ SETUP.md | 135 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 106 +++++++++++++++++++++++++++ workflows/img2img.json | 64 +++++++++++++++++ workflows/img2img.nodes.json | 8 +++ workflows/txt2img.json | 59 +++++++++++++++ workflows/txt2img.nodes.json | 10 +++ 10 files changed, 491 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 SETUP.md create mode 100644 docker-compose.yml create mode 100644 workflows/img2img.json create mode 100644 workflows/img2img.nodes.json create mode 100644 workflows/txt2img.json create mode 100644 workflows/txt2img.nodes.json diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..71d65a2 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# Copy to .env and edit. docker compose picks .env up automatically. + +# Public hostname/scheme your reverse proxy terminates. Used by Open WebUI for +# auth redirects and email-link generation. +OPEN_WEBUI_URL=http://localhost:3000 + +# Cookie-signing key. Generate once with: openssl rand -hex 32 +# Must stay stable across restarts — rotating it logs every user out. +OPEN_WEBUI_SECRET_KEY=change-me-please diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cc0dd56 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.env +*.log +.idea/ +.vscode/ +__pycache__/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a3f81c4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +# ComfyUI on NVIDIA — manual install per +# https://docs.comfy.org/installation/manual_install#nvidia +# +# Base image provides system CUDA / cuDNN libs that some custom nodes dlopen at +# runtime. The PyTorch wheels installed below bundle their own CUDA runtime — +# the base only needs to be ABI-compatible with the host driver via +# nvidia-container-toolkit. +# +# CUDA 12.6 + cu126 wheels chosen for broad driver compatibility (driver >= +# 545). If the host driver is >= 580, switch the wheel index URL to cu130 for +# the newest stable PyTorch. + +FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + COMFYUI_HOME=/opt/comfyui + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.12 \ + python3.12-venv \ + python3-pip \ + git \ + ca-certificates \ + curl \ + libgl1 \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +# Pin a venv so future pip installs (custom nodes) stay isolated. +RUN python3.12 -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH + +# Upstream wheel for NVIDIA. cu126 covers driver >= 545; bump to cu130 for +# driver >= 580. +RUN pip install --upgrade pip && \ + pip install torch torchvision torchaudio \ + --extra-index-url https://download.pytorch.org/whl/cu126 + +# Pull ComfyUI itself. Pinning to a tag would be safer for reproducibility but +# the project ships breaking changes infrequently and most users want latest. +RUN git clone --depth 1 https://github.com/comfyanonymous/ComfyUI.git ${COMFYUI_HOME} +WORKDIR ${COMFYUI_HOME} +RUN pip install -r requirements.txt + +# ComfyUI-Manager — community node manager. Lets users install/update custom +# nodes from the web UI instead of editing the Dockerfile. Remove this block if +# you want a pristine ComfyUI. +RUN git clone --depth 1 https://github.com/ltdrdata/ComfyUI-Manager.git \ + ${COMFYUI_HOME}/custom_nodes/ComfyUI-Manager && \ + pip install -r ${COMFYUI_HOME}/custom_nodes/ComfyUI-Manager/requirements.txt + +EXPOSE 8188 + +# --listen 0.0.0.0 binds to every interface so the Open WebUI container on the +# shared compose network can reach it. --port is explicit for clarity. +CMD ["python", "main.py", "--listen", "0.0.0.0", "--port", "8188"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..f17e51a --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# comfyui-nvidia + +ComfyUI image-generation backend, NVIDIA-accelerated, fronted by Open WebUI for +multi-user chat and image generation/editing. + +Built from the official ComfyUI [manual install for +NVIDIA](https://docs.comfy.org/installation/manual_install#nvidia) — no third- +party base image. Two services on one bridge network: + +| Service | Port (host) | Role | +| ----------- | ----------- | ----------------------------------------------- | +| `comfyui` | `8188` | ComfyUI server + native web UI | +| `open-webui`| `3000` | Multi-user chat with txt2img / img2img panel | + +Open WebUI submits ComfyUI workflows directly via the documented integration — +no MCP shim, no API translation layer. Workflow templates live in +[`workflows/`](workflows/) and get pasted into Open WebUI's admin panel during +setup. + +## Quick start + +```sh +cp .env.example .env +# edit .env — set OPEN_WEBUI_SECRET_KEY at minimum +docker compose up -d --build +``` + +Then drop a checkpoint into the `comfyui-models` volume and finish the Open +WebUI ComfyUI configuration. Full walkthrough in [SETUP.md](SETUP.md). + +## Replaces + +This repo supersedes the previous figment + segment + Forge stack. ComfyUI's +node graph covers everything those services provided (txt2img, img2img, +inpaint, mask generation via SAM/GroundingDINO custom nodes), and Open WebUI +talks to it natively. diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..d09c224 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,135 @@ +# Setup + +End-to-end walkthrough: clean host -> running stack -> first generated image +in Open WebUI. + +## 1. Host prerequisites + +- Linux (or WSL2) with an NVIDIA GPU and a recent driver. + - cu126 wheels (default Dockerfile): driver >= 545 + - cu130 wheels (swap in Dockerfile): driver >= 580 +- Docker Engine + Compose v2. +- [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) + installed and the Docker runtime configured (`nvidia-ctk runtime configure + --runtime=docker && systemctl restart docker`). + +Confirm GPU passthrough works before bringing the stack up: + +```sh +docker run --rm --gpus all nvidia/cuda:12.6.3-base-ubuntu24.04 nvidia-smi +``` + +## 2. Configure environment + +```sh +cp .env.example .env +``` + +Edit `.env`: + +- `OPEN_WEBUI_URL` — public URL Open WebUI is reached at (used for auth + redirects). For local-only, leave the default. +- `OPEN_WEBUI_SECRET_KEY` — generate with `openssl rand -hex 32`. Keep stable; + rotating it logs every user out. + +## 3. Build and start + +```sh +docker compose up -d --build +``` + +First build pulls the CUDA base, PyTorch wheels, and ComfyUI source — expect +several minutes. `docker compose logs -f comfyui` should end with a line like +`To see the GUI go to: http://0.0.0.0:8188`. + +Health-check both services: + +```sh +curl -sf http://localhost:8188/system_stats | head -c 200 +curl -sf http://localhost:3000/health +``` + +## 4. Add at least one checkpoint + +ComfyUI ships no models. The shipped workflow templates reference +`v1-5-pruned-emaonly.safetensors` as a placeholder; drop any SD/SDXL/Flux +checkpoint into the `comfyui-models` volume under `checkpoints/`: + +```sh +docker run --rm -v comfyui-nvidia_comfyui-models:/models -w /models/checkpoints \ + curlimages/curl:latest -L -O \ + https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors +``` + +Or open the ComfyUI web UI at , click the **Manager** +button (added by ComfyUI-Manager), and use **Model Manager** to install one +through the UI. + +## 5. First-user signup in Open WebUI + +Open . The first account created becomes the admin. +Subsequent signups land in `pending` and need admin approval (set by +`DEFAULT_USER_ROLE: pending` in compose). + +## 6. Wire Open WebUI to ComfyUI + +Open WebUI ships the ComfyUI integration but won't know which workflow to +submit until you paste one in. Do this once per workflow (txt2img and +img2img). + +In Open WebUI: **Admin Panel -> Settings -> Images**. + +1. **Image Generation Engine** -> `ComfyUI` (already preselected via env var). +2. **ComfyUI Base URL** -> `http://comfyui:8188` (already preselected). +3. **ComfyUI Workflow** -> paste the entire contents of + [`workflows/txt2img.json`](workflows/txt2img.json). +4. **ComfyUI Workflow Nodes** -> paste the contents of + [`workflows/txt2img.nodes.json`](workflows/txt2img.nodes.json). +5. **Default Model** -> the filename of the checkpoint you dropped in step 4 + (e.g. `v1-5-pruned-emaonly.safetensors`). +6. Save. + +For image editing (img2img), scroll to the **Image Editing** section in the +same panel and repeat with [`workflows/img2img.json`](workflows/img2img.json) +and [`workflows/img2img.nodes.json`](workflows/img2img.nodes.json). + +## 7. Test it + +In any chat, click the image-generation button and prompt for an image. Open +WebUI submits the workflow to ComfyUI; the result drops back into the chat +when KSampler finishes. To test img2img, attach an image and use the edit +action. + +## How the workflow node mappings work + +Open WebUI doesn't introspect the workflow graph. The `*.nodes.json` files +tell it which node IDs and input fields to overwrite when the user provides +a prompt, image, seed, etc. Each entry: + +```json +{ "type": "", "node_ids": [""], "key": "" } +``` + +Recognised `type` strings (per Open WebUI source): `model`, `prompt`, +`negative_prompt`, `width`, `height`, `n` (batch size), `steps`, `seed`, and +`image` (img2img / edit only). + +If you swap in a fancier workflow (SDXL, Flux, ControlNet, custom samplers, +NL masking via SAM nodes, etc.), update the matching `*.nodes.json` so the +node IDs and input keys still line up. + +## Common gotchas + +- **"Model not found" in Open WebUI's image panel.** ComfyUI lists models from + `/opt/comfyui/models/checkpoints/`. Confirm the file is there and matches + the **Default Model** field exactly (filename including extension). +- **Out-of-memory on first generate.** Lower `IMAGE_SIZE` in compose + (e.g. `768x768`) or pass `--lowvram` / `--medvram` in the Dockerfile CMD. +- **Custom nodes need extra pip packages.** Install via ComfyUI-Manager (it + pip-installs into the container's venv). Persisted because + `/opt/comfyui/custom_nodes` is a named volume — but the venv at `/opt/venv` + is not, so packages added by the manager survive container restarts only if + the manager re-installs them on boot. For permanent custom-node deps, add a + `RUN pip install ...` to the Dockerfile and rebuild. +- **GPU not visible inside container.** Re-run the `nvidia-smi` test in step + 1. If it fails, the toolkit is misconfigured. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c1156b2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,106 @@ +# comfyui-nvidia — ComfyUI image-generation backend fronted by Open WebUI. +# +# Open WebUI provides the multi-user chat UI and a built-in image-generation +# panel that submits ComfyUI workflows directly. Two workflow templates ship in +# workflows/ — txt2img.json and img2img.json. Paste them (and the matching +# node-mapping JSON) into the Open WebUI admin panel under +# Settings -> Images. See SETUP.md. +# +# Single GPU, single host. Both services share the comfyui-net bridge so Open +# WebUI addresses ComfyUI by service name (http://comfyui:8188). + +name: comfyui-nvidia + +networks: + comfyui-net: + driver: bridge + +volumes: + comfyui-models: # /opt/comfyui/models (checkpoints, vae, loras, ...) + comfyui-custom-nodes: # /opt/comfyui/custom_nodes + comfyui-input: # /opt/comfyui/input (uploaded source images) + comfyui-output: # /opt/comfyui/output (generated images) + comfyui-user: # /opt/comfyui/user (saved workflows, settings) + open-webui-data: # Open WebUI accounts, chats, settings + +services: + comfyui: + build: + context: . + dockerfile: Dockerfile + image: comfyui-nvidia:local + container_name: comfyui + restart: unless-stopped + networks: [comfyui-net] + ports: + # Optional on the host — Open WebUI reaches comfyui:8188 over the bridge. + # Keep it published if you also want to use ComfyUI's native web UI at + # http://localhost:8188. + - "8188:8188" + volumes: + - comfyui-models:/opt/comfyui/models + - comfyui-custom-nodes:/opt/comfyui/custom_nodes + - comfyui-input:/opt/comfyui/input + - comfyui-output:/opt/comfyui/output + - comfyui-user:/opt/comfyui/user + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:8188/system_stats"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 120s + + open-webui: + image: ghcr.io/open-webui/open-webui:main + container_name: open-webui + restart: unless-stopped + networks: [comfyui-net] + ports: + - "3000:8080" + extra_hosts: + # So Open WebUI can reach an Ollama instance running on the VM host. + # Drop this if you don't run Ollama on the host. + - "host.docker.internal:host-gateway" + depends_on: + comfyui: + condition: service_started + environment: + WEBUI_AUTH: "true" + ENABLE_SIGNUP: "true" + DEFAULT_USER_ROLE: "pending" + + # Public URL used for auth redirects and email-link generation. Set this + # to whatever your reverse proxy terminates. + WEBUI_URL: "${OPEN_WEBUI_URL:-http://localhost:3000}" + + # Cookie-signing key. Stable across restarts (rotating it logs every + # user out). Generate once: openssl rand -hex 32 + WEBUI_SECRET_KEY: "${OPEN_WEBUI_SECRET_KEY:-change-me-please}" + + # Optional: Ollama on the VM host for chat. Comment out if unused. + OLLAMA_BASE_URL: "http://host.docker.internal:11434" + + # Image generation via ComfyUI. The workflow JSON and node-mapping JSON + # are configured through the admin UI — see SETUP.md for the paste-in + # values. Only the engine selection and base URL are wired here so the + # admin panel comes up pre-pointed at the right backend. + ENABLE_IMAGE_GENERATION: "true" + IMAGE_GENERATION_ENGINE: "comfyui" + COMFYUI_BASE_URL: "http://comfyui:8188" + IMAGE_SIZE: "1024x1024" + IMAGE_STEPS: "20" + volumes: + - open-webui-data:/app/backend/data + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:8080/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s diff --git a/workflows/img2img.json b/workflows/img2img.json new file mode 100644 index 0000000..912ecc0 --- /dev/null +++ b/workflows/img2img.json @@ -0,0 +1,64 @@ +{ + "3": { + "class_type": "KSampler", + "inputs": { + "seed": 0, + "steps": 20, + "cfg": 7, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 0.75, + "model": ["4", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["10", 0] + } + }, + "4": { + "class_type": "CheckpointLoaderSimple", + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.safetensors" + } + }, + "6": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "", + "clip": ["4", 1] + } + }, + "7": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "", + "clip": ["4", 1] + } + }, + "8": { + "class_type": "VAEDecode", + "inputs": { + "samples": ["3", 0], + "vae": ["4", 2] + } + }, + "9": { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + } + }, + "10": { + "class_type": "VAEEncode", + "inputs": { + "pixels": ["11", 0], + "vae": ["4", 2] + } + }, + "11": { + "class_type": "LoadImage", + "inputs": { + "image": "example.png" + } + } +} diff --git a/workflows/img2img.nodes.json b/workflows/img2img.nodes.json new file mode 100644 index 0000000..7b0e969 --- /dev/null +++ b/workflows/img2img.nodes.json @@ -0,0 +1,8 @@ +[ + { "type": "model", "node_ids": ["4"], "key": "ckpt_name" }, + { "type": "prompt", "node_ids": ["6"], "key": "text" }, + { "type": "negative_prompt", "node_ids": ["7"], "key": "text" }, + { "type": "steps", "node_ids": ["3"], "key": "steps" }, + { "type": "seed", "node_ids": ["3"], "key": "seed" }, + { "type": "image", "node_ids": ["11"], "key": "image" } +] diff --git a/workflows/txt2img.json b/workflows/txt2img.json new file mode 100644 index 0000000..43198be --- /dev/null +++ b/workflows/txt2img.json @@ -0,0 +1,59 @@ +{ + "3": { + "class_type": "KSampler", + "inputs": { + "seed": 0, + "steps": 20, + "cfg": 7, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": ["4", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["5", 0] + } + }, + "4": { + "class_type": "CheckpointLoaderSimple", + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.safetensors" + } + }, + "5": { + "class_type": "EmptyLatentImage", + "inputs": { + "width": 1024, + "height": 1024, + "batch_size": 1 + } + }, + "6": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "", + "clip": ["4", 1] + } + }, + "7": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "", + "clip": ["4", 1] + } + }, + "8": { + "class_type": "VAEDecode", + "inputs": { + "samples": ["3", 0], + "vae": ["4", 2] + } + }, + "9": { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + } + } +} diff --git a/workflows/txt2img.nodes.json b/workflows/txt2img.nodes.json new file mode 100644 index 0000000..5557420 --- /dev/null +++ b/workflows/txt2img.nodes.json @@ -0,0 +1,10 @@ +[ + { "type": "model", "node_ids": ["4"], "key": "ckpt_name" }, + { "type": "prompt", "node_ids": ["6"], "key": "text" }, + { "type": "negative_prompt", "node_ids": ["7"], "key": "text" }, + { "type": "width", "node_ids": ["5"], "key": "width" }, + { "type": "height", "node_ids": ["5"], "key": "height" }, + { "type": "n", "node_ids": ["5"], "key": "batch_size" }, + { "type": "steps", "node_ids": ["3"], "key": "steps" }, + { "type": "seed", "node_ids": ["3"], "key": "seed" } +]