From 4d996e1205fdb1bf5d470c86a6088923f612404f Mon Sep 17 00:00:00 2001 From: William Gill Date: Sun, 19 Apr 2026 13:14:59 -0500 Subject: [PATCH] smart_image_gen v0.4: emit image to chat, return only confirmation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data URI returned from the tool was being given to the LLM as the tool result — the LLM then either echoed the base64 to the user as plain text (screenshot 1) or hallucinated a description of what it thought the image looked like (screenshot 2 — "an image of a cat sitting on a windowsill" for a fox-warrior prompt). Fix: push the markdown image into the chat directly via __event_emitter__ as a "message" event, and return a short text confirmation as the function value. The confirmation is worded to prevent the LLM from describing the image or repeating the markdown (both common failure modes for tool-using LLMs). Both generate_image and edit_image fixed. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../openwebui-tools/smart_image_gen.py | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/deployments/ai-stack/openwebui-tools/smart_image_gen.py b/deployments/ai-stack/openwebui-tools/smart_image_gen.py index 049f27d..c4b14f7 100644 --- a/deployments/ai-stack/openwebui-tools/smart_image_gen.py +++ b/deployments/ai-stack/openwebui-tools/smart_image_gen.py @@ -1,14 +1,17 @@ """ title: Smart Image Generator & Editor (ComfyUI) author: ai-stack -version: 0.3.0 +version: 0.4.0 description: Generate or edit images via ComfyUI with automatic SDXL checkpoint routing. Two methods — generate_image (txt2img) and edit_image (img2img on the user's most recently attached image). The LLM picks (or auto-detects) the right model — photoreal, Pony score-tag, NoobAI/Illustrious furry, etc. — and each style ships with the creator-recommended sampler, scheduler, CFG, steps, CLIP - skip, prompt-prefix dialect, and negatives. + skip, prompt-prefix dialect, and negatives. The actual image is + pushed into the chat via the event emitter; the function return is + just a short confirmation so the LLM doesn't try to describe or + re-emit the data URI. required_open_webui_version: 0.5.0 """ @@ -483,8 +486,23 @@ class Tools: return err b64 = base64.b64encode(raw).decode("ascii") + # Push the image straight into the chat. The function return is just + # a confirmation for the LLM — if we returned the markdown, the LLM + # would either echo the base64 to the user as text, or hallucinate + # a description of what it thinks the image looks like. + if __event_emitter__: + await __event_emitter__({ + "type": "message", + "data": {"content": f"![{chosen}](data:image/png;base64,{b64})"}, + }) await emit(f"Done — {chosen}", done=True) - return f"![{chosen}](data:image/png;base64,{b64})" + return ( + f"Image generated and shown to the user above (style: {chosen}, " + f"checkpoint: {settings['ckpt']}). Do NOT describe the image, " + f"do NOT repeat any base64 or markdown — the user can see it. " + f"You may briefly note your style choice and offer one or two " + f"iteration ideas (different style, tighter framing, etc)." + ) async def edit_image( self, @@ -582,5 +600,16 @@ class Tools: return err b64 = base64.b64encode(raw_out).decode("ascii") + if __event_emitter__: + await __event_emitter__({ + "type": "message", + "data": {"content": f"![edit:{chosen}](data:image/png;base64,{b64})"}, + }) await emit(f"Done — {chosen} (denoise {denoise:.2f})", done=True) - return f"![edit:{chosen}](data:image/png;base64,{b64})" + return ( + f"Edited image shown to the user above (style: {chosen}, " + f"checkpoint: {settings['ckpt']}, denoise: {denoise:.2f}). Do NOT " + f"describe the image, do NOT repeat any base64 or markdown — the " + f"user can see it. You may briefly note your choice and offer " + f"iterations (different denoise, alternate style, etc)." + )