Add text-targeted inpainting via GroundingDINO+SAM (mask_text param)

Five pieces:

1. Dockerfile installs storyicon/comfyui_segment_anything (GroundingDINO
   + SAM-HQ in one bundle) into custom_nodes and pip-installs its
   requirements at build time. Model weights auto-download to the
   comfyui-models volume on first inpaint (~3 GB one-time cost).

2. install-custom-node-deps.sh — entrypoint wrapper that pip-installs
   requirements.txt for any custom_node present at startup. Lets users
   add custom nodes via ComfyUI-Manager (or by git-cloning into the
   volume) and have the deps picked up on the next restart, without
   editing the Dockerfile.

3. smart_image_gen v0.6: edit_image gains a `mask_text` param. When
   set, builds an inpainting workflow (LoadImage → GroundingDinoSAM
   Segment → SetLatentNoiseMask → KSampler) so only the named region
   is repainted. When unset, falls through to the existing img2img
   path. Denoise default switches: 1.0 with mask_text (full repaint
   within mask), 0.7 without.

4. Image Studio system prompt teaches the LLM the LOCAL vs GLOBAL
   distinction — set mask_text whenever the user names a specific
   object/region ('the ball', 'the dog', 'the sky'); leave it unset
   only for whole-image style/lighting transformations.

5. Deployment README documents the new mode + the first-inpaint
   weight-download caveat.

Image rebuild required — bump tag to pick up the Dockerfile change.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

deployments/ai-stack/openwebui-tools/smart_image_gen.py

@@ -1,7 +1,7 @@
 """
 title: Smart Image Generator & Editor (ComfyUI)
 author: ai-stack
-version: 0.5.0
+version: 0.6.0
 description: Generate or edit images via ComfyUI with automatic SDXL
     checkpoint routing. Two methods — generate_image (txt2img) and
     edit_image (img2img on the user's most recently attached image). The
@@ -254,6 +254,58 @@ def _build_txt2img(positive: str, negative: str, settings: dict,
     }
 
 
+def _build_inpaint(positive: str, negative: str, settings: dict,
+                   image_filename: str, mask_text: str,
+                   denoise: float, seed: int) -> dict:
+    """
+    SDXL inpainting workflow with text-driven masking. Uses
+    comfyui_segment_anything (GroundingDINO + SAM-HQ — installed by the
+    Dockerfile) to derive a mask from `mask_text` (a noun phrase like
+    "the dog's collar"), then SetLatentNoiseMask + KSampler repaint
+    only that region. Everything outside the mask stays pixel-perfect.
+
+    First inpaint downloads ~3 GB of SAM/GroundingDINO weights into
+    /opt/comfyui/models/{sams,grounding-dino}/ — subsequent runs reuse
+    them.
+    """
+    return {
+        "3": {"class_type": "KSampler", "inputs": {
+            "seed": _seed_value(seed),
+            "steps": settings["steps"], "cfg": settings["cfg"],
+            "sampler_name": settings["sampler"], "scheduler": settings["scheduler"],
+            "denoise": denoise,
+            "model": ["4", 0], "positive": ["6", 0],
+            "negative": ["7", 0], "latent_image": ["13", 0],
+        }},
+        "4": {"class_type": "CheckpointLoaderSimple",
+              "inputs": {"ckpt_name": settings["ckpt"]}},
+        "6": {"class_type": "CLIPTextEncode", "inputs": {"text": positive, "clip": ["10", 0]}},
+        "7": {"class_type": "CLIPTextEncode", "inputs": {"text": negative, "clip": ["10", 0]}},
+        "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["4", 2]}},
+        "9": {"class_type": "SaveImage",
+              "inputs": {"filename_prefix": "smartinpaint", "images": ["8", 0]}},
+        "10": {"class_type": "CLIPSetLastLayer",
+               "inputs": {"stop_at_clip_layer": -settings["clip_skip"],
+                          "clip": ["4", 1]}},
+        "11": {"class_type": "VAEEncode", "inputs": {"pixels": ["12", 0], "vae": ["4", 2]}},
+        "12": {"class_type": "LoadImage", "inputs": {"image": image_filename}},
+        "13": {"class_type": "SetLatentNoiseMask",
+               "inputs": {"samples": ["11", 0], "mask": ["16", 1]}},
+        "14": {"class_type": "SAMModelLoader (segment anything)",
+               "inputs": {"model_name": "sam_hq_vit_h (2.57GB)"}},
+        "15": {"class_type": "GroundingDinoModelLoader (segment anything)",
+               "inputs": {"model_name": "GroundingDINO_SwinT_OGC (694MB)"}},
+        "16": {"class_type": "GroundingDinoSAMSegment (segment anything)",
+               "inputs": {
+                   "sam_model": ["14", 0],
+                   "grounding_dino_model": ["15", 0],
+                   "image": ["12", 0],
+                   "prompt": mask_text,
+                   "threshold": 0.3,
+               }},
+    }
+
+
 def _build_img2img(positive: str, negative: str, settings: dict,
                    image_filename: str, denoise: float, seed: int) -> dict:
     """
@@ -595,7 +647,8 @@ class Tools:
         self,
         edit_instruction: str,
         style: Optional[StyleName] = None,
-        denoise: float = 0.7,
+        mask_text: Optional[str] = None,
+        denoise: Optional[float] = None,
         negative_prompt: Optional[str] = None,
         seed: int = 0,
         __request__=None,
@@ -607,31 +660,44 @@ class Tools:
     ) -> str:
         """
         Edit, modify, transform, or restyle an image the user has ATTACHED
-        to the chat. Use this whenever the user uploads or attaches an
-        image and asks to change it — "make this a sunset", "add a hat",
-        "turn this into anime", "remove the background", etc. If no image
-        is attached, use generate_image instead.
+        to the chat. Use whenever the user uploads an image and asks to
+        change it. If no image is attached, use generate_image instead.
 
-        The tool finds the most recent attached image automatically, picks
-        the right SDXL checkpoint via `style` (same options as
-        generate_image), and applies the edit at the requested `denoise`.
+        TWO MODES — choose based on whether the change is local or global:
+
+        - LOCAL change ("change the ball to a basketball", "make the dog
+          wear a hat", "remove the bird") → set `mask_text` to a brief
+          noun phrase describing the region ("the ball", "the dog", "the
+          bird"). The tool uses GroundingDINO+SAM to find that region
+          automatically and only that area is repainted; the rest of the
+          image stays pixel-perfect.
+
+        - GLOBAL change ("make this a sunset", "turn this into anime",
+          "restyle this as oil painting") → leave `mask_text` unset. The
+          whole image is reimagined via img2img.
+
+        Always prefer LOCAL mode when the user names a specific object,
+        person, or region. GLOBAL mode is for whole-image style/lighting
+        transformations.
 
         Denoise tuning:
-        - 0.3 to 0.5 → subtle (style transfer, color tweaks, preserve composition)
-        - 0.6 to 0.8 → moderate (add/remove objects, lighting changes — default)
-        - 0.85 to 1.0 → heavy (basically reimagining; loses original details)
+        - LOCAL (mask_text set): default 1.0 — full repaint within mask.
+          Drop to 0.6–0.8 for subtle local edits that should retain some
+          original structure.
+        - GLOBAL (no mask_text): default 0.7 — moderate edit. Use 0.3–0.5
+          for subtle restyling, 0.85–1.0 for radical reimagining.
 
-        Pick `style` based on the desired output, NOT the input image.
-        Same options as generate_image: photo, juggernaut, pony, general,
-        furry-nai, furry-noob, furry-il.
+        Pick `style` for the DESIRED OUTPUT, not the input image.
 
-        :param edit_instruction: What to change. The tool auto-prepends the
-            style's quality tags so don't include those.
-        :param style: One of the StyleName values. Omit to auto-detect from
-            edit_instruction.
-        :param denoise: 0.0 = no change, 1.0 = ignore source. Default 0.7.
+        :param edit_instruction: What the changed area should look like.
+            Tool auto-prepends quality tags — don't include those.
+        :param style: One of the StyleName values. Omit to auto-detect.
+        :param mask_text: Noun phrase describing the region to edit. Set
+            for LOCAL changes; omit for GLOBAL.
+        :param denoise: 0.0 = no change, 1.0 = ignore source. Defaults to
+            1.0 with mask_text, 0.7 without.
         :param negative_prompt: Extra terms to exclude. Usually unneeded.
-        :param seed: 0 to randomize, otherwise a specific seed.
+        :param seed: 0 to randomize, otherwise specific.
         :return: Markdown image of the result, or an error if no image is attached.
         """
         chosen = style or _route_style(edit_instruction)
@@ -639,6 +705,10 @@ class Tools:
         if not settings:
             return f"Unknown style '{chosen}'. Available: {', '.join(STYLES.keys())}"
 
+        # Denoise default depends on mode: 1.0 (full repaint within mask)
+        # for inpainting, 0.7 for img2img.
+        if denoise is None:
+            denoise = 1.0 if mask_text else 0.7
         denoise = max(0.0, min(1.0, denoise))
 
         async def emit(msg: str, done: bool = False):
@@ -665,8 +735,10 @@ class Tools:
             if not uploaded_name:
                 return "Failed to upload source image to ComfyUI."
 
+            mode = "inpaint" if mask_text else "img2img"
             await emit(
-                f"Routing to {chosen} ({settings['ckpt']}), denoise {denoise:.2f}"
+                f"Routing to {chosen} ({settings['ckpt']}), {mode}, denoise {denoise:.2f}"
+                + (f", mask='{mask_text}'" if mask_text else "")
             )
 
             positive = f"{settings['prefix']}{edit_instruction}"
@@ -674,14 +746,25 @@ class Tools:
             if negative_prompt:
                 negative = f"{negative}, {negative_prompt}"
 
-            workflow = _build_img2img(
-                positive=positive,
-                negative=negative,
-                settings=settings,
-                image_filename=uploaded_name,
-                denoise=denoise,
-                seed=seed,
-            )
+            if mask_text:
+                workflow = _build_inpaint(
+                    positive=positive,
+                    negative=negative,
+                    settings=settings,
+                    image_filename=uploaded_name,
+                    mask_text=mask_text,
+                    denoise=denoise,
+                    seed=seed,
+                )
+            else:
+                workflow = _build_img2img(
+                    positive=positive,
+                    negative=negative,
+                    settings=settings,
+                    image_filename=uploaded_name,
+                    denoise=denoise,
+                    seed=seed,
+                )
 
             raw_out, err = await _submit_and_fetch(
                 session, base, workflow, self.valves.TIMEOUT_SECONDS, emit, settings,