Make caption refusal markers configurable

examples/worker.openai-compatible.yaml

@@ -13,6 +13,13 @@ worker:
     image_format: "jpeg"
     image_quality: 90
 
+    # Optional local override for refusal detection. An empty list disables
+    # text-based refusal detection. This takes precedence over the shared
+    # orchestrator inference.refusal_markers setting.
+    # refusal_markers:
+    #   - "i cannot describe"
+    #   - "unable to provide a caption"
+
     endpoints:
       - name: "zai-coding-plan"
         base_url: "https://api.z.ai/api/coding/paas/v4"

recipes/GLM_5.3_FLASH.md

@@ -192,6 +192,13 @@ the OpenAI-compatible worker can make one text-only fallback attempt:
 ```yaml
 orchestrator:
   inference:
+    # Override the built-in defaults when literal image text could resemble a
+    # refusal. An empty list disables text-based refusal detection entirely.
+    refusal_markers:
+      - "i cannot describe"
+      - "unable to describe"
+      - "cannot provide a caption"
+      - "unable to provide a caption"
     retry_prompt: >-
       Rewrite the following source description as one concise, neutral,
       standalone caption. Preserve only concrete visual details, omit

src/caption_flow/utils/vllm_config.py

@@ -41,6 +41,7 @@ class VLLMConfigManager:
         "batch_size",
         "sampling",
         "inference_prompts",
+        "refusal_markers",
         "retry_prompt",
         "retry_without_image",
     }

src/caption_flow/workers/openai_compatible.py

@@ -566,7 +566,7 @@ class OpenAICompatibleWorker(CaptionWorker):
         "presence_penalty",
         "seed",
     }
-    _REFUSAL_MARKERS = (
+    _DEFAULT_REFUSAL_MARKERS = (
         "i'm sorry",
         "i’m sorry",
         "i cannot",
@@ -621,6 +621,16 @@ def __init__(self, config: Dict[str, Any]):
         self.image_quality = int(raw_config.get("image_quality", 90))
         configured_dimension = int(raw_config.get("max_image_dimension", 0) or 0)
         self.max_image_dimension = configured_dimension if configured_dimension > 0 else None
+        self._local_refusal_markers = (
+            self._normalize_refusal_markers(raw_config["refusal_markers"])
+            if "refusal_markers" in raw_config
+            else None
+        )
+        self.refusal_markers = (
+            self._local_refusal_markers
+            if self._local_refusal_markers is not None
+            else self._DEFAULT_REFUSAL_MARKERS
+        )
 
     async def _pre_start(self):
         """Fetch shared stage settings, then start the API processing thread."""
@@ -652,6 +662,14 @@ def _apply_local_overrides(self) -> None:
         if batch_size is None:
             batch_size = sum(state.config.max_concurrency for state in self.endpoint_pool.states)
         self.vllm_config["batch_size"] = max(1, int(batch_size))
+        if self._local_refusal_markers is not None:
+            self.refusal_markers = self._local_refusal_markers
+        elif "refusal_markers" in self.vllm_config:
+            self.refusal_markers = self._normalize_refusal_markers(
+                self.vllm_config["refusal_markers"]
+            )
+        else:
+            self.refusal_markers = self._DEFAULT_REFUSAL_MARKERS
 
     def _handle_vllm_config_update(self, new_config: Dict[str, Any]) -> bool:
         """Apply shared prompt/stage changes without touching local credentials."""
@@ -822,10 +840,17 @@ def _process_batch_multi_stage(  # noqa: C901
             self.items_processed += 1
         return results
 
-    @classmethod
-    def _is_refusal_text(cls, text: str) -> bool:
-        normalized = text.strip().lower()
-        return any(marker in normalized for marker in cls._REFUSAL_MARKERS)
+    @staticmethod
+    def _normalize_refusal_markers(value: Any) -> Tuple[str, ...]:
+        if not isinstance(value, list) or any(
+            not isinstance(marker, str) or not marker.strip() for marker in value
+        ):
+            raise ValueError("refusal_markers must be a list of non-empty strings")
+        return tuple(dict.fromkeys(marker.strip().casefold() for marker in value))
+
+    def _is_refusal_text(self, text: str) -> bool:
+        normalized = text.strip().casefold()
+        return any(marker in normalized for marker in self.refusal_markers)
 
     @classmethod
     def _is_semantic_caption_failure(cls, error: Exception) -> bool:

tests/test_openai_compatible_worker.py

@@ -10,8 +10,8 @@
 from click.testing import CliRunner
 from PIL import Image
 
-from caption_flow.workers.caption import ProcessingItem
 from caption_flow.utils.image_processor import ImageProcessor
+from caption_flow.workers.caption import ProcessingItem
 from caption_flow.workers.openai_compatible import (
     AdaptiveEndpointPool,
     ChatRequest,
@@ -579,6 +579,18 @@ def test_worker_validates_config_and_applies_shared_updates():
     with pytest.raises(ValueError, match="list of mappings"):
         OpenAICompatibleWorker({**base, "openai_compatible": {"endpoints": ["invalid"]}})
 
+    with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}):
+        with pytest.raises(ValueError, match="list of non-empty strings"):
+            OpenAICompatibleWorker(
+                {
+                    **base,
+                    "openai_compatible": {
+                        "api_key_env": "CAPTIONFLOW_TEST_API_KEY",
+                        "refusal_markers": "i cannot describe",
+                    },
+                }
+            )
+
     config = {
         **base,
         "openai_compatible": {
@@ -604,6 +616,68 @@ def test_worker_validates_config_and_applies_shared_updates():
         worker._process_batch_multi_stage([])
 
 
+def test_worker_configures_refusal_markers_from_shared_or_local_config():
+    base = {
+        "server": "ws://localhost:8765",
+        "token": "orchestrator-token",
+        "openai_compatible": {
+            "api_key_env": "CAPTIONFLOW_TEST_API_KEY",
+            "model": "vision",
+        },
+    }
+    with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}):
+        worker = OpenAICompatibleWorker(base)
+
+    assert worker._is_refusal_text("I'm sorry, I can't describe this image.")
+    assert worker._is_refusal_text("A shirt reading “I can't stay at home.”")
+
+    assert worker._handle_vllm_config_update(
+        {
+            "model": "vision",
+            "inference_prompts": ["Describe"],
+            "refusal_markers": [
+                " I cannot describe ",
+                "i cannot describe",
+                "unable to provide a caption",
+            ],
+        }
+    )
+    assert worker.refusal_markers == (
+        "i cannot describe",
+        "unable to provide a caption",
+    )
+    assert worker._is_refusal_text("I cannot describe this image.")
+    assert not worker._is_refusal_text("A shirt reading “I can't stay at home.”")
+
+    local_config = {
+        **base,
+        "openai_compatible": {
+            **base["openai_compatible"],
+            "refusal_markers": [],
+        },
+    }
+    with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}):
+        local_worker = OpenAICompatibleWorker(local_config)
+    assert local_worker._handle_vllm_config_update(
+        {
+            "model": "vision",
+            "inference_prompts": ["Describe"],
+            "refusal_markers": ["i cannot describe"],
+        }
+    )
+    assert local_worker.refusal_markers == ()
+    assert not local_worker._is_refusal_text("I cannot describe this image.")
+
+    with pytest.raises(ValueError, match="list of non-empty strings"):
+        worker._handle_vllm_config_update(
+            {
+                "model": "vision",
+                "inference_prompts": ["Describe"],
+                "refusal_markers": [""],
+            }
+        )
+
+
 def test_worker_runs_shared_caption_stage_through_endpoint_pool():
     worker_config = {
         "server": "ws://localhost:8765",

tests/test_vllm_config.py

@@ -3,6 +3,7 @@
 from unittest.mock import Mock, patch
 
 import pytest
+
 from caption_flow.utils.vllm_config import VLLMConfigChange, VLLMConfigManager
 
 
@@ -55,6 +56,7 @@ def test_class_constants(self):
         assert "tensor_parallel_size" in VLLMConfigManager.RELOAD_REQUIRED_FIELDS
         assert "batch_size" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS
         assert "sampling" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS
+        assert "refusal_markers" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS
         assert "retry_prompt" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS
         assert "retry_without_image" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS