diff --git a/examples/worker.openai-compatible.yaml b/examples/worker.openai-compatible.yaml index 3858afe..f39d232 100644 --- a/examples/worker.openai-compatible.yaml +++ b/examples/worker.openai-compatible.yaml @@ -13,6 +13,13 @@ worker: image_format: "jpeg" image_quality: 90 + # Optional local override for refusal detection. An empty list disables + # text-based refusal detection. This takes precedence over the shared + # orchestrator inference.refusal_markers setting. + # refusal_markers: + # - "i cannot describe" + # - "unable to provide a caption" + endpoints: - name: "zai-coding-plan" base_url: "https://api.z.ai/api/coding/paas/v4" diff --git a/recipes/GLM_5.3_FLASH.md b/recipes/GLM_5.3_FLASH.md index e8eb812..5cff59b 100644 --- a/recipes/GLM_5.3_FLASH.md +++ b/recipes/GLM_5.3_FLASH.md @@ -192,6 +192,13 @@ the OpenAI-compatible worker can make one text-only fallback attempt: ```yaml orchestrator: inference: + # Override the built-in defaults when literal image text could resemble a + # refusal. An empty list disables text-based refusal detection entirely. + refusal_markers: + - "i cannot describe" + - "unable to describe" + - "cannot provide a caption" + - "unable to provide a caption" retry_prompt: >- Rewrite the following source description as one concise, neutral, standalone caption. Preserve only concrete visual details, omit diff --git a/src/caption_flow/utils/vllm_config.py b/src/caption_flow/utils/vllm_config.py index 1a28459..c818767 100644 --- a/src/caption_flow/utils/vllm_config.py +++ b/src/caption_flow/utils/vllm_config.py @@ -41,6 +41,7 @@ class VLLMConfigManager: "batch_size", "sampling", "inference_prompts", + "refusal_markers", "retry_prompt", "retry_without_image", } diff --git a/src/caption_flow/workers/openai_compatible.py b/src/caption_flow/workers/openai_compatible.py index 83b6b08..2bd850d 100644 --- a/src/caption_flow/workers/openai_compatible.py +++ b/src/caption_flow/workers/openai_compatible.py @@ -566,7 +566,7 @@ class OpenAICompatibleWorker(CaptionWorker): "presence_penalty", "seed", } - _REFUSAL_MARKERS = ( + _DEFAULT_REFUSAL_MARKERS = ( "i'm sorry", "i’m sorry", "i cannot", @@ -621,6 +621,16 @@ def __init__(self, config: Dict[str, Any]): self.image_quality = int(raw_config.get("image_quality", 90)) configured_dimension = int(raw_config.get("max_image_dimension", 0) or 0) self.max_image_dimension = configured_dimension if configured_dimension > 0 else None + self._local_refusal_markers = ( + self._normalize_refusal_markers(raw_config["refusal_markers"]) + if "refusal_markers" in raw_config + else None + ) + self.refusal_markers = ( + self._local_refusal_markers + if self._local_refusal_markers is not None + else self._DEFAULT_REFUSAL_MARKERS + ) async def _pre_start(self): """Fetch shared stage settings, then start the API processing thread.""" @@ -652,6 +662,14 @@ def _apply_local_overrides(self) -> None: if batch_size is None: batch_size = sum(state.config.max_concurrency for state in self.endpoint_pool.states) self.vllm_config["batch_size"] = max(1, int(batch_size)) + if self._local_refusal_markers is not None: + self.refusal_markers = self._local_refusal_markers + elif "refusal_markers" in self.vllm_config: + self.refusal_markers = self._normalize_refusal_markers( + self.vllm_config["refusal_markers"] + ) + else: + self.refusal_markers = self._DEFAULT_REFUSAL_MARKERS def _handle_vllm_config_update(self, new_config: Dict[str, Any]) -> bool: """Apply shared prompt/stage changes without touching local credentials.""" @@ -822,10 +840,17 @@ def _process_batch_multi_stage( # noqa: C901 self.items_processed += 1 return results - @classmethod - def _is_refusal_text(cls, text: str) -> bool: - normalized = text.strip().lower() - return any(marker in normalized for marker in cls._REFUSAL_MARKERS) + @staticmethod + def _normalize_refusal_markers(value: Any) -> Tuple[str, ...]: + if not isinstance(value, list) or any( + not isinstance(marker, str) or not marker.strip() for marker in value + ): + raise ValueError("refusal_markers must be a list of non-empty strings") + return tuple(dict.fromkeys(marker.strip().casefold() for marker in value)) + + def _is_refusal_text(self, text: str) -> bool: + normalized = text.strip().casefold() + return any(marker in normalized for marker in self.refusal_markers) @classmethod def _is_semantic_caption_failure(cls, error: Exception) -> bool: diff --git a/tests/test_openai_compatible_worker.py b/tests/test_openai_compatible_worker.py index bd1f669..ad471f0 100644 --- a/tests/test_openai_compatible_worker.py +++ b/tests/test_openai_compatible_worker.py @@ -10,8 +10,8 @@ from click.testing import CliRunner from PIL import Image -from caption_flow.workers.caption import ProcessingItem from caption_flow.utils.image_processor import ImageProcessor +from caption_flow.workers.caption import ProcessingItem from caption_flow.workers.openai_compatible import ( AdaptiveEndpointPool, ChatRequest, @@ -579,6 +579,18 @@ def test_worker_validates_config_and_applies_shared_updates(): with pytest.raises(ValueError, match="list of mappings"): OpenAICompatibleWorker({**base, "openai_compatible": {"endpoints": ["invalid"]}}) + with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}): + with pytest.raises(ValueError, match="list of non-empty strings"): + OpenAICompatibleWorker( + { + **base, + "openai_compatible": { + "api_key_env": "CAPTIONFLOW_TEST_API_KEY", + "refusal_markers": "i cannot describe", + }, + } + ) + config = { **base, "openai_compatible": { @@ -604,6 +616,68 @@ def test_worker_validates_config_and_applies_shared_updates(): worker._process_batch_multi_stage([]) +def test_worker_configures_refusal_markers_from_shared_or_local_config(): + base = { + "server": "ws://localhost:8765", + "token": "orchestrator-token", + "openai_compatible": { + "api_key_env": "CAPTIONFLOW_TEST_API_KEY", + "model": "vision", + }, + } + with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}): + worker = OpenAICompatibleWorker(base) + + assert worker._is_refusal_text("I'm sorry, I can't describe this image.") + assert worker._is_refusal_text("A shirt reading “I can't stay at home.”") + + assert worker._handle_vllm_config_update( + { + "model": "vision", + "inference_prompts": ["Describe"], + "refusal_markers": [ + " I cannot describe ", + "i cannot describe", + "unable to provide a caption", + ], + } + ) + assert worker.refusal_markers == ( + "i cannot describe", + "unable to provide a caption", + ) + assert worker._is_refusal_text("I cannot describe this image.") + assert not worker._is_refusal_text("A shirt reading “I can't stay at home.”") + + local_config = { + **base, + "openai_compatible": { + **base["openai_compatible"], + "refusal_markers": [], + }, + } + with patch.dict(os.environ, {"CAPTIONFLOW_TEST_API_KEY": "secret"}): + local_worker = OpenAICompatibleWorker(local_config) + assert local_worker._handle_vllm_config_update( + { + "model": "vision", + "inference_prompts": ["Describe"], + "refusal_markers": ["i cannot describe"], + } + ) + assert local_worker.refusal_markers == () + assert not local_worker._is_refusal_text("I cannot describe this image.") + + with pytest.raises(ValueError, match="list of non-empty strings"): + worker._handle_vllm_config_update( + { + "model": "vision", + "inference_prompts": ["Describe"], + "refusal_markers": [""], + } + ) + + def test_worker_runs_shared_caption_stage_through_endpoint_pool(): worker_config = { "server": "ws://localhost:8765", diff --git a/tests/test_vllm_config.py b/tests/test_vllm_config.py index edaea01..a9cac4e 100644 --- a/tests/test_vllm_config.py +++ b/tests/test_vllm_config.py @@ -3,6 +3,7 @@ from unittest.mock import Mock, patch import pytest + from caption_flow.utils.vllm_config import VLLMConfigChange, VLLMConfigManager @@ -55,6 +56,7 @@ def test_class_constants(self): assert "tensor_parallel_size" in VLLMConfigManager.RELOAD_REQUIRED_FIELDS assert "batch_size" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS assert "sampling" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS + assert "refusal_markers" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS assert "retry_prompt" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS assert "retry_without_image" in VLLMConfigManager.RUNTIME_UPDATEABLE_FIELDS