support openrouter as an llm provider

composer/input/files.py

@@ -34,7 +34,21 @@
 
 class ContentRenderer(Protocol):
     def text_block(self, text: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict: ...
-    def file_block(self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict: ...
+
+    def file_block(self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict:
+        """Reference a Files-API upload by id. Providers without a Files API
+        raise ``NotImplementedError`` and produce
+        :class:`InMemoryBytesFile` instead."""
+        ...
+
+    def inline_file_block(
+        self, basename: str, contents: bytes, mime: str,
+        *, cache_level: CacheLevel = CacheLevel.NONE
+    ) -> dict:
+        """Carry the bytes in the request itself, for providers with no Files
+        API. The mirror of :meth:`file_block`; a provider implements one or the
+        other, matching what its uploader produces."""
+        ...
 
 # ---------------------------------------------------------------------------
 # Protocols (the public surface)
@@ -207,6 +221,38 @@ def to_digest(self) -> str:
         return _bytes_digest(self.bytes_contents)
 
 
+@dataclass(frozen=True)
+class InMemoryBytesFile:
+    """Binary content carried inline in the request, for a provider with no
+    Files API. The binary analogue of :class:`InMemoryTextFile`: the bytes ride
+    along in every request that carries the document rather than being uploaded
+    once and referenced by id."""
+
+    basename: str
+    contents: bytes
+    mime: str
+    renderer: ContentRenderer
+
+    def to_dict(self, cache_level: CacheLevel = CacheLevel.NONE) -> dict:
+        return self.renderer.inline_file_block(
+            self.basename, self.contents, self.mime, cache_level=cache_level
+        )
+
+    def to_digest(self) -> str:
+        return _bytes_digest(self.contents)
+
+    @property
+    def bytes_contents(self) -> bytes:
+        return self.contents
+
+    @property
+    def string_contents(self) -> str | None:
+        try:
+            return self.contents.decode("utf-8")
+        except UnicodeDecodeError:
+            return None
+
+
 @dataclass(frozen=True)
 class UploadedFile:
     """A (potentially-binary) file uploaded to the Files API. Bytes are
@@ -259,7 +305,7 @@ def string_contents(self) -> str:
 # ---------------------------------------------------------------------------
 
 @dataclass
-class _FileData:
+class FileData:
     basename: str
     raw_data: bytes
     is_binary: bool
@@ -271,28 +317,28 @@ class _FileData:
 async def _file_data(
     *,
     path: str | pathlib.Path
-) -> _FileData:
+) -> FileData:
     ...
 
 @overload
 async def _file_data(
     *,
     basename: str, data: bytes
-) -> _FileData:
+) -> FileData:
     ...
 
 async def _file_data(
     path: str | pathlib.Path | None = None,
     basename: str | None = None,
     data: bytes | None = None
-) -> _FileData:
+) -> FileData:
     return await asyncio.to_thread(_file_data_impl, path, basename, data)
 
 def _file_data_impl(
     path: str | pathlib.Path | None,
     basename: str | None,
     data: bytes | None
-) -> _FileData:
+) -> FileData:
     if path is not None:
         if isinstance(path, str):
             path = pathlib.Path(path)
@@ -315,18 +361,18 @@ def _file_data_impl(
         mime = "application/octet-stream" if is_binary else "text/plain"
     crc = hex(zlib.crc32(data))
     digest = _bytes_digest(data)
-    return _FileData(raw_data=data, is_binary=is_binary, mime=mime, crc_basename=f"{crc}_{basename}", digest=digest, basename=basename)
+    return FileData(raw_data=data, is_binary=is_binary, mime=mime, crc_basename=f"{crc}_{basename}", digest=digest, basename=basename)
 
 class FileUploader(Protocol):
     """Upload+dedup contract. Obtain via ``ModelProvider.uploader()`` (``composer.llm``)."""
 
     async def upload_file_if_needed(
         self, file_path: str | pathlib.Path
-    ) -> UploadedFile: ...
+    ) -> Document: ...
 
     async def upload_text_file_if_needed(
         self, file_path: str | pathlib.Path
-    ) -> UploadedTextFile: ...
+    ) -> TextDocument: ...
 
     async def get_document(
         self, path: str | pathlib.Path
@@ -350,7 +396,11 @@ class UploaderBase(ABC):
     The dedup cache lives in ``self.uploaded`` (CRC-prefixed filename →
     remote file id) and is seeded by each subclass's ``fresh`` factory
     so we don't reupload a file whose bytes the account has already
-    seen."""
+    seen.
+
+    A provider with no Files API overrides :meth:`_binary_document` and
+    :meth:`_text_upload_document` to return inline shapes instead, and never
+    implements ``_upload_bytes``."""
 
     renderer: ContentRenderer
 
@@ -360,14 +410,9 @@ async def _upload_bytes(
     ) -> str:
         ...
 
-    async def upload_file_if_needed(
-        self, file_path: str | pathlib.Path
-    ) -> UploadedFile:
-        """Upload ``file_path`` (or reuse cached upload). Intended for
-        binary inputs — callers that know they have text should prefer
-        :meth:`get_document` (default text-inline) or
-        :meth:`upload_text_file_if_needed` (explicit upload of text)."""
-        data = await _file_data(path=file_path)
+    async def _binary_document(self, data: FileData) -> Document:
+        """How this provider represents binary content. Uploads by default;
+        override to inline the bytes instead."""
         file_id = await self._upload_bytes(data.crc_basename, data.raw_data, data.mime)
         return UploadedFile(
             file_id=file_id,
@@ -377,15 +422,9 @@ async def upload_file_if_needed(
             renderer=self.renderer,
         )
 
-    async def upload_text_file_if_needed(
-        self, file_path: str | pathlib.Path
-    ) -> UploadedTextFile:
-        """Upload ``file_path`` and tag the result as text. Use for
-        very-large text inputs that would otherwise blow the prompt
-        budget if inlined; ordinary text should go through
-        :meth:`get_document`, which keeps the content in-prompt for
-        transcript debuggability."""
-        data = await _file_data(path=file_path)
+    async def _text_upload_document(self, data: FileData) -> TextDocument:
+        """How this provider represents text explicitly destined for upload.
+        Uploads by default; override to keep it in the prompt instead."""
         file_id = await self._upload_bytes(data.crc_basename, data.raw_data, data.mime)
         return UploadedTextFile(
             file_id=file_id,
@@ -395,6 +434,25 @@ async def upload_text_file_if_needed(
             renderer=self.renderer,
         )
 
+    async def upload_file_if_needed(
+        self, file_path: str | pathlib.Path
+    ) -> Document:
+        """Upload ``file_path`` (or reuse cached upload). Intended for
+        binary inputs — callers that know they have text should prefer
+        :meth:`get_document` (default text-inline) or
+        :meth:`upload_text_file_if_needed` (explicit upload of text)."""
+        return await self._binary_document(await _file_data(path=file_path))
+
+    async def upload_text_file_if_needed(
+        self, file_path: str | pathlib.Path
+    ) -> TextDocument:
+        """Upload ``file_path`` and tag the result as text. Use for
+        very-large text inputs that would otherwise blow the prompt
+        budget if inlined; ordinary text should go through
+        :meth:`get_document`, which keeps the content in-prompt for
+        transcript debuggability."""
+        return await self._text_upload_document(await _file_data(path=file_path))
+
     async def get_document(
         self, path: str | pathlib.Path
     ) -> Document | None:
@@ -413,14 +471,7 @@ async def get_document(
             return None
         data = await _file_data(path=p)
         if data.is_binary:
-            file_id = await self._upload_bytes(data.crc_basename, data.raw_data, data.mime)
-            return UploadedFile(
-                file_id=file_id,
-                basename=data.basename,
-                contents=data.raw_data,
-                digest=data.digest,
-                renderer=self.renderer
-            )
+            return await self._binary_document(data)
         return InMemoryTextFile(
             basename=p.name,
             string_contents=data.raw_data.decode("utf-8"),
@@ -429,18 +480,12 @@ async def get_document(
 
     async def upload_bytes_if_needed(
         self, basename: str, raw: bytes
-    ) -> UploadedFile:
+    ) -> Document:
         """Upload in-memory ``raw`` bytes (e.g. an audit-restored binary
         document) to the Files API, reusing a cached upload by CRC. The
         bytes-sourced analogue of :meth:`upload_file_if_needed`."""
-        data = await _file_data(basename=basename, data=raw)
-        file_id = await self._upload_bytes(data.crc_basename, data.raw_data, data.mime)
-        return UploadedFile(
-            file_id=file_id,
-            basename=data.basename,
-            contents=data.raw_data,
-            digest=data.digest,
-            renderer=self.renderer
+        return await self._binary_document(
+            await _file_data(basename=basename, data=raw)
         )
 
     def text_document_from(self, src: TextUploadable) -> TextDocument:

composer/llm/anthropic.py

@@ -14,7 +14,7 @@
 from composer.input.files import UploaderBase, ContentRenderer
 from composer.input.types import ModelConfiguration
 from composer.llm.provider import (
-    ProviderServiceBase, ProviderSpec, compaction_threshold
+    ProviderServiceBase, ProviderSpec, compaction_threshold, standard_callbacks
 )
 from composer.llm.pricing import PriceProvider, price_provider_for
 from .types import CacheLevel
@@ -139,7 +139,9 @@ def text_block(self, text: str, *, cache_level: CacheLevel = CacheLevel.NONE) ->
             }
         return to_ret
 
-    def file_block(self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict:
+    def file_block(
+        self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE
+    ) -> dict:
         to_ret : dict[str, Any] = {
             "type": "document",
             "source": {
@@ -154,6 +156,14 @@ def file_block(self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE)
             }
         return to_ret
 
+    def inline_file_block(
+        self, basename: str, contents: bytes, mime: str,
+        *, cache_level: CacheLevel = CacheLevel.NONE
+    ) -> dict:
+        raise NotImplementedError(
+            "Anthropic content is uploaded to the Files API, not inlined."
+        )
+
 @cache
 def _get_service():
     return AnthropicService()
@@ -302,8 +312,6 @@ def builder_for(
         self, *, cache_level: CacheLevel = CacheLevel.NONE, disable_thinking: bool = False
     ) -> "BaseChatModel":
         from langchain_anthropic import ChatAnthropic
-        from composer.diagnostics.usage_callback import UsageCallback
-        from composer.diagnostics.cost_callback import CostAccumulator
 
         opts = self.options
         thinking: dict[str, Any] | None
@@ -347,12 +355,9 @@ def builder_for(
             betas=betas,
             thinking=thinking,
             model_kwargs=model_kwargs,
-            callbacks=[
-                UsageCallback(),
-                CostAccumulator(
-                    self.price_provider, long_cache=cache_level == CacheLevel.LONG
-                ),
-            ],
+            callbacks=standard_callbacks(
+                self.price_provider, long_cache=cache_level == CacheLevel.LONG
+            ),
         )
 
 ANTHROPIC_SPEC = ProviderSpec(

composer/llm/openai.py

@@ -22,7 +22,8 @@
 from composer.input.files import UploaderBase, ContentRenderer
 from composer.input.types import ModelConfiguration
 from .provider import (
-    ProviderServiceBase, ProviderSpec, compaction_threshold
+    ProviderServiceBase, ProviderSpec, compaction_threshold, reasoning_effort,
+    standard_callbacks
 )
 from .pricing import PriceProvider, price_provider_for
 from .types import CacheLevel
@@ -135,14 +136,6 @@ def _context_window(features: OpenAIModelFeatures) -> int:
     return _assumed_context_window
 
 
-def _reasoning_effort(thinking_tokens: int) -> Literal["low", "medium", "high"]:
-    """Map a thinking-token budget onto OpenAI's three-step effort knob."""
-    if thinking_tokens <= 2048:
-        return "low"
-    if thinking_tokens <= 8192:
-        return "medium"
-    return "high"
-
 class OpenAIService(ProviderServiceBase):
     def __init__(self):
         from graphcore.tools.memory import openai_async_memory_tool
@@ -168,14 +161,24 @@ class OpenAIRenderer:
     def text_block(self, text: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict:
         to_ret : dict[str, Any] = {"type": "text", "text": text}
         return to_ret
-    def file_block(self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict:
+    def file_block(
+        self, file_id: str, *, cache_level: CacheLevel = CacheLevel.NONE
+    ) -> dict:
         return {
             "type": "file",
             "file": {
                 "file_id": file_id,
             },
         }
 
+    def inline_file_block(
+        self, basename: str, contents: bytes, mime: str,
+        *, cache_level: CacheLevel = CacheLevel.NONE
+    ) -> dict:
+        raise NotImplementedError(
+            "OpenAI content is uploaded to the Files API, not inlined."
+        )
+
 # --- Files API uploader ----------------------------------------------------
 
 @dataclass
@@ -250,8 +253,6 @@ def builder_for(
         self, *, cache_level: CacheLevel = CacheLevel.NONE, disable_thinking: bool = False
     ) -> "BaseChatModel":
         from langchain_openai import ChatOpenAI
-        from composer.diagnostics.usage_callback import UsageCallback
-        from composer.diagnostics.cost_callback import CostAccumulator
 
         opts = self.options
         kwargs: dict[str, Any] = {
@@ -262,7 +263,7 @@ def builder_for(
 
         if opts.thinking_tokens is not None and not disable_thinking and self.features.reasoning:
             kwargs["reasoning"] = {
-                "effort": _reasoning_effort(opts.thinking_tokens),
+                "effort": reasoning_effort(opts.thinking_tokens),
                 "summary": "auto"
             }
 
@@ -273,7 +274,7 @@ def builder_for(
             max_retries=2,
             # OpenAI has no cache-TTL knob, so long_cache stays False; cache_write_1h
             # mirrors cache_write in the table anyway.
-            callbacks=[UsageCallback(), CostAccumulator(self.price_provider)],
+            callbacks=standard_callbacks(self.price_provider),
             **kwargs,
         )