diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67cbb9430..c7e7c0d4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,11 @@ jobs: BAND_ALLOW_MISSING_FRAMEWORKS: "1" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run CrewAI activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + run: uv run --all-packages pytest tests/integrations/crewai/activity/ + - name: Run crewai tests env: # dev-crewai venv lacks langgraph/anthropic/parlant/pydantic-ai/etc.; @@ -171,7 +176,7 @@ jobs: # Only the tests that *need* this venv. Every other crewai test fakes the # package through sys.modules, so the `test` job above already runs it and # a second pass here would add no signal — just a path to forget. - # tests/framework_conformance/test_crewai_job_coverage.py keeps this honest. + # tests/framework_conformance/crewaicoverage/ keeps this honest. run: | uv run --all-packages pytest \ tests/adapters/test_crewai_flow_phase3.py \ @@ -233,6 +238,12 @@ jobs: BAND_TEST_EXTRA: "dev-strands-nooa" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run Strands activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + BAND_TEST_EXTRA: "dev-strands-nooa" + run: uv run --all-packages pytest tests/adapters/strands/activity/ + - name: Run Strands and NOOA tests env: # dev-strands-nooa lacks modern framework packages; @@ -302,6 +313,11 @@ jobs: BAND_ALLOW_MISSING_FRAMEWORKS: "1" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run Parlant activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + run: uv run --all-packages pytest tests/integrations/parlant/activity/ + - name: Run parlant tests env: # dev-parlant venv lacks langgraph/anthropic-adapter/pydantic-ai/etc.; @@ -310,7 +326,7 @@ jobs: # Only the tests that *need* this venv. Every other parlant test fakes the # package through sys.modules, so the `test` job above already runs it and # a second pass here would add no signal — just a path to forget. - # tests/framework_conformance/test_parlant_job_coverage.py keeps this honest. + # tests/framework_conformance/parlantcoverage/ keeps this honest. run: | uv run --all-packages pytest \ tests/integrations/parlant/test_catalog.py \ diff --git a/README.md b/README.md index abdb21f8d..6dc2cd29c 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,28 @@ adapter = GeminiAdapter(GeminiAdapterConfig(model="gemini-3.8-flash")) Use [examples/run_agent.py](examples/run_agent.py) when you want one command that can switch between LangGraph, Pydantic AI, Anthropic, Claude SDK, Parlant, CrewAI, Codex, A2A bridge, and A2A gateway. Use the per-framework directories under [examples/](examples/) when you want the adapter-specific setup. +### Host-side Harness Probes + +`await adapter.preflight(timeout_s=60)` reports a harness handshake as a +`PreflightResult` (`ok`, `reason`, `remedy`). A successful handshake does not prove +that the account can run inference. Adapters without a standalone handshake +implementation explicitly report that it is unsupported. + +`await adapter.probe_models(timeout_s=60)` discovers a canonical `ModelCatalog` +without creating or selecting a Band room. Unsupported adapters raise +`BandConfigError`; an empty catalog is not a substitute for unsupported discovery. +The existing room-facing `list_models` hook remains separate. + +Catalog entries preserve advertised provider and default metadata. An entry's +`efforts=None` means the harness did not report effort support; `efforts=()` means +it explicitly offers none. `current_model` is populated only when the harness +reports an actual selection. + +Timeouts bound how long a caller waits, not ownership of a partially started +transport. Always call `await adapter.cleanup_all()` when disposing of an adapter. +An incomplete close raises `CleanupIncompleteError` and retains its resources for +a later cleanup attempt rather than reporting successful disposal. + ### Process Signals `agent.run()` and `agent.run_forever()` stop the agent on SIGTERM/SIGINT and restore the process's previous handlers when they return. A host that owns its process signals (a launchd/systemd service, a desktop app, a test runner) passes `install_signal_handlers=False` and calls `agent.stop()` from its own handler; `run_forever` then returns. `GracefulShutdown` is a ready-made handler that does this, and `run_with_graceful_shutdown(agent)` wraps it. Put `GracefulShutdown` outside `agent` so its handlers are registered before `start()` and a mid-initialize SIGTERM can still tear down: diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 41d6e5ce8..73127a7b2 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -8,6 +8,86 @@ see [idle room resource retention](../idle-room-release.md). For existing integrations, follow the [SDK 4.x to 5.x migration](../acp.md#upgrading-from-sdk-4x-to-5x). +## Host preflight + +`await adapter.preflight(timeout_s=60.0)` launches a throwaway app-server using +the adapter's `codex_command`, `codex_env`, client name/title/version, +`experimental_api` and `skill_roots`. It sends `initialize` and `account/read` +without creating a Codex thread, spending a model turn, or claiming a Band room +or workspace. + +The result is a `PreflightResult` with `ok`, `reason` and `remedy`. A missing +account fails only when Codex explicitly reports `requiresOpenaiAuth=true`; +an unsupported or refused `account/read` is not proof that login is missing. +A passed handshake is not proof of provider authentication, subscription or +inference entitlement. + +Room and standalone-probe transport diagnostics omit raw app-server stdout, +stderr and exception text, including cleanup that finishes after caller +cancellation. Malformed frames report transport, type and length; full queues +report only the dropped event kind. Original cleanup errors remain available to +callers and retries. + +```python notest +from band.adapters.codex import CodexAdapter, CodexAdapterConfig + +adapter = CodexAdapter(CodexAdapterConfig()) +result = await adapter.preflight(timeout_s=30) +``` + +The timeout bounds handshake/RPC work and a separate cleanup wait. Cancellation +propagates. Cleanup failures raise `CleanupIncompleteError` instead of reporting +a successful or ordinary failed preflight, and the adapter retains the client, +including a partially created process or Windows job. Keep that adapter and +retry `await adapter.cleanup_all()` to join or retry cleanup. Shutdown attempts +both probe and room cleanup even when either fails. A custom-tool callback +requesting shutdown can receive a retryable pending-cleanup error while its own +turn unwinds; it is not cancelled by that stop. Successful calls close their +throwaway process before returning; a second preflight starts a fresh one. + +Standalone probes set the transport's `graceful_shutdown=False`. After joining +any pending subprocess creation, their retained close owner closes stdin and +requests immediate owned POSIX termination before ordinary settlement. The default +`graceful_shutdown=True` leaves room settlement policy unchanged. Process exit and +reader settlement still require confirmation; an unfinished cleanup remains owned +for retry within the existing reporting bound. + +## Host model discovery + +`await adapter.probe_models(timeout_s=60.0)` uses the same owned throwaway +app-server handshake, including the configured skill roots, then paginates +`model/list`. It creates no thread or turn and leaves existing rooms, their +selected models and their workspace claims unchanged. + +The result is the canonical `band.core.model_catalog.ModelCatalog`. Its visible +`ModelChoice` entries preserve Codex IDs, display names, default-model flags, +advertised reasoning efforts and default efforts. Provider identity is not +reported by `model/list`, so `provider` remains `None`, including with custom +providers. Unknown efforts remain `None`; an explicitly empty effort list becomes +`()`. A default-model flag does not invent `current_model`: a standalone app-server +has no selected room model. + +```python notest +catalog = await adapter.probe_models(timeout_s=30) +model_ids = catalog.model_ids +default_choices = tuple(choice for choice in catalog.models if choice.is_default) +``` + +A successful empty catalog has `models=()`. Unsupported or failed RPCs raise +instead of masquerading as an empty catalog. Cancellation and failed cleanup +have the same ownership and retry behavior as preflight. + +One paginator also serves automatic room model selection, `/model list` and the +current model's reasoning-effort lookup. It follows opaque `nextCursor` values, +rejects repeated cursors and stops after 50 pages rather than returning a +misleading partial catalog. Room listings use `retention.transition_timeout_s` +as their RPC budget; explicit host discovery uses its `timeout_s` budget. +Automatic selection retains its existing fallback to the SDK default on a +failed or empty listing, while an explicitly configured or room-selected model +is not replaced by a catalog default. + +## Room behavior + - **Where settings go.** Runtime settings live in `CodexAdapterConfig`, which forbids unknown fields, so a misplaced `emit=` or a typo fails construction. `emit=`, `capabilities=` and `additional_tools=` go on `CodexAdapter`. @@ -38,9 +118,11 @@ For existing integrations, follow the [SDK 4.x to 5.x migration](../acp.md#upgra On Windows, the client owns the helper and its process-tree job from startup through cleanup. It starts the interpreter image directly so a virtual-environment launcher cannot hide another helper process. Cleanup confirms the helper has - exited before terminating and draining the job; an empty job alone does not - establish helper exit. - Failed cleanup retains that same owner and job for retry. + exited, seals the job against new children, and retains member process handles + before requesting termination. It waits for those handles to signal physical + exit as well as empty job accounting; accounting alone does not prove directory + and file handles were released. Failed cleanup retains that same owner, job, + and any pending process handles for retry. - **Leaving is final.** Leaving a room stops it admitting work at once. A leave that fails or runs over keeps the process, thread and workspace claim, and the room refuses work until a retried leave succeeds; a later rejoin diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 6324fa535..a665f07d0 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -424,3 +424,15 @@ once. It does not rewrite the original delivery's settled acknowledgement. Explicit interruption invalidates its activity; leave and shutdown clear the indicator while adapter residency retains responsibility for released workers. The authoritative idle report finishes before a replacement reports working. + +`SessionConfig.enable_working_state` controls both the indicator and activity +detail, independently of chat narration and tool capabilities. Changes and +keep-alives share one serialized reporter. The duration cap stops both kinds +of refresh without cancelling provider work; actual completion still clears. + +Detail compatibility is learned from the `X-API-Version` header the platform +puts on every activity response. Reports stay boolean until a response in the +process advertises agent API 1.25.0 or later; a missing or older version keeps +boolean reporting. A detail-bearing HTTP 422 permanently disables detail across +all links in the process and retries that report once without detail, within +its original deadline. diff --git a/src/band/adapters/agno.py b/src/band/adapters/agno.py index 910881d55..4466b1ab0 100644 --- a/src/band/adapters/agno.py +++ b/src/band/adapters/agno.py @@ -37,6 +37,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity try: from agno.models.message import Message @@ -451,10 +452,7 @@ async def _run_agent( ) -> RunOutput | None: """Run the Agno agent with the room's tools bound for this call. - When ``Emit.TOOL_CALLS`` is enabled the run is streamed so tool_call / - tool_result events are emitted *as each tool runs* (see - :meth:`_run_streamed`), matching the other adapters' live reporting. - Otherwise it runs non-streaming, exactly as before. + Consume provider progress as it happens, independently of chat narration. """ try: agent = self._agent @@ -469,17 +467,14 @@ async def _run_agent( session_id, ) with _bind_room_tools(tools): - if Emit.TOOL_CALLS in self.features.emit: - response = await self._run_streamed( - agent, - messages, - tools, - session_id=session_id, - room_id=room_id, - msg_id=msg_id, - ) - else: - response = await agent.arun(input=messages, session_id=session_id) + response = await self._run_streamed( + agent, + messages, + tools, + session_id=session_id, + room_id=room_id, + msg_id=msg_id, + ) # Agno swallows run exceptions into a normal-looking return value # (status=error); surface it so the shared except path below treats # the turn as failed rather than as a silent empty reply. @@ -532,9 +527,14 @@ async def _run_streamed( # None, which would read as a successful turn with no output. raise AgnoRunError(_error_summary(item.content)) else: - await self._emit_stream_event( - item, tools, room_id=room_id, msg_id=msg_id - ) + if isinstance(item, ToolCallStartedEvent) and item.tool is not None: + await report_tool_activity(tools, item.tool.tool_name or "") + elif isinstance(item, ToolCallCompletedEvent): + await end_tool_activity(tools) + if Emit.TOOL_CALLS in self.features.emit: + await self._emit_stream_event( + item, tools, room_id=room_id, msg_id=msg_id + ) return final def _persist_turn(self, room_id: str, response: RunOutput) -> None: diff --git a/src/band/adapters/anthropic.py b/src/band/adapters/anthropic.py index 41c524026..982ceab26 100644 --- a/src/band/adapters/anthropic.py +++ b/src/band/adapters/anthropic.py @@ -40,6 +40,7 @@ redact_tool_call_args, ) from band.runtime.tools.schema import is_failed_tool_output +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -478,61 +479,62 @@ async def _process_tool_calls( logger.debug("Executing tool: %s with input: %s", tool_name, tool_input) - # Report tool call if enabled (JSON format with tool_call_id for linking) - # Best-effort: event reporting must never crash tool execution - if Emit.TOOL_CALLS in self.features.emit: + async with tool_activity(tools, tool_name): + # Report tool call if enabled (JSON format with tool_call_id for linking) + # Best-effort: event reporting must never crash tool execution + if Emit.TOOL_CALLS in self.features.emit: + try: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, tool_input + ), + ToolEventKey.TOOL_CALL_ID: tool_use_id, + } + ), + message_type="tool_call", + ) + except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning( + "Failed to send tool_call event: %s", + e, + ) + + # Execute tool (check custom tools first, then platform tools) try: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, tool_input - ), - ToolEventKey.TOOL_CALL_ID: tool_use_id, - } - ), - message_type="tool_call", - ) + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) + succeeded = not is_failed_tool_output(result) + else: + outcome = await tools.execute_tool_call_structured( + tool_name, tool_input + ) + result = outcome.value + succeeded = outcome.ok + if succeeded and tool_name == BandTool.NO_REPLY: + declined = True + if not succeeded: + failed = True + if is_image_passthrough_result(tool_name, result): + content = _image_tool_result_content(result) + result_str = image_block_placeholder(len(content)) + else: + content = result_str = ( + json.dumps(result, default=str) + if not isinstance(result, str) + else result + ) + is_error = False except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning( - "Failed to send tool_call event: %s", - e, - ) - - # Execute tool (check custom tools first, then platform tools) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, tool_input, turn=tools.turn - ) - succeeded = not is_failed_tool_output(result) - else: - outcome = await tools.execute_tool_call_structured( - tool_name, tool_input - ) - result = outcome.value - succeeded = outcome.ok - if succeeded and tool_name == BandTool.NO_REPLY: - declined = True - if not succeeded: failed = True - if is_image_passthrough_result(tool_name, result): - content = _image_tool_result_content(result) - result_str = image_block_placeholder(len(content)) - else: - content = result_str = ( - json.dumps(result, default=str) - if not isinstance(result, str) - else result - ) - is_error = False - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - failed = True - content = result_str = f"Error: {e}" - is_error = True - logger.error("Tool %s failed: %s", tool_name, e) + content = result_str = f"Error: {e}" + is_error = True + logger.error("Tool %s failed: %s", tool_name, e) # Report tool result if enabled (JSON format with tool_call_id for linking) # Best-effort: event reporting must never crash tool execution diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index 04e5f6ab2..b8b973362 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -23,6 +23,12 @@ from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, Literal, NoReturn, Self, cast +from band.runtime.working_state import ( + end_tool_activity, + report_tool_activity, + set_permission_wait, +) + try: from claude_agent_sdk import ( # type: ignore[import-not-found] AssistantMessage, @@ -1406,6 +1412,7 @@ async def _on_tool_use( # namespaces our tools mcp__band__*; see _semantic_tool_name). tool_name = self._semantic_tool_name(block.name) pending_tool_names[block.id] = tool_name + await report_tool_activity(tools, tool_name) await self._narrate_tool_call(block, tool_name, room_id, tools) async def _narrate_tool_call( @@ -1555,6 +1562,7 @@ async def _on_tool_result( Shared by both envelopes a result can arrive in: user-type messages (the protocol shape) and assistant messages (accepted defensively). """ + await end_tool_activity(tools) result_tool_name = pending_tool_names.pop(block.tool_use_id, None) await self._narrate_tool_result(block, result_tool_name, room_id, tools) @@ -1858,48 +1866,56 @@ async def _resolve_manual_approval( # so the caller isn't left waiting for a timeout nobody will see. tools = self._room_tools.get(room_id) mention = [requester["id"]] if requester else None - if tools: - try: - await tools.send_notice( - APPROVAL_REQUESTED_TEMPLATE.format(summary=summary, token=token), - mentions=mention, - ) - except Exception: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Room %s: Failed to send approval notification", room_id) - # A reply that claimed it meanwhile owns the answer; otherwise - # decline without a notice, so the missing-reply guard stays. - if registry.withdraw(entry): - self._clear_pending_approval(room_id, entry) - return PermissionResultDeny( - message="Could not deliver approval prompt, tool use declined" + try: + if tools: + await set_permission_wait(tools, True) + try: + await tools.send_notice( + APPROVAL_REQUESTED_TEMPLATE.format( + summary=summary, token=token + ), + mentions=mention, ) + except Exception: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning( + "Room %s: Failed to send approval notification", room_id + ) + # A reply that claimed it meanwhile owns the answer; otherwise + # decline without a notice, so the missing-reply guard stays. + if registry.withdraw(entry): + return PermissionResultDeny( + message="Could not deliver approval prompt, tool use declined" + ) + + # The request has been posted (or there was nowhere to post it) -- + # either way, on_message must return now so Band's room loop can + # dispatch the reply that will resolve this wait. The turn is then + # judged at its real end (see _on_turn_complete). + if tools: + tools.turn.detach() + self._release_turn(room_id) - # The request has been posted (or there was nowhere to post it) -- - # either way, on_message must return now so Band's room loop can - # dispatch the reply that will resolve this wait. The turn is then - # judged at its real end (see _on_turn_complete). - if tools: - tools.turn.detach() - self._release_turn(room_id) - - try: decision = await registry.wait( - entry, pending.future, timeout_s=self._approvals.wait_timeout_s + entry, + pending.future, + timeout_s=self._approvals.wait_timeout_s, + retain_entry=True, ) + match decision: + case Timeout.TIMED_OUT: + return await self._apply_approval_timeout( + tools, room_id, token, mention, tool_use_id + ) + case ApprovalReply(): + return await self._apply_approval_reply( + tools, room_id, token, decision, tool_use_id + ) + # Forced by eviction or room teardown: no notice names this call. + return PermissionResultDeny(message="User declined tool use") finally: self._clear_pending_approval(room_id, entry) - - match decision: - case Timeout.TIMED_OUT: - return await self._apply_approval_timeout( - tools, room_id, token, mention, tool_use_id - ) - case ApprovalReply(): - return await self._apply_approval_reply( - tools, room_id, token, decision, tool_use_id - ) - # Forced by eviction or room teardown: no notice names this call. - return PermissionResultDeny(message="User declined tool use") + if tools: + await set_permission_wait(tools, bool(registry)) async def _apply_approval_reply( self, diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 3a41024fd..6430db079 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -7,15 +7,15 @@ import logging import time as _time from collections import OrderedDict -from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager, nullcontext +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import AsyncExitStack, asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, fields from dataclasses import field as dataclass_field from datetime import UTC, datetime from enum import StrEnum from functools import partial -from typing import Any, ClassVar, Literal, NamedTuple, Protocol +from typing import Any, ClassVar, Literal, NamedTuple, Protocol, TypeVar from band_sdk_core import AgentFailure from pydantic import ( @@ -35,6 +35,7 @@ from band.converters.codex import CodexHistoryConverter from band.converters.helpers import build_replay_messages +from band.core import exceptions, harness, model_catalog from band.core.adapterconfig import EnvAdapterConfig from band.core.content import has_visible_content from band.core.defaultmodels import OPENAI_MODEL @@ -101,6 +102,7 @@ build_agent_failure, parse_plan_steps, ) +from band.integrations.codex.winspawn import MISSING_EXECUTABLE, SpawnFailure from band.runtime.custom_tools import ( CustomToolDef, custom_tool_to_openai_schema, @@ -122,6 +124,13 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, + tool_activity, +) from band.workspaces import ( RoomWorkspaces, WorkspaceResolver, @@ -130,6 +139,8 @@ logger = logging.getLogger(__name__) +_ProbeResult = TypeVar("_ProbeResult") + def _image_content_items(result: dict[str, Any]) -> list[dict[str, Any]]: """An image tool result as Codex app-server ``inputImage`` content items. @@ -1654,6 +1665,7 @@ async def _process_turn_events( # --- Phase 2: Context compaction events --- if event.method == "context/compacted": + await report_memory_activity(tools) if ( self.config.emit_turn_lifecycle_events and Emit.TASK_EVENTS in self.features.emit @@ -1697,6 +1709,12 @@ async def _process_turn_events( if event.method == "item/started": item = params.get("item") + if isinstance(item, dict) and item.get("type") in _TOOL_ITEM_TYPES: + await report_tool_activity( + tools, self._extract_tool_item(item["type"], item).name + ) + elif isinstance(item, dict): + await end_tool_activity(tools) if ( isinstance(item, dict) and item.get("type") == CodexItemType.AGENT_MESSAGE @@ -1738,6 +1756,8 @@ async def _process_turn_events( if event.method == "item/completed": item = params.get("item") + if isinstance(item, dict) and item.get("type") in _TOOL_ITEM_TYPES: + await end_tool_activity(tools) if isinstance(item, dict): if item.get("type") == CodexItemType.AGENT_MESSAGE: message = result.native_messages.setdefault( @@ -1891,9 +1911,10 @@ async def on_cleanup(self, room_id: str) -> None: await room.retained.dispose() async def cleanup_all(self) -> None: - """Stop the residency owner: admission closes at once, every room's - turn gets ``turn_settle_timeout_s`` to wind down, then its process - closes, all reported within one ``retention.transition_timeout_s``. + """Stop every room and any standalone probe, attempting both on failure. + + Rooms share ``retention.transition_timeout_s``; probe cleanup keeps its + independent budget and custody until the transport physically exits. Callers join the same stop, and a failed one is retried. An unfinished or failed stop keeps every room fenced with its process, thread and @@ -1901,8 +1922,18 @@ async def cleanup_all(self) -> None: """ for room_id in list(self._room_clients): self._clear_pending_approvals_for_room(room_id) - await self._retention.stop() + failures: list[BaseException] = [] + # Retention must see an enrolled callback as the stop requester. + for cleanup in (self._retention.stop, super().cleanup_all): + try: + await cleanup() + except BaseException as error: # noqa: BLE001 -- attempt both cleanups before propagating cancellation or failures + failures.append(error) await self.backend_turns.drain() + if len(failures) == 1: + raise failures[0] + if failures: + raise BaseExceptionGroup("Codex adapter cleanup failed", failures) async def _acquire( self, room: RoomCodexClient, generation: int, resume: bool @@ -2181,6 +2212,82 @@ def _forget_room_thread(self, room_id: str) -> str | None: self._raw_history_by_room.pop(room_id, None) return thread_id + async def preflight(self, *, timeout_s: float = 60.0) -> harness.PreflightResult: + """Check the configured app-server and its reported login without a turn.""" + try: + return await self._probe_harness(self._login_state, timeout_s=timeout_s) + except exceptions.CleanupIncompleteError: + raise + except exceptions.BandConnectionError as error: + cause = error.__cause__ + if isinstance(cause, FileNotFoundError) or ( + isinstance(cause, SpawnFailure) and cause.winerror in MISSING_EXECUTABLE + ): + return harness.PreflightResult.failed( + "Codex CLI binary not found.", + "Install Codex (`npm install -g @openai/codex`) or set " + "CodexAdapterConfig.codex_command.", + ) + return self._handshake_failed() + except Exception: # noqa: BLE001 -- ordinary probe failures are safe host-facing results + return self._handshake_failed() + + @staticmethod + def _handshake_failed() -> harness.PreflightResult: + return harness.PreflightResult.failed( + "Codex app-server did not complete its handshake.", + "Run `codex app-server` in a terminal to check the install, then retry.", + ) + + @staticmethod + async def _login_state(client: CodexClientProtocol) -> harness.PreflightResult: + try: + account = await client.request(CodexRequestMethod.ACCOUNT_READ, {}) + except CodexJsonRpcError: + # Unsupported or refused metadata is not evidence of a missing login. + return harness.PreflightResult.passed() + if account.get("account") is None and account.get("requiresOpenaiAuth") is True: + return harness.PreflightResult.failed( + "Codex is not logged in.", "Run `codex login`, then retry." + ) + return harness.PreflightResult.passed() + + async def probe_models( + self, *, timeout_s: float = 60.0 + ) -> model_catalog.ModelCatalog: + """Discover the visible catalog on an owned, no-room app-server.""" + return await self._probe_harness( + partial(_fetch_codex_models, timeout_s=timeout_s), timeout_s=timeout_s + ) + + async def _probe_harness( + self, + operation: Callable[[CodexClientProtocol], Awaitable[_ProbeResult]], + *, + timeout_s: float, + ) -> _ProbeResult: + async def inspect() -> _ProbeResult: + client = CodexStdioClient( + command=self.config.codex_command, + env=self.config.codex_env, + graceful_shutdown=False, + ) + self._harness_probe.own(client.close) + await client.connect() + await self._initialize_probe(client) + return await operation(client) + + return await self._harness_probe.run(inspect, timeout_s=timeout_s) + + async def _initialize_probe(self, client: CodexClientProtocol) -> None: + await client.initialize( + client_name=self.config.client_name, + client_title=self.config.client_title, + client_version=self.config.client_version, + experimental_api=self.config.experimental_api, + ) + await self._register_skill_roots(client) + async def _register_skill_roots(self, client: CodexClientProtocol) -> None: roots = self.config.skill_roots if not roots: @@ -2204,7 +2311,9 @@ async def _select_model(self, client: CodexClientProtocol) -> str: return self.config.model try: - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) except Exception: logger.warning( "model/list failed; using default Codex model", @@ -2212,9 +2321,8 @@ async def _select_model(self, client: CodexClientProtocol) -> str: ) return OPENAI_MODEL - visible_model_ids = self._visible_model_ids(result) - if visible_model_ids: - return visible_model_ids[0] + if catalog.models: + return catalog.models[0].id logger.warning("model/list has no visible models; using default Codex model") return OPENAI_MODEL @@ -2524,98 +2632,103 @@ async def _handle_server_request( arguments = {} call_id = str(params.get("callId") or "") - # Don't emit reporting for codex-local slash commands — they already - # surface their outcome in the room themselves. - should_report = ( - Emit.TOOL_CALLS in self.features.emit - and tool_name not in _SILENT_REPORTING_TOOLS - ) - - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, arguments - ), - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_call", + async with tool_activity(tools, tool_name): + # Don't emit reporting for codex-local slash commands — they already + # surface their outcome in the room themselves. + should_report = ( + Emit.TOOL_CALLS in self.features.emit + and tool_name not in _SILENT_REPORTING_TOOLS ) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, arguments, turn=tools.turn - ) - success = True - else: - # Structured: a base tool (e.g. band_send_message) can fail without - # raising (bad args, API error) via ok=False; the plain variant would - # report that failure to Codex as success. - outcome = await tools.execute_tool_call_structured( - tool_name, arguments - ) - result = outcome.value - success = outcome.ok - if success and is_image_passthrough_result(tool_name, result): - content_items = _image_content_items(result) - text_result = image_block_placeholder(len(content_items)) - else: - text_result = ( - result - if isinstance(result, str) - else json.dumps(result, default=str) - ) - content_items = [_text_item(text_result)] - await client.respond( - event.id, _tool_response(content_items, success=success) - ) if should_report: await tools.send_event( content=json.dumps( { ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: text_result, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, arguments + ), ToolEventKey.TOOL_CALL_ID: call_id, } ), - message_type="tool_result", + message_type="tool_call", ) - except ValidationError as exc: - errors = format_validation_error(exc) - error_text = f"Invalid arguments for {tool_name}: {errors}" - logger.error("Validation error for tool %s: %s", tool_name, exc) - await client.respond(event.id, _tool_result(error_text, success=False)) - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: error_text, - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_result", + + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, arguments, turn=tools.turn + ) + success = True + else: + # Structured: a base tool (e.g. band_send_message) can fail without + # raising (bad args, API error) via ok=False; the plain variant would + # report that failure to Codex as success. + outcome = await tools.execute_tool_call_structured( + tool_name, arguments + ) + result = outcome.value + success = outcome.ok + if success and is_image_passthrough_result(tool_name, result): + content_items = _image_content_items(result) + text_result = image_block_placeholder(len(content_items)) + else: + text_result = ( + result + if isinstance(result, str) + else json.dumps(result, default=str) + ) + content_items = [_text_item(text_result)] + await client.respond( + event.id, _tool_response(content_items, success=success) ) - except Exception as exc: - error_text = f"Error: {exc}" - logger.exception("Tool execution failed for %s", tool_name) - await client.respond(event.id, _tool_result(error_text, success=False)) - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: error_text, - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_result", + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: text_result, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) + except ValidationError as exc: + errors = format_validation_error(exc) + error_text = f"Invalid arguments for {tool_name}: {errors}" + logger.error("Validation error for tool %s: %s", tool_name, exc) + await client.respond( + event.id, _tool_result(error_text, success=False) + ) + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: error_text, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) + except Exception as exc: + error_text = f"Error: {exc}" + logger.exception("Tool execution failed for %s", tool_name) + await client.respond( + event.id, _tool_result(error_text, success=False) ) + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: error_text, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) return @@ -2659,89 +2772,98 @@ async def _handle_approval_request( room_session = self._session_approved.get(room_id) session_hit = bool(session_key and room_session and session_key in room_session) - if session_hit: - decision: ApprovalDecision = "acceptForSession" - decided_by = "session_policy" - elif self._room_closing(room_id): - decision = "decline" - decided_by = "room_cleanup" - elif self.config.approval_mode == "manual": - try: - decision = await self._resolve_manual_approval( - tools=tools, - msg=msg, - room_id=room_id, - event=event, - summary=summary, - params=params, - ) - decided_by = msg.sender_name or msg.sender_type or "user" - except Exception: - # Ensure we still answer the server request even if the - # human-facing notification flow fails. - logger.exception("Manual approval flow failed; defaulting to decline") + async with AsyncExitStack() as approvals: + if session_hit: + decision: ApprovalDecision = "acceptForSession" + decided_by = "session_policy" + elif self._room_closing(room_id): decision = "decline" - decided_by = "system_fallback" - else: - decision = ( - "accept" if self.config.approval_mode == "auto_accept" else "decline" - ) - decided_by = f"policy:{self.config.approval_mode}" + decided_by = "room_cleanup" + elif self.config.approval_mode == "manual": + try: + decision = await approvals.enter_async_context( + self._manual_approval( + tools=tools, + msg=msg, + room_id=room_id, + event=event, + summary=summary, + params=params, + ) + ) + decided_by = msg.sender_name or msg.sender_type or "user" + except Exception: + # Ensure we still answer the server request even if the + # human-facing notification flow fails. + logger.exception( + "Manual approval flow failed; defaulting to decline" + ) + decision = "decline" + decided_by = "system_fallback" + else: + decision = ( + "accept" + if self.config.approval_mode == "auto_accept" + else "decline" + ) + decided_by = f"policy:{self.config.approval_mode}" - await client.respond(event.id, _approval_response(decision)) + await client.respond(event.id, _approval_response(decision)) - audit_entry = self._record_approval_audit( - room_id=room_id, - request_id=str(event.id), - method=event.method, - decision=decision, - decided_by=decided_by, - summary=summary, - session_level=session_hit, - ) - await self._emit_approval_audit_event( - tools=tools, - room_id=room_id, - entry=audit_entry, - ) + audit_entry = self._record_approval_audit( + room_id=room_id, + request_id=str(event.id), + method=event.method, + decision=decision, + decided_by=decided_by, + summary=summary, + session_level=session_hit, + ) + await self._emit_approval_audit_event( + tools=tools, + room_id=room_id, + entry=audit_entry, + ) - if ( - self.config.approval_mode != "manual" - and self.config.approval_text_notifications - ): - mention = [ - { - "id": msg.sender_id, - "name": msg.sender_name or msg.sender_type, - } - ] - try: - await tools.send_notice( - f"Approval requested ({summary}). Policy decision: {decision}.", - mentions=mention, - ) - except Exception: - logger.exception("Failed to send approval policy notification") + if ( + self.config.approval_mode != "manual" + and self.config.approval_text_notifications + ): + mention = [ + { + "id": msg.sender_id, + "name": msg.sender_name or msg.sender_type, + } + ] + try: + await tools.send_notice( + f"Approval requested ({summary}). Policy decision: {decision}.", + mentions=mention, + ) + except Exception: + logger.exception("Failed to send approval policy notification") - if Emit.THOUGHTS in self.features.emit: - # Best-effort telemetry — never fail the turn on thought emission - # failures. - await send_event_safe( - tools, - content=(f"Codex approval request handled automatically ({decision})."), - message_type="thought", - metadata={ - "codex_approval_method": event.method, - "codex_approval_type": self._approval_type(event.method), - "codex_approval_options": [ - "accept", - "acceptForSession", - "decline", - ], - }, - log_label="approval thought event", - log_level=logging.DEBUG, - ) + if Emit.THOUGHTS in self.features.emit: + # Best-effort telemetry — never fail the turn on thought emission + # failures. + await send_event_safe( + tools, + content=( + f"Codex approval request handled automatically ({decision})." + ), + message_type="thought", + metadata={ + "codex_approval_method": event.method, + "codex_approval_type": self._approval_type(event.method), + "codex_approval_options": [ + "accept", + "acceptForSession", + "decline", + ], + }, + log_label="approval thought event", + log_level=logging.DEBUG, + ) @staticmethod def _turn_usage(usage: CodexTokenUsage | None) -> TurnUsage: @@ -3173,7 +3295,8 @@ def _extract_thought_text(item_type: str, item: dict[str, Any]) -> str | None: case _: return None - async def _resolve_manual_approval( + @asynccontextmanager + async def _manual_approval( self, *, tools: AgentToolsProtocol, @@ -3182,7 +3305,7 @@ async def _resolve_manual_approval( event: RpcEvent, summary: str, params: dict[str, Any], - ) -> ApprovalDecision: + ) -> AsyncIterator[ApprovalDecision]: mention = self._sender_mention(msg) if event.id is None: raise RuntimeError("approval request must have an id") @@ -3207,7 +3330,8 @@ async def _resolve_manual_approval( pending.native_key, room_id, ) - return "decline" + yield "decline" + return entry = registration.entry token = entry.token for removed in registration.removed: @@ -3220,6 +3344,7 @@ async def _resolve_manual_approval( self.config.max_pending_approvals_per_room, ) try: + await set_permission_wait(tools, True) approval_msg = APPROVAL_REQUESTED_TEMPLATE.format( summary=summary, token=token ) @@ -3286,10 +3411,14 @@ async def _resolve_manual_approval( self._release_turn(room_id, tools) decision = await registry.wait( - entry, pending.future, timeout_s=self.config.approval_wait_timeout_s + entry, + pending.future, + timeout_s=self.config.approval_wait_timeout_s, + retain_entry=True, ) if decision is not Timeout.TIMED_OUT: - return decision + yield decision + return timeout_decision = self.config.approval_timeout_decision try: await tools.send_notice( @@ -3300,9 +3429,10 @@ async def _resolve_manual_approval( ) except Exception: logger.exception("Failed to send approval timeout notification") - return timeout_decision + yield timeout_decision finally: self._clear_pending_approval(room_id, entry) + await set_permission_wait(tools, bool(registry)) async def _forward_raw_task_event( self, @@ -3724,8 +3854,10 @@ async def _handle_local_command( if model_arg.lower() in _MODEL_LIST_WORDS: client = self._require_active_client_state().require_client() - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) - models = self._visible_model_ids(result) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) + models = catalog.model_ids if models: preview = ", ".join(models[:10]) if len(models) > 10: @@ -3981,7 +4113,7 @@ async def _handle_approval_command( if (decision_value := _APPROVAL_COMMAND_DECISIONS.get(command)) is None: return False - if not pending: + if not pending.has_unclaimed(): await deliver_notice( tools, NO_APPROVALS_TO_RESOLVE_MESSAGE, @@ -4433,53 +4565,87 @@ def _clear_pending_approvals_for_room(self, room_id: str) -> None: for entry in registry.cancel_all(): entry.payload.future.set_result("decline") - @staticmethod - def _visible_model_ids(result: dict[str, Any]) -> list[str]: - data = result.get("data") if isinstance(result, dict) else None - if not isinstance(data, list): - return [] - models: list[str] = [] - for entry in data: - if not isinstance(entry, dict): - continue - model_id = entry.get("id") - if not isinstance(model_id, str) or not model_id: - continue - if bool(entry.get("hidden", False)): - continue - models.append(model_id) - return models - - @staticmethod - def _supported_efforts(result: dict[str, Any], model_id: str) -> list[str]: - data = result.get("data") if isinstance(result, dict) else None - if not isinstance(data, list): - return [] - entry = next( - (e for e in data if isinstance(e, dict) and e.get("id") == model_id), - None, - ) - efforts = entry.get("supportedReasoningEfforts") if entry else None - if not isinstance(efforts, list): - return [] - return [ - effort["reasoningEffort"] - for effort in efforts - if isinstance(effort, dict) - and isinstance(effort.get("reasoningEffort"), str) - ] - async def _current_model_efforts(self) -> tuple[str, list[str]]: """The room's model and the efforts Codex reports for it (empty if unknown).""" room = self._require_active_client_state() client = room.require_client() model_id = room.selected_model or await self._select_model(client) try: - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) except Exception: logger.warning( "model/list failed; supported reasoning efforts are unknown", exc_info=True, ) return model_id, [] - return model_id, self._supported_efforts(result, model_id) + choice = catalog.choice(model_id) + return model_id, list(choice.efforts or ()) if choice is not None else [] + + +# A catalog this large indicates a broken pagination stream. +_MAX_MODEL_LIST_PAGES = 50 + + +async def _fetch_codex_models( + client: CodexClientProtocol, *, timeout_s: float +) -> model_catalog.ModelCatalog: + """Read every visible page for host discovery and all room model consumers.""" + models: list[model_catalog.ModelChoice] = [] + params: dict[str, Any] = {} + seen_cursors: set[str] = set() + async with asyncio.timeout(timeout_s): + for _ in range(_MAX_MODEL_LIST_PAGES): + result = await client.request(CodexRequestMethod.MODEL_LIST, params) + models.extend(_codex_model_choices(result)) + cursor = result.get("nextCursor") + if not cursor: + return model_catalog.ModelCatalog(models=tuple(models)) + if not isinstance(cursor, str): + raise RuntimeError("Codex model/list returned an invalid cursor") + if cursor in seen_cursors: + raise RuntimeError("Codex model/list repeated cursor") + seen_cursors.add(cursor) + params = {"cursor": cursor} + raise RuntimeError( + f"Codex model/list returned more than {_MAX_MODEL_LIST_PAGES} pages" + ) + + +def _codex_model_choices(result: dict[str, Any]) -> list[model_catalog.ModelChoice]: + data = result.get("data") + if not isinstance(data, list): + return [] + models: list[model_catalog.ModelChoice] = [] + for entry in data: + if not isinstance(entry, dict) or entry.get("hidden", False): + continue + model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id: + continue + supported = entry.get("supportedReasoningEfforts") + efforts = ( + tuple( + effort["reasoningEffort"] + for effort in supported + if isinstance(effort, dict) + and isinstance(effort.get("reasoningEffort"), str) + ) + if isinstance(supported, list) + else None + ) + label = entry.get("displayName") + default_effort = entry.get("defaultReasoningEffort") + models.append( + model_catalog.ModelChoice( + id=model_id, + label=label if isinstance(label, str) and label else model_id, + efforts=efforts, + default_effort=( + default_effort if isinstance(default_effort, str) else None + ), + is_default=bool(entry.get("isDefault", False)), + ) + ) + return models diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index 5a3c8fde1..5d48f2ac4 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio +import inspect import json import logging import re @@ -59,6 +60,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import set_permission_wait, tool_activity try: from copilot import ( @@ -101,6 +103,8 @@ from band.core.types import FeatureKwargs, PlatformMessage from band.runtime.custom_tools import CustomToolDef +ASK_USER_TOOL_NAME = "ask_user" + logger = logging.getLogger(__name__) _PROVIDER = "copilot_sdk" @@ -652,14 +656,41 @@ def _session_kwargs( if self._ask_user is not None: # ask_user is session-isolated (no shell/file/host access), so # allowing it does not weaken the approve_all stance above. - available_tools.append("ask_user") - kwargs["on_user_input_request"] = ( - self._ask_user - if callable(self._ask_user) - else self._make_room_ask_user_handler(room_id) - ) + available_tools.append(ASK_USER_TOOL_NAME) + kwargs["on_user_input_request"] = self._activity_user_input_handler(room_id) return kwargs + def _activity_user_input_handler(self, room_id: str) -> UserInputHandler: + ask_user = self._ask_user + if callable(ask_user): + handler: UserInputHandler = ask_user + manual = True + else: + handler = self._make_room_ask_user_handler(room_id) + manual = False + pending = 0 + + async def handle( + request: UserInputRequest, context: dict[str, str] + ) -> UserInputResponse: + nonlocal pending + tools = self._room_tools.get(room_id) + async with tool_activity(tools, ASK_USER_TOOL_NAME): + result = handler(request, context) + if not inspect.isawaitable(result): + return result + if not manual: + return await result + pending += 1 + try: + await set_permission_wait(tools, True) + return await result + finally: + pending -= 1 + await set_permission_wait(tools, pending > 0) + + return handle + def _make_room_ask_user_handler(self, room_id: str) -> UserInputHandler: """Bind the room id for the session's ask_user → room bridge.""" @@ -775,46 +806,47 @@ async def _execute_bridged_tool( error="room inactive", ) - should_report = Emit.TOOL_CALLS in self.features.emit - if should_report: - await self._report_tool_call(room_tools, invocation, arguments) + async with tool_activity(room_tools, tool_name): + should_report = Emit.TOOL_CALLS in self.features.emit + if should_report: + await self._report_tool_call(room_tools, invocation, arguments) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, arguments, turn=room_tools.turn + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, arguments, turn=room_tools.turn + ) + else: + # Structured variant: a base tool (e.g. band_send_message) can fail + # without raising (bad args, API error) — that surfaces as ok=False, + # not an exception, and must reach the model as a failed call. + outcome = await room_tools.execute_tool_call_structured( + tool_name, arguments + ) + if not outcome.ok: + return await self._fail_tool_call( + room_tools, + invocation, + outcome.error_message or str(outcome.value), + report=should_report, + ) + result = outcome.value + except ValidationError as exc: + logger.error("Validation error for tool %s: %s", tool_name, exc) + error_text = ( + f"Invalid arguments for {tool_name}: {format_validation_error(exc)}" ) - else: - # Structured variant: a base tool (e.g. band_send_message) can fail - # without raising (bad args, API error) — that surfaces as ok=False, - # not an exception, and must reach the model as a failed call. - outcome = await room_tools.execute_tool_call_structured( - tool_name, arguments + return await self._fail_tool_call( + room_tools, invocation, error_text, report=should_report + ) + except Exception as exc: + logger.exception("Tool execution failed for %s", tool_name) + return await self._fail_tool_call( + room_tools, invocation, f"Error: {exc}", report=should_report ) - if not outcome.ok: - return await self._fail_tool_call( - room_tools, - invocation, - outcome.error_message or str(outcome.value), - report=should_report, - ) - result = outcome.value - except ValidationError as exc: - logger.error("Validation error for tool %s: %s", tool_name, exc) - error_text = ( - f"Invalid arguments for {tool_name}: {format_validation_error(exc)}" - ) - return await self._fail_tool_call( - room_tools, invocation, error_text, report=should_report - ) - except Exception as exc: - logger.exception("Tool execution failed for %s", tool_name) - return await self._fail_tool_call( - room_tools, invocation, f"Error: {exc}", report=should_report - ) - binary_results: list[ToolBinaryResult] | None = None + binary_results: list[ToolBinaryResult] | None = None if is_image_passthrough_result(tool_name, result): # Same failure path as the tool-execution try/except above: a # malformed or future-extended content block must still route diff --git a/src/band/adapters/crewai_flow.py b/src/band/adapters/crewai_flow.py index 25eca8064..70bcef705 100644 --- a/src/band/adapters/crewai_flow.py +++ b/src/band/adapters/crewai_flow.py @@ -77,6 +77,7 @@ execute_custom_tool, get_custom_tool_name, ) +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -610,16 +611,17 @@ async def invoke() -> Any: async def _execute_tool( self, name: str, tool: CustomToolDef, input_data: dict[str, Any] ) -> Any: - await self._report_call(name, input_data) - try: - result = await execute_custom_tool( - tool=tool, arguments=input_data, turn=self._tools.turn - ) - except Exception as exc: - await self._report_result(name, str(exc), is_error=True) - raise - await self._report_result(name, result) - return result + async with tool_activity(self._tools, name): + await self._report_call(name, input_data) + try: + result = await execute_custom_tool( + tool=tool, arguments=input_data, turn=self._tools.turn + ) + except Exception as exc: + await self._report_result(name, str(exc), is_error=True) + raise + await self._report_result(name, result) + return result async def _report_call(self, tool_name: str, input_data: dict[str, Any]) -> None: await self._reporter.report_call(self._tools, tool_name, input_data) diff --git a/src/band/adapters/cursor_acp.py b/src/band/adapters/cursor_acp.py index e956decb4..31b7efaac 100644 --- a/src/band/adapters/cursor_acp.py +++ b/src/band/adapters/cursor_acp.py @@ -5,7 +5,7 @@ import asyncio import logging from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from dataclasses import dataclass, field from functools import partial from typing import ClassVar, Literal, Self @@ -58,6 +58,7 @@ permission_option_ids, select_allow_option_id, ) +from band.integrations.acp.room_emitter import RoomTurnEmitter from band.integrations.acp.session_config import SessionConfigResolver from band.runtime.custom_tools import CustomToolDef from band.runtime.decisions import ( @@ -67,6 +68,7 @@ ) from band.runtime.formatters import strip_leading_mentions from band.runtime.tools.registry import BAND_MCP_SERVER_NAME +from band.runtime.working_state import set_permission_wait from band.workspaces import WorkspaceResolver logger = logging.getLogger(__name__) @@ -462,6 +464,12 @@ async def cleanup_all(self) -> None: self._cancel_all_decisions() await super().cleanup_all() + def _permission_activity( + self, emitter: RoomTurnEmitter | None + ) -> AbstractAsyncContextManager[None]: + # Cursor's decision registry owns manual permission, question and plan waits. + return nullcontext() + async def _resolve_cursor_permission( self, request: ACPPermissionRequest ) -> str | None: @@ -628,28 +636,33 @@ async def _wait_for_decision( self._resolve_evicted_decision(registration.evicted) entry = registration.entry try: - await turn.tools.send_notice( - prompt.replace("{token}", entry.token), - mentions=_requester_mentions(turn), + await set_permission_wait(turn.tools, True) + try: + await turn.tools.send_notice( + prompt.replace("{token}", entry.token), + mentions=_requester_mentions(turn), + ) + except Exception: # noqa: BLE001 -- best-effort room notify; any failure (network, REST, unresolved mention) should not block the decision wait below + logger.warning("Could not deliver Cursor %s decision prompt", kind) + # A reply that claimed it meanwhile owns the answer; wait for it. + if decisions.withdraw(entry): + return None + finally: + # Release on_message here, not only in _run_turn's finally -- + # the room needs its queue back the instant a decision is + # outstanding, whether or not the prompt itself landed. + _detach(turn) + + result = await decisions.wait( + entry, future, timeout_s=self.config.decision_timeout_s ) - except Exception: # noqa: BLE001 -- best-effort room notify; any failure (network, REST, unresolved mention) should not block the decision wait below - logger.warning("Could not deliver Cursor %s decision prompt", kind) - # A reply that claimed it meanwhile owns the answer; wait for it. - if decisions.withdraw(entry): + if result is Timeout.TIMED_OUT: + await self._notify_decision_timeout(turn, kind, entry.token) return None + return result finally: - # Release on_message here, not only in _run_turn's finally -- - # the room needs its queue back the instant a decision is - # outstanding, whether or not the prompt itself landed. - _detach(turn) - - result = await decisions.wait( - entry, future, timeout_s=self.config.decision_timeout_s - ) - if result is Timeout.TIMED_OUT: - await self._notify_decision_timeout(turn, kind, entry.token) - return None - return result + decisions.forget(entry) + await set_permission_wait(turn.tools, bool(decisions)) @staticmethod async def _notify_decision_timeout( diff --git a/src/band/adapters/gemini.py b/src/band/adapters/gemini.py index a38b733d2..a725af4f8 100644 --- a/src/band/adapters/gemini.py +++ b/src/band/adapters/gemini.py @@ -18,6 +18,8 @@ ) from typing_extensions import Unpack +from band.runtime.working_state import tool_activity + try: from google import genai # type: ignore[missing-module-attribute] from google.genai import types # type: ignore[missing-import] @@ -541,51 +543,54 @@ async def _process_function_calls( tool_input = dict(function_call.args or {}) tool_call_id = function_call.id or f"gemini_tool_call_{index}" - if Emit.TOOL_CALLS in self.features.emit: - try: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, tool_input - ), - ToolEventKey.TOOL_CALL_ID: tool_call_id, - } - ), - message_type="tool_call", - ) - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Failed to send tool_call event: %s", e) + async with tool_activity(tools, tool_name): + if Emit.TOOL_CALLS in self.features.emit: + try: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, tool_input + ), + ToolEventKey.TOOL_CALL_ID: tool_call_id, + } + ), + message_type="tool_call", + ) + except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning("Failed to send tool_call event: %s", e) - response_parts: list[types.FunctionResponsePart] | None = None - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, tool_input, turn=tools.turn - ) - else: - result = await tools.execute_tool_call(tool_name, tool_input) - if is_image_passthrough_result(tool_name, result): - response_parts = _image_function_response_parts(result) - result_str = image_block_placeholder(len(response_parts)) - else: - result_str = ( - json.dumps(result, default=str) - if not isinstance(result, str) - else result + response_parts: list[types.FunctionResponsePart] | None = None + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) + else: + result = await tools.execute_tool_call(tool_name, tool_input) + if is_image_passthrough_result(tool_name, result): + response_parts = _image_function_response_parts(result) + result_str = image_block_placeholder(len(response_parts)) + else: + result_str = ( + json.dumps(result, default=str) + if not isinstance(result, str) + else result + ) + is_error = False + except ValidationError as exc: + errors = format_validation_error(exc) + result_str = f"Invalid arguments for {tool_name}: {errors}" + is_error = True + logger.warning( + "Validation error for tool %s: %s", tool_name, errors ) - is_error = False - except ValidationError as exc: - errors = format_validation_error(exc) - result_str = f"Invalid arguments for {tool_name}: {errors}" - is_error = True - logger.warning("Validation error for tool %s: %s", tool_name, errors) - except Exception as e: - result_str = f"Error: {e}" - is_error = True - logger.exception("Tool %s failed", tool_name) + except Exception as e: + result_str = f"Error: {e}" + is_error = True + logger.exception("Tool %s failed", tool_name) if Emit.TOOL_CALLS in self.features.emit: try: await tools.send_event( diff --git a/src/band/adapters/google_adk.py b/src/band/adapters/google_adk.py index 374c84f65..116fd9ba7 100644 --- a/src/band/adapters/google_adk.py +++ b/src/band/adapters/google_adk.py @@ -47,6 +47,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity if TYPE_CHECKING: from google.adk.runners import InMemoryRunner @@ -660,12 +661,10 @@ async def on_message( event.error_code, event.error_message ) - # Report tool calls/results if enabled - if Emit.TOOL_CALLS in self.features.emit: - try: - await self._report_event(event, tools) - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Failed to report event: %s", e) + try: + await self._report_event(event, tools) + except Exception as e: # noqa: BLE001 -- best-effort tool narration + logger.warning("Failed to report event: %s", e) if event.is_final_response(): # Extract text from the final response for history tracking @@ -788,6 +787,9 @@ async def _report_event(self, event: Any, tools: AgentToolsProtocol) -> None: function_calls = event.get_function_calls() if function_calls: for fc in function_calls: + await report_tool_activity(tools, getattr(fc, "name", "") or "") + if Emit.TOOL_CALLS not in self.features.emit: + continue try: tool_name = getattr(fc, "name", "unknown") try: @@ -812,6 +814,9 @@ async def _report_event(self, event: Any, tools: AgentToolsProtocol) -> None: function_responses = event.get_function_responses() if function_responses: for fr in function_responses: + await end_tool_activity(tools) + if Emit.TOOL_CALLS not in self.features.emit: + continue try: tool_name = getattr(fr, "name", "unknown") await tools.send_event( diff --git a/src/band/adapters/langgraph.py b/src/band/adapters/langgraph.py index 8bc60fc09..f4fe72c32 100644 --- a/src/band/adapters/langgraph.py +++ b/src/band/adapters/langgraph.py @@ -35,11 +35,13 @@ image_block_placeholder, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity if TYPE_CHECKING: from langchain_core.language_models import BaseChatModel from langgraph.checkpoint.base import BaseCheckpointSaver + logger = logging.getLogger(__name__) @@ -437,6 +439,7 @@ async def _handle_stream_event( event_type = event.get("event") if event_type == "on_tool_start": + await report_tool_activity(tools, event.get("name") or "") if Emit.TOOL_CALLS not in self.features.emit: return @@ -459,6 +462,7 @@ async def _handle_stream_event( logger.warning("Failed to send tool_call event: %s", e) elif event_type in {"on_tool_end", "on_tool_error"}: + await end_tool_activity(tools) if Emit.TOOL_CALLS not in self.features.emit: return diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 28dcb83e6..f15eb6549 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -52,6 +52,7 @@ redact_tool_call_args, turn_effect, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity __all__ = [ "LettaAdapter", @@ -491,10 +492,7 @@ async def _send_message( Returns the list of assistant text parts collected during the turn. """ messages = [{"role": "user", "content": content}] - # Per-turn token usage. Reachable only on the per_room path, where the - # non-streamed LettaResponse carries an aggregate .usage; the shared-mode - # Conversations stream exposes no such aggregate, so it stays empty (N-A). - # Emitted on every exit via the finally. + # Shared Conversations has no aggregate usage; per-room step streams do. turn_usage = TurnUsage() run = LettaRun(self._client, agent_id, self._shared_conversation(room_ctx)) @@ -510,7 +508,7 @@ async def _send_message( room_ctx.tools = tools try: response_messages, turn_usage = await asyncio.wait_for( - self._call_provider(run, messages), + self._call_provider(run, messages, tools), timeout=self.config.turn_timeout_s, ) except TimeoutError: @@ -556,22 +554,55 @@ def _shared_conversation(self, room_ctx: RoomContext) -> str | None: return room_ctx.conversation_id if self.config.mode == "shared" else None async def _call_provider( - self, run: LettaRun, messages: list[dict[str, str]] + self, run: LettaRun, messages: list[dict[str, str]], tools: AgentToolsProtocol ) -> tuple[list[Any], TurnUsage]: - """Round-trip to the Letta API -- no response processing or delivery.""" - if run.conversation_id: - conversation_stream = await run.client.conversations.messages.create( + """Stream the provider response and return its messages and usage. + + Records tool activity as events arrive. Does not narrate or deliver. + """ + per_room = not run.conversation_id + if per_room: + stream = await run.client.agents.messages.create( + agent_id=run.agent_id, + messages=messages, + streaming=True, + stream_tokens=False, + ) + else: + stream = await run.client.conversations.messages.create( conversation_id=run.conversation_id, messages=messages, ) - response_messages = [resp_msg async for resp_msg in conversation_stream] - return response_messages, TurnUsage() + response_messages: list[Any] = [] + usage = TurnUsage() + async with stream: + async for message in stream: + await self._observe_provider_message(message, tools) + if ( + per_room + and getattr(message, "message_type", None) == "usage_statistics" + ): + usage = TurnUsage.from_object( + message, + input="prompt_tokens", + output="completion_tokens", + cache_read="cached_input_tokens", + cache_write="cache_write_tokens", + ) + else: + response_messages.append(message) + return response_messages, usage - response = await run.client.agents.messages.create( - agent_id=run.agent_id, - messages=messages, - ) - return list(response.messages), self._usage_from_response(response) + @staticmethod + async def _observe_provider_message( + message: Any, tools: AgentToolsProtocol + ) -> None: + match getattr(message, "message_type", None): + case "tool_call_message": + for call in message.tool_calls or [message.tool_call]: + await report_tool_activity(tools, call.name) + case "tool_return_message": + await end_tool_activity(tools) @staticmethod async def _cancel_runs(run: LettaRun) -> None: @@ -1212,22 +1243,6 @@ async def _consolidate_memory(self, agent_id: str, room_id: str) -> None: # Helpers # ------------------------------------------------------------------ - @staticmethod - def _usage_from_response(response: Any) -> TurnUsage: - """Map a Letta ``LettaResponse.usage`` (a ``Usage``) onto TurnUsage. - - Field names verified against letta-client: ``prompt_tokens`` / - ``completion_tokens`` / ``cached_input_tokens`` / ``cache_write_tokens``. - A missing ``usage`` yields empty usage. - """ - return TurnUsage.from_object( - getattr(response, "usage", None), - input="prompt_tokens", - output="completion_tokens", - cache_read="cached_input_tokens", - cache_write="cache_write_tokens", - ) - @staticmethod def _format_time_ago(dt: datetime) -> str: """Format a datetime as a human-readable time-ago string.""" diff --git a/src/band/adapters/nooa/tools.py b/src/band/adapters/nooa/tools.py index 40359c572..6e83b968b 100644 --- a/src/band/adapters/nooa/tools.py +++ b/src/band/adapters/nooa/tools.py @@ -46,6 +46,7 @@ redact_tool_call_args, serialize_tool_result, ) +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -158,34 +159,37 @@ async def _call( self, tools: AgentToolsProtocol, name: str, arguments: dict[str, Any] ) -> Any: """Run one tool call and report it, inside the turn that owns it.""" - call_id = uuid4().hex - await self._report( - tools, - MessageType.TOOL_CALL, - { - ToolEventKey.NAME: name, - ToolEventKey.ARGS: redact_tool_call_args(name, arguments), - ToolEventKey.TOOL_CALL_ID: call_id, - }, - ) - try: - value = serialize_tool_result(await self._specs[name].run(tools, arguments)) - except asyncio.CancelledError: - message = f"{name} was cancelled because its turn ended first" - await self._report_result(tools, name, call_id, message, is_error=True) - raise - except Exception as error: - message = f"{name} failed: {error}" - logger.warning("Band tool %s failed: %s", name, error) - await self._report_result(tools, name, call_id, message, is_error=True) - raise BandToolError(message) from error - if (image_result := image_passthrough_result(name, value)) is not None: - images = images_from_result(image_result) - output = image_block_placeholder(len(images)) - await self._report_result(tools, name, call_id, output, is_error=False) - return images - await self._report_result(tools, name, call_id, value, is_error=False) - return value + async with tool_activity(tools, name): + call_id = uuid4().hex + await self._report( + tools, + MessageType.TOOL_CALL, + { + ToolEventKey.NAME: name, + ToolEventKey.ARGS: redact_tool_call_args(name, arguments), + ToolEventKey.TOOL_CALL_ID: call_id, + }, + ) + try: + value = serialize_tool_result( + await self._specs[name].run(tools, arguments) + ) + except asyncio.CancelledError: + message = f"{name} was cancelled because its turn ended first" + await self._report_result(tools, name, call_id, message, is_error=True) + raise + except Exception as error: + message = f"{name} failed: {error}" + logger.warning("Band tool %s failed: %s", name, error) + await self._report_result(tools, name, call_id, message, is_error=True) + raise BandToolError(message) from error + if (image_result := image_passthrough_result(name, value)) is not None: + images = images_from_result(image_result) + output = image_block_placeholder(len(images)) + await self._report_result(tools, name, call_id, output, is_error=False) + return images + await self._report_result(tools, name, call_id, value, is_error=False) + return value async def _report_result( self, diff --git a/src/band/adapters/omp_acp.py b/src/band/adapters/omp_acp.py index 47f5e099d..844c3f126 100644 --- a/src/band/adapters/omp_acp.py +++ b/src/band/adapters/omp_acp.py @@ -278,6 +278,7 @@ async def handler( options=options, room_id=room_id, session_id=session_id, + emitter=emitter, ) if option_id == OMP_APPROVE_OPTION_ID: return AcceptElicitationResponse( diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index 98fbb2136..50d3cf082 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -65,9 +65,14 @@ from band.runtime.custom_tools import CustomToolDef, get_custom_tool_name from band.runtime.prompts import render_system_prompt from band.runtime.tools import CHAT_ID_FIELD_NAME, iter_tool_definitions +from band.runtime.working_state import end_tool_activity, report_tool_activity logger = logging.getLogger(__name__) +_ENDED_TOOL_ACTIVITY = frozenset( + {OpencodeToolStatus.COMPLETED, OpencodeToolStatus.ERROR} +) + _PROVIDER = "opencode" _OPENCODE_SYSTEM_NOTE = """\ @@ -99,6 +104,7 @@ class TurnState: assistant_message_ids: set[str] = field(default_factory=set) assistant_part_types: dict[str, str] = field(default_factory=dict) reported_tool_calls: set[str] = field(default_factory=set) + activity_tool_status: dict[str, str | None] = field(default_factory=dict) reported_tool_results: set[str] = field(default_factory=set) last_error_message: str | None = None # Per-assistant-message usage for the current turn (last-write-wins per id, @@ -882,11 +888,36 @@ async def _handle_part_update( case "tool": await self._report_tool_part(room_state, part) + async def _observe_tool_activity( + self, room_state: RoomState, part: OpencodePart + ) -> None: + turn = room_state.turn + state = part.state + if turn is None or part.id is None or state is None: + return + call_id = part.call_id or part.id + previous = turn.activity_tool_status.get(call_id) + status = state.status + turn.activity_tool_status[call_id] = status + if ( + status in {OpencodeToolStatus.PENDING, OpencodeToolStatus.RUNNING} + and previous is None + ): + await report_tool_activity( + turn.tools, self._canonical_tool_name(part.tool or "") + ) + return + if status in _ENDED_TOOL_ACTIVITY and previous not in _ENDED_TOOL_ACTIVITY: + await end_tool_activity(turn.tools) + async def _report_tool_part( self, room_state: RoomState, part: OpencodePart ) -> None: """Report the tool's call/result.""" - if part.state is None or Emit.TOOL_CALLS not in self.features.emit: + if part.state is None: + return + await self._observe_tool_activity(room_state, part) + if Emit.TOOL_CALLS not in self.features.emit: return state = part.state diff --git a/src/band/adapters/opencode/approvals.py b/src/band/adapters/opencode/approvals.py index 2404893f1..dcdf90c5e 100644 --- a/src/band/adapters/opencode/approvals.py +++ b/src/band/adapters/opencode/approvals.py @@ -32,6 +32,7 @@ ) from band.runtime.decisions import DecisionEntry, DecisionRegistry from band.runtime.formatters import format_tokens, strip_leading_mentions +from band.runtime.working_state import set_permission_wait logger = logging.getLogger(__name__) @@ -227,6 +228,10 @@ def _parked_on_human(self) -> bool: """ return self._permissions.has_unclaimed() or self._questions.has_unclaimed() + async def _report_manual_wait(self) -> None: + if tools := self._ports.tools(): + await set_permission_wait(tools, self.awaiting_human()) + async def wait_until_idle(self) -> None: """Block until no manual ask is awaiting a human reply.""" await self._idle.wait() @@ -329,6 +334,7 @@ async def _ask_room(self, entry: DecisionEntry[PendingPermission]) -> None: entry, self._config.approval_wait_timeout_s, self._expire_permission ) self._park_on_human() + await self._report_manual_wait() pattern_text = ", ".join(pending.patterns) if pending.patterns else "n/a" await self._notify_room( APPROVAL_REQUESTED_TEMPLATE.format( @@ -385,6 +391,7 @@ async def on_question_asked(self, request: OpencodeQuestionRequest) -> None: entry, self._config.question_wait_timeout_s, self._expire_question ) self._park_on_human() + await self._report_manual_wait() await self._notify_room( format_question_prompt(pending.questions, request_id), self._ports.turn_mentions(), @@ -662,6 +669,7 @@ async def _send_permission_reply( reply, ) self._forget(self._permissions, entry) + await self._report_manual_wait() return True async def _reply_permission( @@ -686,6 +694,7 @@ async def _send_question_reply( entry.token, ) self._forget(self._questions, entry) + await self._report_manual_wait() return True async def _reply_question( @@ -715,6 +724,7 @@ async def _send_question_reject( entry.token, ) self._forget(self._questions, entry) + await self._report_manual_wait() return True async def _reject_question(self, pending: PendingQuestion) -> bool: @@ -735,6 +745,7 @@ async def _fail_request( # abort_session kills the whole session, so no sibling ask can be # answered either -- drop them all, timers included. self.cancel() + await self._report_manual_wait() self._ports.fail_turn(message) await self._ports.abort_session() diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index 732a4dcdb..e6592f1e1 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -83,6 +83,7 @@ redact_tool_call_args, ) from band.runtime.tools.schema import is_failed_tool_output +from band.runtime.working_state import end_tool_activity, report_tool_activity logger = logging.getLogger(__name__) @@ -626,6 +627,7 @@ async def on_message( if event_count & (event_count - 1) == 0: _log_native_progress("stream_progress", events=event_count) if isinstance(event, FunctionToolCallEvent): + await report_tool_activity(tools, event.part.tool_name) _log_native_progress( "tool_started", events=event_count, @@ -650,6 +652,7 @@ async def on_message( except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn logger.warning("Failed to send tool_call event: %s", e) elif isinstance(event, FunctionToolResultEvent): + await end_tool_activity(tools) tool_succeeded = isinstance( event.part, ToolReturnPart ) and not is_failed_tool_output(event.part.content) diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index 809b6eba8..0a90a511a 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -10,6 +10,8 @@ from band_sdk_core import AgentFailure from pydantic import BaseModel +from band.runtime.working_state import end_tool_activity, report_tool_activity + try: from strands import Agent from strands.agent import AgentResult @@ -383,6 +385,7 @@ def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: registry.add_callback(AfterToolCallEvent, self._on_after_tool) async def _on_before_tool(self, event: BeforeToolCallEvent) -> None: + await report_tool_activity(self._tools, event.tool_use["name"]) call: dict[ToolEventKey, object] = { ToolEventKey.NAME: event.tool_use["name"], ToolEventKey.ARGS: redact_tool_call_args( @@ -399,6 +402,7 @@ async def _on_before_tool(self, event: BeforeToolCallEvent) -> None: ) async def _on_after_tool(self, event: AfterToolCallEvent) -> None: + await end_tool_activity(self._tools) name = event.tool_use["name"] output = _result_text(event.result) succeeded = event.result.get("status") == "success" and not band_tool_errored( diff --git a/src/band/core/activity.py b/src/band/core/activity.py new file mode 100644 index 000000000..5da2a76c0 --- /dev/null +++ b/src/band/core/activity.py @@ -0,0 +1,57 @@ +"""Room-visible activity detail and the choice that governs tool names. + +A tool name reaches the room the same way its narration does: an agent that +does not report tool calls must not leak their names through the live +indicator. Every reporter funnels through the execution context's observer, +so the decision is applied once there and never per adapter. +""" + +from __future__ import annotations + +from collections.abc import Callable, Collection +from typing import TYPE_CHECKING + +from band_rest.types.agent_activity_detail import AgentActivityDetail + +from band.core.types import Emit + +if TYPE_CHECKING: + from band.runtime.working_state import ActivityObserver + + +class ToolNamesObserver: + """Hide tool names while leaving every other activity detail intact. + + Permission waits and memory activity name no content, so they report + whatever the tool-name choice is. ``hidden`` is consulted at report time, + never bound at construction: an observer attached before the adapter's + emit sync must still follow the turn's live choice. + """ + + def __init__( + self, inner: ActivityObserver, hidden: Callable[[], bool] = lambda: True + ) -> None: + self._inner = inner + self._hidden = hidden + + async def update(self, activity: AgentActivityDetail) -> None: + if activity.kind == "tool_call" and self._hidden(): + await self._inner.update(AgentActivityDetail(kind="reasoning")) + return + await self._inner.update(activity) + + async def set_manual_wait(self, waiting: bool) -> None: + await self._inner.set_manual_wait(waiting) + + +def sync_activity_tool_names(tools: object, emit: Collection[Emit]) -> None: + """Push the adapter's tool-narration choice onto the bound context. + + ``Emit.TOOL_CALLS`` absent means the room sees ``reasoning`` where a tool + runs. Tools without an execution context (fakes, unbound instances) have + no room indicator to gate. + """ + ctx = getattr(tools, "_ctx", None) + setter = getattr(ctx, "set_activity_tool_names", None) + if setter is not None: + setter(Emit.TOOL_CALLS in emit) diff --git a/src/band/core/harness.py b/src/band/core/harness.py new file mode 100644 index 000000000..5d3e2998f --- /dev/null +++ b/src/band/core/harness.py @@ -0,0 +1,127 @@ +"""Host-facing handshake results and custody of throwaway harness probes.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar + +from pydantic import BaseModel, ConfigDict + +from band.core.exceptions import CleanupIncompleteError +from band.core.retention.cleanup import rejoin_owned, report_owned + +_T = TypeVar("_T") +_Close = Callable[[], Awaitable[None]] + + +class PreflightResult(BaseModel): + """Whether a harness answered its handshake, not proof of inference access.""" + + model_config = ConfigDict(frozen=True) + + ok: bool + reason: str | None = None + remedy: str | None = None + + @classmethod + def passed(cls) -> PreflightResult: + return cls(ok=True) + + @classmethod + def failed(cls, reason: str, remedy: str) -> PreflightResult: + return cls(ok=False, reason=reason, remedy=remedy) + + +class HarnessProbe: + """One adapter's serialized probes and retryable resource cleanup. + + Operations register resource closers before starting their transports. A + timeout bounds reporting, not ownership: cleanup joins late startup before + closing anything it may still publish. A later probe or adapter cleanup + rejoins that same work and retries failed closers. + """ + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._operation: asyncio.Task[Any] | None = None + self._cleanup: asyncio.Task[None] | None = None + self._closers: list[_Close] = [] + self._cancel_requested = False + + def own(self, close: _Close) -> None: + """Retain an idempotent closer before awaiting resource startup.""" + if self._operation is None or self._operation.done(): + raise RuntimeError("A harness resource must belong to a running probe") + self._closers.append(close) + + async def run( + self, + operation: Callable[[], Awaitable[_T]], + *, + timeout_s: float, + ) -> _T: + """Run metadata discovery with separate operation and cleanup budgets.""" + async with self._lock: + await self.close(timeout_s=timeout_s) + enrolled = asyncio.get_running_loop().create_future() + + async def invoke() -> _T: + await enrolled + return await operation() + + task = asyncio.create_task(invoke()) + self._operation = task + self._cleanup = None + self._cancel_requested = False + enrolled.set_result(None) + try: + result = await asyncio.wait_for(asyncio.shield(task), timeout_s) + except BaseException as primary: + try: + await self.close(timeout_s=timeout_s) + except BaseException as cleanup: + if isinstance(primary, asyncio.CancelledError): + raise primary from cleanup + raise cleanup from primary + raise + await self.close(timeout_s=timeout_s) + return result + + async def close(self, *, timeout_s: float = 60.0) -> None: + """Join or retry settlement without abandoning an unfinished transport.""" + if self._operation is None and not self._closers: + return + self._cleanup = rejoin_owned( + self._cleanup, + self._finish, + label="Harness probe cleanup", + diagnostic_details=False, + ) + await report_owned(self._cleanup, timeout_s) + + async def _finish(self) -> None: + if (operation := self._operation) is not None: + if not operation.done() and not self._cancel_requested: + self._cancel_requested = True + operation.cancel() + await asyncio.gather(operation, return_exceptions=True) + + retained: list[_Close] = [] + failures: list[Exception] = [] + for close in reversed(self._closers): + try: + await close() + except Exception as error: # noqa: BLE001 -- attempt every closer while retaining each failed resource + retained.append(close) + failures.append(error) + self._closers = list(reversed(retained)) + if failures: + raise CleanupIncompleteError("Harness probe resources remain owned") from ( + ExceptionGroup("Harness probe cleanup failed", failures) + ) + self._operation = None + self._cancel_requested = False + + +__all__ = ["PreflightResult"] diff --git a/src/band/core/model_catalog.py b/src/band/core/model_catalog.py index 8381b96b7..f0b95f4f1 100644 --- a/src/band/core/model_catalog.py +++ b/src/band/core/model_catalog.py @@ -52,6 +52,8 @@ class ModelChoice(BaseModel): label: str | None = None efforts: tuple[str, ...] | None = None default_effort: str | None = None + provider: str | None = None + is_default: bool = False class ModelCatalog(BaseModel): diff --git a/src/band/core/retention/cleanup.py b/src/band/core/retention/cleanup.py index 0a0822300..fe9a6059b 100644 --- a/src/band/core/retention/cleanup.py +++ b/src/band/core/retention/cleanup.py @@ -20,16 +20,25 @@ _reporters: Counter[asyncio.Task[Any]] = Counter() # Finished tasks whose failure a caller re-raised or this module logged. _reported: weakref.WeakSet[asyncio.Task[Any]] = weakref.WeakSet() -_labels: weakref.WeakKeyDictionary[asyncio.Task[Any], str] = weakref.WeakKeyDictionary() +_labels: weakref.WeakKeyDictionary[asyncio.Task[Any], tuple[str, bool]] = ( + weakref.WeakKeyDictionary() +) def spawn_owned( - operation: Coroutine[object, object, None], *, label: str + operation: Coroutine[object, object, None], + *, + label: str, + diagnostic_details: bool = True, ) -> asyncio.Task[None]: """Run work that outlives a cancelled caller; log its failure only when - no caller is waiting to report it.""" + no caller is waiting to report it. + + Disable diagnostic details when transport errors may contain credentials; + the original exception remains on the owned task for callers and retries. + """ task = asyncio.create_task(operation) - _labels[task] = label + _labels[task] = (label, diagnostic_details) task.add_done_callback(_log_unreported) return task @@ -39,23 +48,30 @@ def rejoin_owned( operation: Callable[[], Coroutine[object, object, None]], *, label: str, + diagnostic_details: bool = True, ) -> asyncio.Task[None]: """Join the running or settled owned task; start a new one only after it failed, so concurrent callers share one attempt and a retry follows a settled failure.""" if current is None or (current.done() and task_failure(current) is not None): - return spawn_owned(operation(), label=label) + return spawn_owned( + operation(), label=label, diagnostic_details=diagnostic_details + ) return current def _log_unreported(task: asyncio.Task[Any]) -> None: if task in _reporters or task in _reported or task.cancelled(): return - if (label := _labels.get(task)) is None: + if (metadata := _labels.get(task)) is None: return _reported.add(task) if (error := task.exception()) is not None: - logger.warning("%s failed: %s", label, error, exc_info=error) + label, diagnostic_details = metadata + if diagnostic_details: + logger.warning("%s failed: %s", label, error, exc_info=error) + else: + logger.warning("%s failed (%s)", label, type(error).__name__) def task_failure(task: asyncio.Task[Any]) -> BaseException | None: diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index 39623e2a0..fc060773c 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -12,9 +12,11 @@ from band.client.rest import AsyncRestClient from band.client.streaming import ControlMode +from band.core.activity import sync_activity_tool_names from band.core.disposal import AdapterDisposal, CleanupCallback from band.core.exceptions import BandConfigError, InputNotAccepted from band.core.handoff import context_handoff +from band.core.harness import HarnessProbe, PreflightResult from band.core.model_catalog import ( ModelCatalog, ModelSelection, @@ -140,6 +142,7 @@ def __init__( # platform access read it via require_platform(). self.platform: PlatformConnection | None = None self.features = self._resolve_features(**features) + self._harness_probe_owner: HarnessProbe | None = None def add_cleanup(self, callback: CleanupCallback) -> None: """Register async resource cleanup before host startup, e.g. client.aclose. @@ -345,6 +348,8 @@ async def cleanup_all(self) -> None: Resources the caller supplies release through ``add_cleanup`` instead, only after work and this teardown settle; a direct call leaves them pending. """ + if self._harness_probe_owner is not None: + await self._harness_probe_owner.close() @property def model_selection(self) -> ModelSelection: @@ -366,6 +371,35 @@ async def list_models(self) -> ModelCatalog | None: """ return None + @property + def _harness_probe(self) -> HarnessProbe: + if self._harness_probe_owner is None: + self._harness_probe_owner = HarnessProbe() + return self._harness_probe_owner + + async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: + """Check a throwaway harness handshake without starting a Band room. + + A passed handshake does not prove provider authentication or inference + access. Cleanup failures propagate and stay owned for ``cleanup_all``. + """ + del timeout_s + return PreflightResult.failed( + f"{type(self).__name__} does not support harness preflight.", + "Use the harness's own startup check.", + ) + + async def probe_models(self, *, timeout_s: float = 60.0) -> ModelCatalog: + """Discover models without using or creating a Band room. + + Unlike ``list_models``, this explicit host operation may start a + throwaway transport. Cleanup failures remain owned by this adapter. + """ + del timeout_s + raise BandConfigError( + f"{type(self).__name__} does not support standalone model discovery" + ) + async def apply_model_selection( self, selection: ModelSelection, *, room_id: str ) -> None: @@ -441,6 +475,9 @@ async def on_event(self, inp: AgentInput) -> None: else: converted_history = inp.history + # One turn, one gate decision: the live tool-name detail follows + # the same choice as tool-call narration (see runtime/activity.py). + sync_activity_tool_names(inp.tools, self.features.emit) await self.run_judged_turn( msg=inp.msg, tools=inp.tools, diff --git a/src/band/integrations/acp/client_adapter.py b/src/band/integrations/acp/client_adapter.py index e8fc99fae..dae1933b5 100644 --- a/src/band/integrations/acp/client_adapter.py +++ b/src/band/integrations/acp/client_adapter.py @@ -12,7 +12,12 @@ Mapping, Sequence, ) -from contextlib import AsyncExitStack, asynccontextmanager +from contextlib import ( + AbstractAsyncContextManager, + AsyncExitStack, + asynccontextmanager, + nullcontext, +) from dataclasses import dataclass from enum import StrEnum from functools import partial @@ -890,6 +895,7 @@ async def _prompt_room( self._install_turn_handlers( runtime, emitter=emitter, + tools=tools, room_id=room_id, session_id=session_id, ) @@ -937,6 +943,8 @@ async def _prompt_room( except Exception as error: await self._fail_turn(room, generation, tools, error) raise + finally: + runtime.set_activity_tools(session_id, None) @staticmethod async def _fail_turn( @@ -1015,9 +1023,11 @@ def _install_turn_handlers( runtime: ACPRuntime, *, emitter: RoomTurnEmitter, + tools: AgentToolsProtocol, room_id: str, session_id: str, ) -> None: + runtime.set_activity_tools(session_id, tools) runtime.set_permission_handler( session_id, self._make_permission_handler(emitter, room_id), @@ -1059,6 +1069,7 @@ async def handler( options=options, room_id=room_id, session_id=session_id, + emitter=emitter, ) logger.info( @@ -1103,6 +1114,11 @@ async def _narrate_cancelled_permission( else: await narrate(narration) + def _permission_activity( + self, emitter: RoomTurnEmitter | None + ) -> AbstractAsyncContextManager[None]: + return emitter.permission_wait() if emitter is not None else nullcontext() + async def _resolve_permission_option( self, *, @@ -1110,20 +1126,22 @@ async def _resolve_permission_option( options: object, room_id: str, session_id: str, + emitter: RoomTurnEmitter | None = None, ) -> str | None: """Return a validated permission choice, or cancel the request.""" if self._resolve_permission is None: return select_allow_option_id(options) offered = self._permission_options(options) - option_id = await self._resolve_permission( - ACPPermissionRequest( - room_id=room_id, - session_id=session_id, - tool_call=call, - options=offered, + async with self._permission_activity(emitter): + option_id = await self._resolve_permission( + ACPPermissionRequest( + room_id=room_id, + session_id=session_id, + tool_call=call, + options=offered, + ) ) - ) if option_id is None: return None if not isinstance(option_id, str): diff --git a/src/band/integrations/acp/collecting.py b/src/band/integrations/acp/collecting.py index b576f7618..8287d7e8c 100644 --- a/src/band/integrations/acp/collecting.py +++ b/src/band/integrations/acp/collecting.py @@ -5,10 +5,12 @@ import asyncio import logging from collections.abc import Awaitable, Callable, Sequence +from typing import NamedTuple from acp.interfaces import Client from acp.schema import ConfigOptionUpdate, DeclineElicitationResponse +from band.core.protocols import AgentToolsProtocol from band.integrations.acp.client_profiles import ACPClientProfile, NoopACPClientProfile from band.integrations.acp.permissions import ( ElicitationHandler, @@ -18,12 +20,33 @@ ) from band.integrations.acp.session_config import SessionConfigOption from band.integrations.acp.stream import ACPChunkStream, ChunkSink -from band.integrations.acp.types import CollectedChunk +from band.integrations.acp.types import ( + ACPToolCall, + ChunkType, + CollectedChunk, + ToolStatus, +) from band.integrations.acp.updates import ACPUpdateParser +from band.runtime.working_state import ( + UNKNOWN_ACTIVITY_TOOL_NAME, + end_tool_activity, + report_tool_activity, +) logger = logging.getLogger(__name__) +class _ActivityCall(NamedTuple): + name: str + ended: bool + + +def _recorded_activity_name(call: _ActivityCall | None) -> str: + if call is None: + return UNKNOWN_ACTIVITY_TOOL_NAME + return call.name + + class ACPCollectingClient(ACPUpdateParser, Client): # type: ignore[misc] # ACP Client has optional methods treated as abstract by pyrefly """Generic ACP client that buffers session updates by session_id. @@ -44,6 +67,8 @@ def __init__( self._stream = ACPChunkStream() self._permission_handlers: dict[str, PermissionHandler] = {} self._elicitation_handlers: dict[str, ElicitationHandler] = {} + self._activity_tools: dict[str, AgentToolsProtocol] = {} + self._activity_calls: dict[str, dict[str, _ActivityCall]] = {} # Retain locks while a straggler can still hold one. self._session_locks: dict[str, asyncio.Lock] = {} # Each session's latest advertised config catalog; like the locks, @@ -76,8 +101,47 @@ async def session_update( async with self._session_lock(session_id): chunk = self._chunk_from_update(update) if chunk is not None: + await self._observe_activity(session_id, update, chunk) await self._stream.ingest(session_id, chunk) + def set_activity_tools( + self, session_id: str, tools: AgentToolsProtocol | None + ) -> None: + if tools is None: + self._activity_tools.pop(session_id, None) + self._activity_calls.pop(session_id, None) + else: + self._activity_tools[session_id] = tools + self._activity_calls[session_id] = {} + + async def _observe_activity( + self, session_id: str, update: object, chunk: CollectedChunk + ) -> None: + tools = self._activity_tools.get(session_id) + if tools is None or chunk.chunk_type not in { + ChunkType.TOOL_CALL, + ChunkType.TOOL_RESULT, + }: + return + call_id = str(getattr(update, "tool_call_id", "")) + calls = self._activity_calls[session_id] + previous = calls.get(call_id) + status = getattr(update, "status", None) + if status in {ToolStatus.COMPLETED, ToolStatus.FAILED}: + if previous is None or not previous.ended: + calls[call_id] = _ActivityCall(_recorded_activity_name(previous), True) + await end_tool_activity(tools) + return + if previous is not None and previous.ended: + return + name = _recorded_activity_name(previous) + call = chunk.tool + if isinstance(call, ACPToolCall) and call.mcp_server is not None: + name = call.name + if previous is None or previous.name != name: + calls[call_id] = _ActivityCall(name, False) + await report_tool_activity(tools, name) + async def request_permission( # type: ignore[override] # ACP Client uses specific types; we widen to object self, options: object, @@ -155,6 +219,7 @@ def config_options(self, session_id: str) -> tuple[SessionConfigOption, ...]: def forget_session(self, session_id: str) -> None: """Drop a closed session's catalog and any state its profile keeps.""" + self.set_activity_tools(session_id, None) self._config_options.pop(session_id, None) # A custom profile written before this hook has none. forget = getattr(self._profile, ACPClientProfile.forget_session.__name__, None) @@ -162,6 +227,7 @@ def forget_session(self, session_id: str) -> None: forget(session_id) def reset_session(self, session_id: str) -> None: + self.set_activity_tools(session_id, None) self._stream.reset_session(session_id) self._permission_handlers.pop(session_id, None) self._elicitation_handlers.pop(session_id, None) diff --git a/src/band/integrations/acp/room_emitter.py b/src/band/integrations/acp/room_emitter.py index 07b8d5525..91c3c7027 100644 --- a/src/band/integrations/acp/room_emitter.py +++ b/src/band/integrations/acp/room_emitter.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from typing import Self from band.core.content import has_visible_content @@ -16,6 +18,7 @@ ToolStatus, ) from band.runtime.tools import TurnEffect, turn_effect +from band.runtime.working_state import set_permission_wait logger = logging.getLogger(__name__) @@ -77,6 +80,17 @@ def __init__( self._emit = frozenset(emit) if emit is not None else frozenset(Emit) self._pending_text: list[str] = [] self._staged_effects: list[TurnEffect] = [] + self._manual_waits = 0 + + @asynccontextmanager + async def permission_wait(self) -> AsyncIterator[None]: + self._manual_waits += 1 + try: + await set_permission_wait(self._tools, True) + yield + finally: + self._manual_waits -= 1 + await set_permission_wait(self._tools, self._manual_waits > 0) async def emit(self, chunk: CollectedChunk) -> None: self._stage_tool_outcome(chunk) diff --git a/src/band/integrations/acp/sessions.py b/src/band/integrations/acp/sessions.py index 7d92d470b..75e724135 100644 --- a/src/band/integrations/acp/sessions.py +++ b/src/band/integrations/acp/sessions.py @@ -16,6 +16,7 @@ SetSessionConfigOptionResponse, ) +from band.core.protocols import AgentToolsProtocol from band.integrations.acp.collecting import ACPCollectingClient from band.integrations.acp.permissions import ElicitationHandler, PermissionHandler from band.integrations.acp.session_config import ( @@ -235,6 +236,12 @@ def reset_session(self, session_id: str) -> None: if self._client is not None: self._client.reset_session(session_id) + def set_activity_tools( + self, session_id: str, tools: AgentToolsProtocol | None + ) -> None: + if self._client is not None: + self._client.set_activity_tools(session_id, tools) + def set_permission_handler( self, session_id: str, diff --git a/src/band/integrations/claude_sdk/dedup_tools.py b/src/band/integrations/claude_sdk/dedup_tools.py index 70f9716f4..2016cd31b 100644 --- a/src/band/integrations/claude_sdk/dedup_tools.py +++ b/src/band/integrations/claude_sdk/dedup_tools.py @@ -42,6 +42,8 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any +from band.runtime.working_state import ActivityObserver, _observer + if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -199,6 +201,10 @@ async def send_message( # --- transparent passthrough -------------------------------------- + @property + def _activity_observer(self) -> ActivityObserver | None: + return _observer(self._inner) + def __getattr__(self, name: str) -> Any: # ``__getattr__`` only runs for attributes not found via the # normal MRO. ``send_message`` is defined above, so it never diff --git a/src/band/integrations/codex/rpc_base.py b/src/band/integrations/codex/rpc_base.py index 2fd379ff3..3a2e62e04 100644 --- a/src/band/integrations/codex/rpc_base.py +++ b/src/band/integrations/codex/rpc_base.py @@ -263,11 +263,21 @@ async def _dispatch_rpc_message(self, text: str) -> None: try: message = json.loads(text) except json.JSONDecodeError: - logger.warning("Skipping non-JSON %s: %s", self._transport_label, text) + logger.warning( + "Skipping non-JSON %s (type=%s, length=%d)", + self._transport_label, + type(text).__name__, + len(text), + ) return if not isinstance(message, dict): - logger.warning("Skipping non-object JSON-RPC payload: %s", message) + logger.warning( + "Skipping non-object JSON-RPC %s (type=%s, length=%d)", + self._transport_label, + type(message).__name__, + len(text), + ) return msg_id = message.get("id") @@ -285,7 +295,7 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning("Event queue full; dropping server request %s", method) + logger.warning("Event queue full; dropping server request") return if method: @@ -300,7 +310,7 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning("Event queue full; dropping notification %s", method) + logger.warning("Event queue full; dropping notification") return if msg_id is not None: @@ -309,7 +319,12 @@ async def _dispatch_rpc_message(self, text: str) -> None: future.set_result(message) return - logger.debug("Ignoring unknown JSON-RPC payload: %s", message) + logger.debug( + "Ignoring unknown JSON-RPC %s (type=%s, length=%d)", + self._transport_label, + type(message).__name__, + len(text), + ) def _next_request_id(self) -> int: self._request_id += 1 diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index c87961e5a..bd2007157 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -95,7 +95,7 @@ async def _wait_readers(readers: Sequence[asyncio.Task[None]]) -> None: return _done, pending = await asyncio.wait(readers, timeout=_EXIT_OBSERVE_S) if pending: - logger.warning("Codex stdio readers still pending after tree kill") + raise TimeoutError("Codex stdio readers still pending after tree kill") async def kill_process_tree(process: asyncio.subprocess.Process) -> None: @@ -113,11 +113,17 @@ async def kill_process_tree(process: asyncio.subprocess.Process) -> None: class CodexStdioClient(BaseJsonRpcClient): - """Bidirectional JSON-RPC client with a single stdout read loop.""" + """Bidirectional JSON-RPC client with a single stdout read loop. + + Transport logs report metadata only; callers still receive original errors. + ``graceful_shutdown=False`` requests immediate owned POSIX termination before + ordinary settlement; the default leaves that settlement policy unchanged. + """ def __init__( self, *, + graceful_shutdown: bool = True, command: Sequence[str] | None = None, cwd: str | None = None, env: Mapping[str, str] | None = None, @@ -137,6 +143,8 @@ def __init__( self._proc: asyncio.subprocess.Process | None = None self._windows_process: WindowsProcess | None = None + self._creation_task: asyncio.Task[None] | None = None + self._graceful_shutdown = graceful_shutdown self._reader_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None self._close_task: asyncio.Task[None] | None = None @@ -158,6 +166,7 @@ async def connect(self) -> None: if self._close_task is not None: await self.close() self._close_task = None + self._creation_task = None if self._connected: return @@ -186,8 +195,24 @@ async def _spawn(self) -> asyncio.subprocess.Process: except BaseException: await self.close() raise + if self._creation_task is not None: + raise BandConnectionError("Codex process is already starting") + self._creation_task = asyncio.create_task(self._create_process()) + try: + # A cancelled caller cannot discard a child created before the + # subprocess coroutine returns its Process handle. + await asyncio.shield(self._creation_task) + if self._close_task is not None: + raise BandConnectionError("Codex process closed during startup") + assert self._proc is not None + return self._proc + except BaseException: + await self.close() + raise + + async def _create_process(self) -> None: # The child leads its own session, so its pid is the group id. - return await asyncio.create_subprocess_exec( + self._proc = await asyncio.create_subprocess_exec( *self.command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, @@ -224,7 +249,11 @@ def _spawn_connection_error(self, failure: SpawnFailure) -> BandConnectionError: async def close(self) -> None: """Share retryable cleanup that survives a cancelled or timed-out caller.""" - if self._windows_process is None and self._proc is None: + if ( + self._windows_process is None + and self._proc is None + and self._creation_task is None + ): return if self._windows_process is not None: self._windows_process.closing = True @@ -239,18 +268,41 @@ async def close(self) -> None: self._close_task.add_done_callback(self._log_close_failure) await asyncio.shield(self._close_task) - @staticmethod - def _log_close_failure(task: asyncio.Task[None]) -> None: + def _log_close_failure(self, task: asyncio.Task[None]) -> None: if not task.cancelled() and (error := task.exception()) is not None: - logger.error("Codex subprocess cleanup failed", exc_info=error) + logger.error( + "Codex subprocess cleanup failed (type=%s)", type(error).__name__ + ) async def _close_process(self) -> None: + await self._settle_process() + if self._proc is not None and self._proc.returncode is None: + raise TimeoutError( + f"Codex process {self._proc.pid} still alive after tree kill" + ) + + async def _settle_process(self) -> None: readers = [ task for task in (self._reader_task, self._stderr_task) if task is not None ] for task in readers: task.cancel() try: + if self._creation_task is not None: + done, _pending = await asyncio.wait( + [self._creation_task], timeout=_EXIT_OBSERVE_S + ) + if not done: + raise TimeoutError("Codex process creation is still pending") + if not self._creation_task.cancelled(): + self._creation_task.exception() + if ( + not self._graceful_shutdown + and os.name != "nt" + and (process := self._proc) is not None + ): + await close_stdin(process) + await kill_process_tree(process) # A cancelled pipe read on Windows stays pending while the child is # blocked in a synchronous stdin read, and awaiting that read never # times out. Terminate the tree and close our write end first. @@ -291,8 +343,8 @@ async def _read_stdout_loop(self) -> None: await self._dispatch_rpc_message(line.decode("utf-8", errors="replace")) except asyncio.CancelledError: raise - except Exception: - logger.exception("Codex stdout loop failed") + except Exception: # noqa: BLE001 -- fail pending RPCs without logging sensitive exception text + logger.error("Codex stdout loop failed") self._fail_pending("Codex stdout loop failed") async def _read_stderr_loop(self) -> None: @@ -303,10 +355,9 @@ async def _read_stderr_loop(self) -> None: line = await self._proc.stderr.readline() if not line: return - text = line.decode("utf-8", errors="replace").rstrip() - if text: - logger.debug("codex stderr: %s", text) + if line.rstrip(): + logger.debug("codex stderr (type=bytes, length=%d)", len(line)) except asyncio.CancelledError: raise - except Exception: - logger.exception("Codex stderr loop failed") + except Exception: # noqa: BLE001 -- contain reader faults without logging sensitive exception text + logger.error("Codex stderr loop failed") diff --git a/src/band/integrations/codex/types.py b/src/band/integrations/codex/types.py index 641fb9430..108c02269 100644 --- a/src/band/integrations/codex/types.py +++ b/src/band/integrations/codex/types.py @@ -71,6 +71,7 @@ class CodexRequestMethod(StrEnum): """Client-to-server request methods the SDK sends to ``codex app-server``.""" INITIALIZE = "initialize" + ACCOUNT_READ = "account/read" MODEL_LIST = "model/list" SKILLS_EXTRA_ROOTS_SET = "skills/extraRoots/set" THREAD_RESUME = "thread/resume" diff --git a/src/band/integrations/codex/winspawn.py b/src/band/integrations/codex/winspawn.py index a7306ba9c..75520e9d0 100644 --- a/src/band/integrations/codex/winspawn.py +++ b/src/band/integrations/codex/winspawn.py @@ -32,12 +32,16 @@ _JOB_POLL_S = 0.05 _ERROR_ALREADY_EXISTS = 183 _ERROR_BROKEN_PIPE = 109 +_ERROR_INVALID_PARAMETER = 87 +_ERROR_MORE_DATA = 234 _STATUS_READ_SIZE = 256 _PATH_BUFFER_SIZE = 260 _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 +_JOB_OBJECT_LIMIT_ACTIVE_PROCESS = 0x00000008 _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 _JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION = 1 +_JOB_OBJECT_BASIC_PROCESS_ID_LIST = 3 _JOB_OBJECT_ASSIGN_PROCESS = 0x0001 _CREATE_SUSPENDED = 0x00000004 @@ -46,7 +50,11 @@ _HANDLE_FLAG_INHERIT = 0x00000001 _PROC_THREAD_ATTRIBUTE_HANDLE_LIST = 0x00020002 _PROC_THREAD_ATTRIBUTE_JOB_LIST = 0x0002000D +_PROCESS_QUERY_LIMITED_INFORMATION = 0x00001000 +_SYNCHRONIZE = 0x00100000 _INFINITE = 0xFFFFFFFF +_WAIT_OBJECT_0 = 0 +_WAIT_TIMEOUT = 0x00000102 _WAIT_FAILED = 0xFFFFFFFF @@ -60,7 +68,7 @@ def __init__(self, winerror: int | None) -> None: class JobNotDrained(Exception): - """Termination was requested and members were still active afterwards.""" + """Termination has not established that every process exited.""" def __init__(self, active: int | None) -> None: self.active = active @@ -116,6 +124,14 @@ class BasicAccountingInformation(ctypes.Structure): ) +class BasicProcessIdList(ctypes.Structure): + _fields_ = ( + ("NumberOfAssignedProcesses", wintypes.DWORD), + ("NumberOfProcessIdsInList", wintypes.DWORD), + ("ProcessIdList", ctypes.c_size_t * 1), + ) + + class StartupInfo(ctypes.Structure): _fields_ = ( ("cb", wintypes.DWORD), @@ -201,6 +217,10 @@ def _kernel32() -> Any: library.CloseHandle.restype = wintypes.BOOL library.OpenJobObjectW.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR] library.OpenJobObjectW.restype = handle + library.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + library.OpenProcess.restype = handle + library.IsProcessInJob.argtypes = [handle, handle, ctypes.POINTER(wintypes.BOOL)] + library.IsProcessInJob.restype = wintypes.BOOL library.CreateProcessW.argtypes = [ wintypes.LPCWSTR, ctypes.POINTER(ctypes.c_wchar), @@ -363,8 +383,12 @@ class Job: def __init__(self, handle: ctypes.c_void_p, name: str) -> None: self._handle: ctypes.c_void_p | None = handle self.name = name + self._processes: dict[int, ctypes.c_void_p] = {} + self._members_captured = False def close(self) -> None: + if self._processes: + raise JobNotDrained(None) handle = self._handle if handle is not None and not _kernel32().CloseHandle(handle): raise _win_os_error(_last_error()) @@ -394,26 +418,118 @@ def active_processes(self) -> int | None: return None return int(info.ActiveProcesses) + def _seal_membership(self) -> None: + # A zero limit denies late children without terminating existing members. + limits = ExtendedLimitInformation() + limits.BasicLimitInformation.LimitFlags = ( + _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | _JOB_OBJECT_LIMIT_ACTIVE_PROCESS + ) + limits.BasicLimitInformation.ActiveProcessLimit = 0 + if not _kernel32().SetInformationJobObject( + self._open_handle(), + _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, + ctypes.byref(limits), + ctypes.sizeof(limits), + ): + raise _win_os_error(_last_error()) + + def _member_pids(self) -> tuple[int, ...]: + capacity = 1 + while True: + buffer = ctypes.create_string_buffer( + ctypes.sizeof(BasicProcessIdList) + + (capacity - 1) * ctypes.sizeof(ctypes.c_size_t) + ) + info = ctypes.cast(buffer, ctypes.POINTER(BasicProcessIdList)).contents + complete = _kernel32().QueryInformationJobObject( + self._open_handle(), + _JOB_OBJECT_BASIC_PROCESS_ID_LIST, + buffer, + ctypes.sizeof(buffer), + None, + ) + if not complete and (code := _last_error()) != _ERROR_MORE_DATA: + raise _win_os_error(code) + assigned = int(info.NumberOfAssignedProcesses) + count = int(info.NumberOfProcessIdsInList) + if not complete or count < assigned: + capacity = max(capacity * 2, assigned) + continue + values = ctypes.cast( + ctypes.byref(buffer, BasicProcessIdList.ProcessIdList.offset), + ctypes.POINTER(ctypes.c_size_t), + ) + return tuple(int(values[index]) for index in range(count)) + + def _capture_members(self) -> None: + for pid in self._member_pids(): + if pid in self._processes: + continue + process = _kernel32().OpenProcess( + _SYNCHRONIZE | _PROCESS_QUERY_LIMITED_INFORMATION, False, pid + ) + if not process: + code = _last_error() + if code == _ERROR_INVALID_PARAMETER and pid not in self._member_pids(): + continue + raise _win_os_error(code) + self._processes[pid] = process + # Retained handles keep object identity stable across PID reuse. Failed + # membership checks keep the handles owned, but cannot authorize termination. + for pid, process in tuple(self._processes.items()): + member = wintypes.BOOL() + if not _kernel32().IsProcessInJob( + process, self._open_handle(), ctypes.byref(member) + ): + raise _win_os_error(_last_error()) + if not member.value: + self._close_process_handle(pid) + + def _close_process_handle(self, pid: int) -> None: + if not _kernel32().CloseHandle(self._processes[pid]): + raise _win_os_error(_last_error()) + del self._processes[pid] + + def _reap_members(self) -> None: + for pid, process in tuple(self._processes.items()): + state = _kernel32().WaitForSingleObject(process, 0) + if state == _WAIT_TIMEOUT: + continue + if state != _WAIT_OBJECT_0: + raise _win_os_error(_last_error()) + self._close_process_handle(pid) + def terminate(self) -> None: - """Mark every member for termination. + """Retain physical-exit evidence before marking members for termination. Members blocked in a synchronous pipe read stay inside that call until the other end closes. Wait only after that end is closed. """ + if not self._members_captured: + self._seal_membership() + self._capture_members() + self._members_captured = True if not _kernel32().TerminateJobObject(self._open_handle(), 1): raise _win_os_error(_last_error()) async def wait_until_idle(self, timeout: float) -> None: - """Observe tree exit without releasing ownership of the job.""" + """Observe physical exit and job accounting without releasing the job.""" + if self._processes and not self._members_captured: + raise JobNotDrained(None) loop = asyncio.get_running_loop() deadline = loop.time() + timeout while True: + self._reap_members() active = self.active_processes() - if active == 0: + if active == 0 and not self._processes: return remaining = deadline - loop.time() if active is None or remaining <= 0: - logger.warning("Codex job still has %s active processes", active) + logger.warning( + "Codex job still has %s active processes and %s pending exits", + active, + len(self._processes), + ) raise JobNotDrained(active) await asyncio.sleep(min(_JOB_POLL_S, remaining)) diff --git a/src/band/integrations/crewai/catalog.py b/src/band/integrations/crewai/catalog.py index e33df22f1..8105d6ba4 100644 --- a/src/band/integrations/crewai/catalog.py +++ b/src/band/integrations/crewai/catalog.py @@ -29,6 +29,7 @@ serialize_tool_result, validate_tool_arguments, ) +from band.runtime.working_state import model_tool_activity # --- Rendering a tool result as the text CrewAI hands the model --- @@ -288,9 +289,10 @@ async def _send_event(call: Invocation, **kwargs: Any) -> Any: # bypassing args_schema, and _execute_tool turns the ValueError into the # error result the caller expects. args = validate_tool_arguments(BandTool.SEND_EVENT, SEND_EVENT_ARGS_SCHEMA, kwargs) - await call.tools.send_event( - args["content"], args["message_type"], metadata=args.get("metadata") - ) + async with model_tool_activity(call.tools, BandTool.SEND_EVENT): + await call.tools.send_event( + args["content"], args["message_type"], metadata=args.get("metadata") + ) return succeeded("Event sent") diff --git a/src/band/integrations/crewai/tools.py b/src/band/integrations/crewai/tools.py index 104e1a573..e2817c795 100644 --- a/src/band/integrations/crewai/tools.py +++ b/src/band/integrations/crewai/tools.py @@ -26,6 +26,8 @@ from pydantic import BaseModel +from band.runtime.working_state import tool_activity + if TYPE_CHECKING: from crewai.tools import BaseTool @@ -95,7 +97,8 @@ def _execute_tool( async def _execute() -> str: try: - return await coro_factory(tools) + async with tool_activity(tools, tool_name): + return await coro_factory(tools) except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn error_msg = str(e) if tool_name == BandTool.SEND_MESSAGE and isinstance( diff --git a/src/band/integrations/mcp/engine.py b/src/band/integrations/mcp/engine.py index c89c6ea4f..8ce0322bb 100644 --- a/src/band/integrations/mcp/engine.py +++ b/src/band/integrations/mcp/engine.py @@ -64,6 +64,7 @@ ) from band.runtime.tools.inputs.chat import require_visible_content from band.runtime.tools.inputs.identifiers import ResourceId +from band.runtime.working_state import model_tool_activity logger = logging.getLogger(__name__) @@ -219,7 +220,8 @@ async def dispatch_tool( """ method = resolve_tool_method(tools, definition) try: - return await method(**arguments) + async with model_tool_activity(tools, definition.name): + return await method(**arguments) except (ValueError, BandToolError) as error: raise enrich_send_message_error(definition, tools, error) from error diff --git a/src/band/integrations/parlant/bandtools/chat.py b/src/band/integrations/parlant/bandtools/chat.py index 6886419d2..0c78221b2 100644 --- a/src/band/integrations/parlant/bandtools/chat.py +++ b/src/band/integrations/parlant/bandtools/chat.py @@ -16,6 +16,7 @@ ) from band.integrations.parlant.sessiontools import require_session_tools from band.runtime.tools import ParticipantAddStatus, serialize_tool_result +from band.runtime.working_state import model_tool_activity EVENT_MESSAGE_TYPES: tuple[str, ...] = get_args(EventMessageType) @@ -28,6 +29,8 @@ "the 'data'/'metadata' dict described above." ) +from band.runtime.tools.types import BandTool + @band_tool( "sending message", @@ -59,7 +62,8 @@ async def band_send_event( if message_type not in EVENT_MESSAGE_TYPES: return invalid_choice("message_type", message_type, EVENT_MESSAGE_TYPES) - await tools.send_event(content, message_type, None) + async with model_tool_activity(tools, BandTool.SEND_EVENT): + await tools.send_event(content, message_type, None) return ToolResult(data=f"Event ({message_type}) sent successfully") diff --git a/src/band/integrations/parlant/customtools.py b/src/band/integrations/parlant/customtools.py index 0f5c1f8d9..eb2ea0334 100644 --- a/src/band/integrations/parlant/customtools.py +++ b/src/band/integrations/parlant/customtools.py @@ -17,6 +17,7 @@ ) from band.runtime.custom_tools import CustomToolDef, execute_custom_tool from band.runtime.tools import tool_result_text +from band.runtime.working_state import tool_activity # Parlant delivers every argument as a string, which only lax validation # converts, so even a strict field is validated laxly. @@ -30,12 +31,13 @@ def build_custom_tool(tool_def: CustomToolDef) -> Any: async def run(context: ToolContext, **arguments: Any) -> ToolResult: tools = require_session_tools(context) - result = await execute_custom_tool( - tool_def, - parse_delivered(arguments, described.fields), - turn=tools.turn, - strict=STRICT_VALIDATION, - ) + async with tool_activity(tools, described.name): + result = await execute_custom_tool( + tool_def, + parse_delivered(arguments, described.fields), + turn=tools.turn, + strict=STRICT_VALIDATION, + ) return ToolResult(data=tool_result_text(result)) run.__name__ = described.name diff --git a/src/band/integrations/pydantic_ai/tools.py b/src/band/integrations/pydantic_ai/tools.py index a9d724b3b..caa967a8a 100644 --- a/src/band/integrations/pydantic_ai/tools.py +++ b/src/band/integrations/pydantic_ai/tools.py @@ -37,6 +37,7 @@ turn_effect, validate_tool_arguments, ) +from band.runtime.working_state import model_tool_activity logger = logging.getLogger(__name__) @@ -202,7 +203,8 @@ async def dispatch(ctx: RunContext[AgentToolsProtocol], **arguments: Any) -> Any # something to hand the model as a tool error it could act on. method = _resolve_method(ctx.deps, definition) try: - result = await method(**kwargs) + async with model_tool_activity(ctx.deps, definition.name): + result = await method(**kwargs) except Exception as error: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn return await _dispatch_failed(ctx, definition, error, notify_room=True) try: diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index 2842bf067..cdc8c6412 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -162,6 +162,7 @@ def __init__( # adapter judging ``wrap`` sees what the brain did through this tee. self._ctx = wrap._ctx self._scope = wrap._scope + self._activity_observer = wrap._activity_observer self.turn = wrap.turn self._slack = slack self._binding = binding diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py new file mode 100644 index 000000000..b9c9b6e3f --- /dev/null +++ b/src/band/platform/activity.py @@ -0,0 +1,66 @@ +"""Process-wide negotiation for optional room activity detail. + +Every agent API response names its version in ``X-API-Version``. Detail is sent +once any activity response reported a supporting version. A detail rejection on +any link permanently downgrades every link; boolean liveness remains available. +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Mapping +from threading import Lock + +logger = logging.getLogger(__name__) +# The first agent API that accepts ``activity``; older ones reject it with 422. +MINIMUM_ACTIVITY_VERSION = (1, 25, 0) +API_VERSION_HEADER = "x-api-version" +RELEASE_VERSION = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)") + + +def _supports_detail(version: str | None) -> bool: + match = RELEASE_VERSION.fullmatch(version) if version is not None else None + return ( + match is not None + and tuple(map(int, match.groups())) >= MINIMUM_ACTIVITY_VERSION + ) + + +class ActivitySupport: + """Learn detail support from response headers; a rejection latches it off.""" + + def __init__(self) -> None: + self._lock = Lock() + self._learned = False + self._disabled = False + + @property + def supported(self) -> bool: + with self._lock: + return self._learned and not self._disabled + + def observe(self, headers: Mapping[str, str]) -> None: + version = headers.get(API_VERSION_HEADER) + if not _supports_detail(version): + logger.debug( + "Agent API version %r does not advertise activity detail", version + ) + return + with self._lock: + self._learned = True + + def disable(self, *, room_id: str, field: str) -> None: + with self._lock: + already_disabled = self._disabled + self._disabled = True + if not already_disabled: + logger.info( + "Activity report with detail returned 422 in room %s (field %s); " + "using boolean reporting for this process", + room_id, + field, + ) + + +activity_support = ActivitySupport() diff --git a/src/band/platform/link.py b/src/band/platform/link.py index c512952d9..f85634185 100644 --- a/src/band/platform/link.py +++ b/src/band/platform/link.py @@ -13,6 +13,7 @@ from functools import partial from typing import TYPE_CHECKING +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import AgentTopicKind, SessionState from band.client.rest import AsyncRestClient @@ -647,6 +648,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -654,6 +656,7 @@ async def report_activity( self.rest, room_id, working, + activity=activity, timeout_seconds=timeout_seconds, on_stopped=on_stopped, ) diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index 2e48b52fb..2450cfe35 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio import logging from collections import OrderedDict from collections.abc import Awaitable, Callable @@ -14,9 +15,14 @@ from http import HTTPStatus from typing import Any, TypeVar +from band_rest.agent_api_activity.types.report_agent_chat_activity_response import ( + ReportAgentChatActivityResponse, +) from band_rest.core.api_error import ApiError from band_rest.core.http_response import AsyncHttpResponse from band_rest.core.request_options import RequestOptions +from band_rest.types.agent_activity_detail import AgentActivityDetail +from band_rest.types.agent_activity_response import AgentActivityResponse from band_rest.types.chat_message import ChatMessage from band_rest.types.error import Error as PlatformError from band_rest.types.validation_error import ValidationError as ErrorEnvelope @@ -32,6 +38,7 @@ from band.core.exceptions import RoomExecutionStoppedError from band.core.settlement import SettlementResult from band.core.types import metadata_to_dict +from band.platform.activity import activity_support from band.runtime.messages import PlatformMessage logger = logging.getLogger(__name__) @@ -47,6 +54,10 @@ PROCESSING_ATTEMPT_POINTER = "/processing_attempt" NOT_FOUND_ERROR_CODE = "not_found" +# A tool name over this length is a 422 on the activity endpoint, which the +# compatibility path would read as an old platform; clip first, never reject. +MAX_ACTIVITY_NAME_CHARACTERS = 1024 + def _accepted_response( response: AsyncHttpResponse[ResponseData | None], @@ -109,6 +120,63 @@ def _failure_result(error: Exception) -> SettlementResult: return SettlementResult.UNCONFIRMED +def _first_rejected_field(body: object) -> str: + """A content-free cause: the first JSON pointer a validation error names.""" + envelope = _validated(ErrorEnvelope, body) + if envelope is None: + return "unparsed" + return next(iter(envelope.error.details), "unnamed") + + +async def _post_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AgentActivityResponse: + detail = activity if activity_support.supported else None + try: + response = await _send_activity(rest, room_id, working, detail, timeout_seconds) + except ApiError as error: + if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: + raise + activity_support.disable( + room_id=room_id, field=_first_rejected_field(error.body) + ) + logger.debug("Activity 422 body for room %s: %r", room_id, error.body) + response = await _send_activity(rest, room_id, working, None, timeout_seconds) + activity_support.observe(response.headers) + return response.data.data + + +async def _send_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: + kwargs: dict[str, Any] = {} + if activity is not None: + if activity.name and len(activity.name) > MAX_ACTIVITY_NAME_CHARACTERS: + # I-08: clip before the POST, never let a long name read as an + # old platform (a detail-bearing 422 latches detail off). + activity = activity.model_copy( + update={"name": activity.name[:MAX_ACTIVITY_NAME_CHARACTERS]} + ) + kwargs["activity"] = activity + return await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + **NO_RETRY_REQUEST_OPTIONS, + "timeout_in_seconds": timeout_seconds, + }, + **kwargs, + ) + + def _platform_message(item: ChatMessage, room_id: str) -> PlatformMessage: return PlatformMessage( id=item.id, @@ -306,6 +374,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -318,26 +387,25 @@ async def report_activity( the backstop and activity reporting must never break message processing. - ``timeout_seconds`` bounds each POST so a slow/half-open endpoint - can't stall the keep-alive or wedge teardown. Retries are off: a - dropped keep-alive gets re-sent next cadence tick, and a dropped - ``false`` is cleared by the platform TTL either way — retrying would - only add latency. + Detail is attached once a response reported a supporting agent API + version. One deadline bounds the POST and its sole compatibility retry: + a rejected detail permanently downgrades the process to boolean + reports; other failures wait for the next keep-alive or platform TTL. A successful ``working=True`` report echoed as ``False`` identifies a stopped execution. Notify ``on_stopped`` separately from the request's success boolean; transport failures never imply STOP. """ try: - response = await rest.agent_api_activity.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - **NO_RETRY_REQUEST_OPTIONS, - "timeout_in_seconds": timeout_seconds, - }, - ) - stopped = working and response.data.working is False + async with asyncio.timeout(timeout_seconds): + response = await _post_activity( + rest, + room_id, + working, + activity if working else None, + timeout_seconds, + ) + stopped = working and response.working is False except Exception as e: # noqa: BLE001 -- best-effort event emission must not crash the turn/link if not self._activity_report_failing: self._activity_report_failing = True diff --git a/src/band/runtime/cycle.py b/src/band/runtime/cycle.py index 6cc42de9f..ef48c9672 100644 --- a/src/band/runtime/cycle.py +++ b/src/band/runtime/cycle.py @@ -18,6 +18,7 @@ class TurnScope: observer_task: asyncio.Task[None] | None = None handler: TurnTask | None = None admitted: bool = False + activity_allowed: bool = False settled: bool = False kind: ControlMode | None = None deadline: float | None = None diff --git a/src/band/runtime/decisions.py b/src/band/runtime/decisions.py index fee965171..795a3ba63 100644 --- a/src/band/runtime/decisions.py +++ b/src/band/runtime/decisions.py @@ -161,11 +161,16 @@ def cancel_all(self) -> list[DecisionEntry[T]]: return self._drop_cancelled(self._core.cancel_all()) async def wait( - self, entry: DecisionEntry[T], future: asyncio.Future[R], *, timeout_s: float + self, + entry: DecisionEntry[T], + future: asyncio.Future[R], + *, + timeout_s: float, + retain_entry: bool = False, ) -> R | Literal[Timeout.TIMED_OUT]: """The answer to ``entry``, or ``TIMED_OUT`` if the deadline claims it first or it was replaced; a reply that claimed it first is always - waited for.""" + waited for. With ``retain_entry``, the caller retires it after delivery.""" try: # asyncio.timeout(), not wait_for: on Python 3.11 wait_for returns # an answer that lands in the same tick as a cancel of this task @@ -184,7 +189,8 @@ async def wait( return await future return Timeout.TIMED_OUT finally: - self.forget(entry) + if not retain_entry: + self.forget(entry) def _holds(self, entry: DecisionEntry[T]) -> bool: # Tickets are per registry, so an entry from a registry a room has diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 971ee0d0f..94f9a6f3b 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -19,6 +19,7 @@ import logging from collections import deque from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum, StrEnum from functools import partial @@ -30,6 +31,7 @@ runtime_checkable, ) +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import ( AgentFailure, ClaimRegistry, @@ -45,6 +47,7 @@ MessageCreatedPayload, MessageMetadata, ) +from band.core.activity import ToolNamesObserver from band.core.exceptions import ( CleanupIncompleteError, InputNotAccepted, @@ -84,7 +87,7 @@ from band.runtime.resync import ResyncSchedule from band.runtime.sessionconfig import SessionConfig from band.runtime.tools.agent import AgentTools -from band.runtime.working_state import WorkingStateReporter +from band.runtime.working_state import ActivityObserver, WorkingStateReporter if TYPE_CHECKING: from band.platform.link import BandLink @@ -282,6 +285,18 @@ async def resume_room(self) -> None: ExecutionHandler = Callable[["ExecutionContext", PlatformEvent], Awaitable[None]] +@dataclass(frozen=True) +class ScopeActivityObserver: + context: ExecutionContext + scope: TurnScope + + async def update(self, activity: AgentActivityDetail) -> None: + await self.context.observe_activity(self.scope, activity=activity) + + async def set_manual_wait(self, waiting: bool) -> None: + await self.context.observe_activity(self.scope, waiting=waiting) + + class ExecutionContext: """ Default execution: context accumulation model. @@ -417,6 +432,10 @@ def __init__( self._activity_owner: TurnScope | None = None self._activity_stop_scope: TurnScope | None = None self._activity_completion: asyncio.Task[None] | None = None + # Tool names follow the adapter's tool-narration choice; the default + # matches emit's opt-out semantics, so only adapters that drop + # Emit.TOOL_CALLS (via sync_activity_tool_names) hide them. + self._activity_tool_names = True self._control_revision = 0 self._on_platform_stop = on_platform_stop self._shutdown_task: asyncio.Task[bool] | None = None @@ -432,11 +451,6 @@ def __init__( # The first stop's graceful result; a retry only settles retained work. self._stop_graceful = True - # Optional seam for clearing user-visible activity state ("reasoning…") - # when a cycle is interrupted/stopped. Filled by the activity-signal - # work; a no-op (None) here. - self._on_activity_clear: Callable[[], Awaitable[None]] | None = None - @property def thread_id(self) -> str: """LangGraph thread_id = room_id.""" @@ -871,17 +885,6 @@ async def _stop_retained_tasks(self, deadline: float) -> None: "task(s) still running" ) - async def _clear_activity(self) -> None: - """Invoke the optional activity-clear seam after an aborted cycle.""" - if self._on_activity_clear is None: - return - try: - await self._on_activity_clear() - except Exception: - logger.exception( - "ExecutionContext %s: activity-clear hook failed", self.room_id - ) - # --- Participant management --- def add_participant(self, participant: dict[str, Any]) -> bool: @@ -1878,6 +1881,7 @@ async def _report_working_state(self, working: bool) -> bool: return await self.link.report_activity( self.room_id, working, + activity=self._working_reporter.activity, timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( self._observe_activity_stop, @@ -1908,6 +1912,7 @@ async def _execute_message_cycle( ) -> None: scope = self.current_scope if scope is not None: + scope.activity_allowed = True await self._acquire_activity(scope) if scope is not None and scope.terminating: raise asyncio.CancelledError @@ -1928,6 +1933,8 @@ async def _acquire_activity(self, scope: TurnScope) -> None: await self._start_activity(scope) async def _start_activity(self, scope: TurnScope) -> None: + if not scope.activity_allowed: + return previous = self._activity_owner if previous is not None and previous is not scope: if not previous.aborted and any( @@ -1943,11 +1950,57 @@ async def _start_activity(self, scope: TurnScope) -> None: self._activity_owner = scope await self._working_reporter.start() + def set_activity_tool_names(self, enabled: bool) -> None: + """Gate tool names in room activity to match tool-call narration. + + Permission waits and memory activity are unaffected: they name no + content. The gate is consulted at report time, so an observer bound + before this change (a continuation's adapter pumping tool events while + the runner re-reads the choice) still follows the live choice. + """ + self._activity_tool_names = enabled + + def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: + if scope is None or not scope.activity_allowed: + return None + return ToolNamesObserver( + ScopeActivityObserver(self, scope), + hidden=lambda: not self._activity_tool_names, + ) + + def owns_activity(self, scope: TurnScope) -> bool: + return self._activity_owner is scope and not scope.aborted and not self._closing + + async def observe_activity( + self, + scope: TurnScope, + *, + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + if not scope.activity_allowed: + return + if ( + self._activity_owner is None + and scope is self.current_scope + and not scope.settled + ): + await self._acquire_activity(scope) + await self._working_reporter.update( + current=partial(self.owns_activity, scope), + activity=activity, + waiting=waiting, + ) + def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" scope.bind(owner) owner.task.add_done_callback(lambda _: self._finish_activity(scope)) - if scope is self.current_scope and self._activity_owner is None: + if ( + scope.activity_allowed + and scope is self.current_scope + and self._activity_owner is None + ): previous = self._activity_completion async def acquire() -> None: @@ -2153,7 +2206,6 @@ async def _abort_cycle(self, kind: ControlMode, msg_id: str | None) -> bool: # poisoned into permanently_failed before it's ever actually attempted. if msg_id: self._retry_tracker.discard_attempt(msg_id) - await self._clear_activity() if kind is ControlMode.INTERRUPT and msg_id: # Consume the message so the idle /next resync does not re-return it # (excludes-only-processed) and re-fire the cycle the user just diff --git a/src/band/runtime/sessionconfig.py b/src/band/runtime/sessionconfig.py index 9c9b73171..05e5a98e6 100644 --- a/src/band/runtime/sessionconfig.py +++ b/src/band/runtime/sessionconfig.py @@ -43,14 +43,13 @@ class SessionConfig(BaseSettings): # Queue traffic cannot postpone a check; active handlers remain serial. idle_resync_seconds: PositiveFiniteSeconds = 60.0 - # --- Working-state (boolean "is the agent reasoning") reporting --- - # Kill-switch: disable all working-state reporting instantly if needed. + # Indicator and activity detail share this switch, including approval waits. enable_working_state: bool = True # Keep-alive cadence: how often working:true is refreshed while a cycle runs. # Must be < TTL/2 so a single missed/slow ping still lands inside the TTL. working_keep_alive_seconds: float = 3.0 - # Per-POST deadline for each activity report (must be < cadence so a slow - # POST can't pile onto the next keep-alive tick). + # Total deadline for one report's POST and its compatibility fallback. + # Must be < cadence so a slow report cannot pile onto the next tick. working_request_timeout_seconds: int = 2 # Optional upper bound on how long we keep asserting working:true for one # cycle. When exceeded we stop refreshing (platform TTL clears it); we never diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 88d53634b..56d340f23 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -73,7 +73,8 @@ serialize_tool_result, validate_tool_arguments, ) -from band.runtime.turn import records_turn_effects +from band.runtime.turn import records_turn_effects, reports_tool_activity +from band.runtime.working_state import ActivityObserver, model_tool_activity if TYPE_CHECKING: from anthropic.types import ToolParam @@ -285,6 +286,7 @@ def cache_parameters(self) -> Any: ... @records_turn_effects +@reports_tool_activity class AgentTools(AgentToolsProtocol): """ Room-bound tools for LLM platform interaction. @@ -344,6 +346,7 @@ def __init__( self._agent_id = agent_id self._ctx: ExecutionContext | None = None self._scope: TurnScope | None = None + self._activity_observer: ActivityObserver | None = None self.turn = Turn() @property @@ -384,6 +387,7 @@ def from_context( ) tools._ctx = ctx tools._scope = ctx.current_scope if bind_scope else None + tools._activity_observer = ctx.activity_observer(tools._scope) tools.turn = Turn(posts_missing_reply=ctx.config.report_turn_failures_to_room) return tools @@ -2039,7 +2043,8 @@ async def execute_tool_call_structured( try: method = getattr(self, definition.method_name) - result = await method(**arguments) + async with model_tool_activity(self, tool_name): + result = await method(**arguments) return ToolCallOutcome(value=serialize_tool_result(result), ok=True) except (BandToolError, RoomExecutionStoppedError): # Stopped execution must escape ordinary tool-error conversion. diff --git a/src/band/runtime/turn.py b/src/band/runtime/turn.py index c99551e48..8898eff86 100644 --- a/src/band/runtime/turn.py +++ b/src/band/runtime/turn.py @@ -9,7 +9,12 @@ from band.core.turn import Turn from band.runtime.tools.effects import turn_effect from band.runtime.tools.registry import TOOL_DEFINITIONS -from band.runtime.tools.types import Surface, TurnEffect +from band.runtime.tools.types import BandTool, Surface, TurnEffect +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, +) _ToolsClass = TypeVar("_ToolsClass", bound=type) @@ -48,3 +53,37 @@ def records_turn_effects(cls: _ToolsClass) -> _ToolsClass: if definition.surface == Surface.AGENT and method is not None: setattr(cls, definition.method_name, _recording(definition.name, method)) return cls + + +MEMORY_SAVE_TOOLS = frozenset( + {BandTool.STORE_MEMORY, BandTool.SUPERSEDE_MEMORY, BandTool.ARCHIVE_MEMORY} +) + + +def _observing( + tool_name: str, method: Callable[..., Awaitable[Any]] +) -> Callable[..., Awaitable[Any]]: + @functools.wraps(method) + async def observe(self: Any, *args: Any, **kwargs: Any) -> Any: + try: + if tool_name in MEMORY_SAVE_TOOLS: + await report_memory_activity(self) + else: + await report_tool_activity(self, tool_name) + return await method(self, *args, **kwargs) + finally: + await end_tool_activity(self) + + return observe + + +def reports_tool_activity(cls: _ToolsClass) -> _ToolsClass: + for definition in TOOL_DEFINITIONS.values(): + method = cls.__dict__.get(definition.method_name) + if ( + definition.surface == Surface.AGENT + and definition.name != BandTool.SEND_EVENT + and method is not None + ): + setattr(cls, definition.method_name, _observing(definition.name, method)) + return cls diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 4d842c7fe..07114594b 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -1,39 +1,100 @@ -"""Boolean working-state keep-alive reporter. - -Owns the per-room "is the agent working" signal for a single reasoning cycle: -sends ``working: true`` when a cycle starts, refreshes it on a keep-alive -cadence while the cycle is active, and sends an authoritative ``working: false`` -when the cycle ends. The platform applies a short TTL, so if refreshes stop for -any reason (crash, disconnect, loop-block) the indicator clears on its own — the -keep-alive is a *liveness* signal, not a correctness signal. - -The reporter is transport-agnostic: it is given an async ``report(working)`` -callback (wired to ``BandLink.report_activity`` by the runtime) and never raises -into the caller's reasoning loop. - -Contract: ``report`` MUST be time-bounded. The reporter awaits it (including the -authoritative false-send during teardown), so an unbounded callback could wedge -cycle teardown and cancellation propagation. ``BandLink.report_activity`` applies -a per-POST timeout to satisfy this. - -Cadence accuracy: the keep-alive sleeps ``keep_alive_seconds`` and *then* reports, -so the effective interval between successful refreshes is -``keep_alive_seconds + report-latency``. The platform-TTL headroom is preserved by -the ``cadence < TTL/2`` guard on ``SessionConfig`` (with the per-POST timeout kept -below the cadence), so even one missed/slow ping stays within the TTL window. +"""Serialized working-state and names-only activity for one provider turn. + +The boolean callback must be time bounded. It reads immutable detail under the +same lock as changes and heartbeats. Terminal false is shielded; platform TTL +backstops failed reports. A duration cap stops refreshes without cancelling work. """ from __future__ import annotations import asyncio import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from inspect import getattr_static +from typing import Protocol + +from band_rest.types.agent_activity_detail import AgentActivityDetail + +from band.runtime.tools.types import BandTool logger = logging.getLogger(__name__) ReportFn = Callable[[bool], Awaitable[object]] +UNKNOWN_ACTIVITY_TOOL_NAME = "unknown" +REASONING = AgentActivityDetail(kind="reasoning") +MEMORY = AgentActivityDetail(kind="memory") +PERMISSION = AgentActivityDetail(kind="permission") + + +class ActivityObserver(Protocol): + async def update(self, activity: AgentActivityDetail) -> None: ... + + async def set_manual_wait(self, waiting: bool) -> None: ... + + +def tool_call_activity(name: str | None) -> AgentActivityDetail: + """The platform requires a name on every ``tool_call`` detail.""" + return AgentActivityDetail( + kind="tool_call", name=name or UNKNOWN_ACTIVITY_TOOL_NAME + ) + + +def names_only(activity: AgentActivityDetail) -> AgentActivityDetail: + """Keep only what the room may see: the kind and, for a tool, its name.""" + if activity.kind == "tool_call": + return tool_call_activity(activity.name) + return AgentActivityDetail(kind=activity.kind) + + +def _observer(tools: object) -> ActivityObserver | None: + # Dynamic attribute proxies do not opt into runtime instrumentation. + if getattr_static(tools, "_activity_observer", None) is None: + return None + return getattr(tools, "_activity_observer", None) + + +async def report_tool_activity(tools: object, name: str) -> None: + if observer := _observer(tools): + await observer.update(tool_call_activity(name)) + + +async def end_tool_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(REASONING) + + +async def report_memory_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(MEMORY) + + +async def set_permission_wait(tools: object, waiting: bool) -> None: + if observer := _observer(tools): + await observer.set_manual_wait(waiting) + + +@asynccontextmanager +async def tool_activity(tools: object, name: str) -> AsyncIterator[None]: + try: + await report_tool_activity(tools, name) + yield + finally: + await end_tool_activity(tools) + + +@asynccontextmanager +async def model_tool_activity(tools: object, name: str) -> AsyncIterator[None]: + """Observe the dual-use event method only at its model invocation boundary.""" + if name == BandTool.SEND_EVENT: + async with tool_activity(tools, name): + yield + else: + yield + + def _retrieve_task_error(task: asyncio.Task[None]) -> None: if not task.cancelled(): task.exception() @@ -60,6 +121,8 @@ def __init__( self._enabled = enabled self._active = False + self._ordinary = REASONING + self._manual_wait = False self._started_at: float | None = None self._task: asyncio.Task[None] | None = None self._false_send: asyncio.Future[None] | None = None @@ -68,19 +131,49 @@ def __init__( # (boolean payload carries no sequence number to disambiguate). self._lock = asyncio.Lock() + @property + def activity(self) -> AgentActivityDetail: + """The immutable effective detail, read by the bool callback under its lock.""" + return PERMISSION if self._manual_wait else self._ordinary + async def start(self) -> None: - """Begin a reasoning cycle: send ``working: true`` and arm keep-alive.""" - if not self._enabled or self._active: - # Re-entrant start while active is a no-op: the signal is already - # alive and the keep-alive task is already running. + """Begin a cycle after the previous false; a re-entrant start is a no-op.""" + if not self._enabled: return await self._finish_false_send() - self._active = True - self._started_at = asyncio.get_running_loop().time() - await self._safe_report(True) - self._task = asyncio.create_task( - self._keep_alive(), name="working-state-keepalive" - ) + async with self._lock: + if self._active: + return + self._active = True + + self._reset_detail() + self._started_at = asyncio.get_running_loop().time() + await self._report_locked(True) + if self._active: + self._task = asyncio.create_task( + self._keep_alive(), name="working-state-keepalive" + ) + + async def update( + self, + *, + current: Callable[[], bool], + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + """Apply an observation; ``current`` rechecks ownership under the lock.""" + if not self._enabled: + return + async with self._lock: + if not current() or not self._active or self._cap_reached(): + return + previous = self.activity + if activity is not None: + self._ordinary = names_only(activity) + if waiting is not None: + self._manual_wait = waiting + if self.activity != previous: + await self._report_locked(True) def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. @@ -123,14 +216,24 @@ async def stop(self) -> None: task.exception() await self._finish_false_send() + def _reset_detail(self) -> None: + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False + def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False + + self._reset_detail() task, self._task = self._task, None if task is not None: task.cancel() self._false_send = asyncio.ensure_future(self._safe_report(False)) return task + def _reset_detail(self) -> None: + self._ordinary = REASONING + self._manual_wait = False + async def _finish_false_send(self) -> None: send = self._false_send if send is None: @@ -176,13 +279,17 @@ def _cap_reached(self) -> bool: return elapsed >= self._max_working_state_seconds async def _safe_report(self, working: bool) -> None: - """Report under the serialization lock; never raise.""" async with self._lock: - try: - await self._report(working) - except asyncio.CancelledError: - raise - except Exception: - logger.exception( - "working-state report failed (working=%s); swallowing", working - ) + if working and (not self._active or self._cap_reached()): + return + await self._report_locked(working) + + async def _report_locked(self, working: bool) -> None: + try: + await self._report(working) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "working-state report failed (working=%s); swallowing", working + ) diff --git a/src/band/testing/activity.py b/src/band/testing/activity.py new file mode 100644 index 000000000..3f8610a7b --- /dev/null +++ b/src/band/testing/activity.py @@ -0,0 +1,31 @@ +"""Record the optional producer observation seam without scheduling reports.""" + +from __future__ import annotations + +from typing import Any + +from band_rest.types.agent_activity_detail import AgentActivityDetail + + +class ActivityRecorder: + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + @property + def waiting(self) -> bool: + return next( + (event["waiting"] for event in reversed(self.events) if "waiting" in event), + False, + ) + + async def update(self, activity: AgentActivityDetail) -> None: + self.events.append(activity.model_dump(exclude_unset=True)) + + async def set_manual_wait(self, waiting: bool) -> None: + self.events.append({"waiting": waiting}) + + def assert_completed_tool(self, name: str) -> None: + assert self.events == [ + {"kind": "tool_call", "name": name}, + {"kind": "reasoning"}, + ] diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index 35248e458..ee498fd90 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -82,6 +82,7 @@ strip_handle_prefix, ) from band.runtime.turn import record_tool_result, records_turn_effects +from band.runtime.working_state import ActivityObserver # Synthetic identity FakeAgentTools uses for the "joins you to the task on # first status/active_form write" semantics band_update_task documents. @@ -326,6 +327,7 @@ def __init__( self._held_messages: list[HeldMessage] = [] self._observers: list[tuple[Callable[[], bool], asyncio.Future[None]]] = [] self.turn = Turn() + self._activity_observer: ActivityObserver | None = None @property def agent_id(self) -> str | None: diff --git a/tests/activitysupport.py b/tests/activitysupport.py new file mode 100644 index 000000000..eeba99419 --- /dev/null +++ b/tests/activitysupport.py @@ -0,0 +1,25 @@ +"""Isolate the process-wide activity-detail negotiation per test.""" + +from __future__ import annotations + +import pytest + +import band.platform.message_lifecycle as lifecycle +from band.platform.activity import ( + API_VERSION_HEADER, + MINIMUM_ACTIVITY_VERSION, + ActivitySupport, +) + +SUPPORTED_VERSION = ".".join(map(str, MINIMUM_ACTIVITY_VERSION)) + + +def install_activity_support( + monkeypatch: pytest.MonkeyPatch, *, learned: bool +) -> ActivitySupport: + """Give the lifecycle a fresh negotiation, optionally already supported.""" + support = ActivitySupport() + if learned: + support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) + monkeypatch.setattr(lifecycle, "activity_support", support) + return support diff --git a/tests/adapters/activity/__init__.py b/tests/adapters/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/activity/kit.py b/tests/adapters/activity/kit.py new file mode 100644 index 000000000..d9aa6bea0 --- /dev/null +++ b/tests/adapters/activity/kit.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator + +import httpx +from anthropic.types import Message, ToolUseBlock, Usage +from pydantic import BaseModel + +from tests.adapters.lettakit import make_tool_call_message, make_tool_return_message + + +class ActivityLookup(BaseModel): + """Look up an opaque test value.""" + + note: str + + +def anthropic_tool_response(name: str, note: str) -> Message: + return Message( + id="message", + role="assistant", + type="message", + model="test-model", + content=[ + ToolUseBlock(id="tool-id", name=name, type="tool_use", input={"note": note}) + ], + usage=Usage(input_tokens=1, output_tokens=1), + stop_reason="tool_use", + ) + + +class LettaActivityStream(httpx.AsyncByteStream): + def __init__(self) -> None: + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.closed = False + + async def __aiter__(self) -> AsyncIterator[bytes]: + call = make_tool_call_message("shell", '{"command":"private-argument"}') + yield f"data: {call.model_dump_json()}\n\n".encode() + self.entered.set() + await self.release.wait() + result = make_tool_return_message("shell", "private-result") + yield f"data: {result.model_dump_json()}\n\n".encode() + yield b'data: {"message_type":"usage_statistics","prompt_tokens":17,"completion_tokens":5,"cached_input_tokens":3,"cache_write_tokens":2}\n\n' + yield b"data: [DONE]\n\n" + + async def aclose(self) -> None: + self.closed = True diff --git a/tests/adapters/activity/test_acp.py b/tests/adapters/activity/test_acp.py new file mode 100644 index 000000000..a5e0d9aab --- /dev/null +++ b/tests/adapters/activity/test_acp.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from acp.schema import ToolCallProgress, ToolCallStart + +from band.integrations.acp.collecting import ACPCollectingClient +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder + + +class TestACPActivity: + async def test_activity_precedes_buffered_narration_and_ignores_private_titles( + self, + ) -> None: + client = ACPCollectingClient() + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + client.set_activity_tools("session", tools) + await client.session_update( + "session", + ToolCallStart( + sessionUpdate="tool_call", + toolCallId="call", + title="private command and arguments", + status="pending", + ), + ) + assert recorder.events == [{"kind": "tool_call", "name": "unknown"}] + await client.session_update( + "session", + ToolCallProgress( + sessionUpdate="tool_call_update", + toolCallId="call", + title="private-result", + status="in_progress", + rawOutput={"text": "private-result"}, + ), + ) + assert recorder.events == [{"kind": "tool_call", "name": "unknown"}] + await client.session_update( + "session", + ToolCallProgress( + sessionUpdate="tool_call_update", toolCallId="call", status="completed" + ), + ) + recorder.assert_completed_tool("unknown") + client.set_activity_tools("session", None) + await client.session_update( + "session", + ToolCallStart( + sessionUpdate="tool_call", toolCallId="late", title="private-late" + ), + ) + assert recorder.events[-1] == {"kind": "reasoning"} diff --git a/tests/adapters/activity/test_execution.py b/tests/adapters/activity/test_execution.py new file mode 100644 index 000000000..8e681e992 --- /dev/null +++ b/tests/adapters/activity/test_execution.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio + +import pytest +from google.genai import types + +from band.adapters.anthropic import AnthropicAdapter +from band.adapters.gemini import GeminiAdapter +from band.runtime.custom_tools import get_custom_tool_name +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools +from tests.adapters.activity.kit import ActivityLookup, anthropic_tool_response + + +class TestExecutedToolActivity: + @pytest.mark.parametrize("provider", ["anthropic", "gemini"]) + @pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) + async def test_real_custom_execution_reports_names_only_and_always_ends( + self, provider: str, outcome: str + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + name = get_custom_tool_name(ActivityLookup) + + async def lookup(args: ActivityLookup) -> str: + assert args.note == "private-argument" + entered.set() + await release.wait() + if outcome == "failure": + raise ValueError("private-result") + return "private-result" + + tool = (ActivityLookup, lookup) + match provider: + case "anthropic": + adapter = AnthropicAdapter(additional_tools=[tool], emit=()) + operation = adapter._process_tool_calls( + anthropic_tool_response(name, "private-argument"), tools + ) + case "gemini": + adapter = GeminiAdapter(additional_tools=[tool], emit=()) + operation = adapter._process_function_calls( + [ + types.FunctionCall( + id="call", name=name, args={"note": "private-argument"} + ) + ], + tools, + ) + task = asyncio.create_task(operation) + await entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": name}] + if outcome == "cancelled": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + await task + recorder.assert_completed_tool(name) + assert not tools.events_sent diff --git a/tests/adapters/activity/test_letta_stream.py b/tests/adapters/activity/test_letta_stream.py new file mode 100644 index 000000000..c77c73419 --- /dev/null +++ b/tests/adapters/activity/test_letta_stream.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import httpx +import pytest +from letta_client import AsyncLetta + +from band.adapters.letta import LettaAdapter, LettaAdapterConfig, LettaRun +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.activity.kit import LettaActivityStream + + +class TestLettaStreamActivity: + @pytest.mark.parametrize("cancelled", [False, True]) + async def test_real_sdk_observes_stream_before_response_buffering_and_closes( + self, cancelled: bool + ) -> None: + stream = LettaActivityStream() + requests: list[dict[str, Any]] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + requests.append(json.loads(request.content)) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=stream + ) + + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(respond)) as transport, + AsyncLetta( + api_key="test", base_url="https://letta.test", http_client=transport + ) as client, + ): + adapter = LettaAdapter(LettaAdapterConfig(agent_id="agent"), emit=()) + adapter._client = client + task = asyncio.create_task( + adapter._call_provider( + LettaRun(client, "agent", None), + [{"role": "user", "content": "hello"}], + tools, + ) + ) + try: + async with asyncio.timeout(5): + await stream.entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": "shell"}] + assert not task.done() + if cancelled: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + stream.release.set() + messages, usage = await task + assert [message.message_type for message in messages] == [ + "tool_call_message", + "tool_return_message", + ] + assert ( + usage.input_tokens, + usage.output_tokens, + usage.cache_read_tokens, + usage.cache_write_tokens, + ) == (17, 5, 3, 2) + assert recorder.events[-1] == {"kind": "reasoning"} + finally: + stream.release.set() + assert stream.closed + + assert requests[0]["streaming"] is True + assert requests[0]["stream_tokens"] is False + assert not tools.events_sent diff --git a/tests/adapters/activity/test_model_event.py b/tests/adapters/activity/test_model_event.py new file mode 100644 index 000000000..cc2db0cee --- /dev/null +++ b/tests/adapters/activity/test_model_event.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import pytest +from pydantic_ai import RunContext +from pydantic_ai.models.test import TestModel +from pydantic_ai.usage import RunUsage + +from band.integrations.mcp.engine import dispatch_tool +from band.integrations.pydantic_ai.tools import _dispatcher +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools + + +class TestModelEventActivity: + @pytest.mark.parametrize("surface", ["mcp", "pydantic"]) + async def test_model_event_reports_while_raw_telemetry_stays_quiet( + self, surface: str + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + await tools.send_event("internal", "thought") + assert not recorder.events + definition = TOOL_DEFINITIONS[BandTool.SEND_EVENT] + arguments = {"content": "model", "message_type": "thought"} + if surface == "mcp": + await dispatch_tool(tools, definition, arguments) + else: + dispatch = _dispatcher(definition, definition.input_model) + await dispatch( + RunContext(deps=tools, model=TestModel(), usage=RunUsage()), **arguments + ) + recorder.assert_completed_tool(BandTool.SEND_EVENT) diff --git a/tests/adapters/activity/test_native.py b/tests/adapters/activity/test_native.py new file mode 100644 index 000000000..12d3fb0dd --- /dev/null +++ b/tests/adapters/activity/test_native.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import pytest +from claude_agent_sdk import ToolResultBlock, ToolUseBlock +from google.adk.events import Event +from google.genai import types + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.adapters.google_adk import GoogleADKAdapter +from band.adapters.langgraph import LangGraphAdapter +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools + + +class TestNativeActivity: + @pytest.mark.parametrize("provider", ["claude", "langgraph", "adk"]) + async def test_start_and_end_are_independent_of_narration( + self, provider: str + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + match provider: + case "claude": + adapter = ClaudeSDKAdapter(emit=()) + pending: dict[str, str] = {} + await adapter._on_tool_use( + ToolUseBlock( + id="call", name="Bash", input={"command": "private-command"} + ), + pending, + "room", + tools, + ) + await adapter._on_tool_result( + ToolResultBlock(tool_use_id="call", content="private-result"), + pending, + "room", + tools, + ) + case "langgraph": + adapter = LangGraphAdapter(graph_factory=lambda tools: None, emit=()) + await adapter._handle_stream_event( + { + "event": "on_tool_start", + "name": "Bash", + "data": {"input": "private-command"}, + }, + "room", + tools, + ) + await adapter._handle_stream_event( + { + "event": "on_tool_error", + "name": "Bash", + "data": {"error": "private-result"}, + }, + "room", + tools, + ) + case "adk": + adapter = GoogleADKAdapter(emit=()) + await adapter._report_event( + Event( + author="test", + content=types.Content( + parts=[ + types.Part( + function_call=types.FunctionCall( + name="Bash", args={"command": "private-command"} + ) + ) + ] + ), + ), + tools, + ) + await adapter._report_event( + Event( + author="test", + content=types.Content( + parts=[ + types.Part( + function_response=types.FunctionResponse( + name="Bash", + response={"output": "private-result"}, + ) + ) + ] + ), + ), + tools, + ) + recorder.assert_completed_tool("Bash") + assert not tools.events_sent diff --git a/tests/adapters/activity/test_opencode_waits.py b/tests/adapters/activity/test_opencode_waits.py new file mode 100644 index 000000000..558ff150e --- /dev/null +++ b/tests/adapters/activity/test_opencode_waits.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import patch + +from band.adapters.opencode import OpencodeAdapterConfig +from band.adapters.opencode.approvals import ApprovalPorts, RoomApprovals +from band.runtime.working_state import end_tool_activity, report_tool_activity +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.opencode.helpers import AskFactory, FakeOpencodeClient + + +class TestOpenCodePermissionActivity: + async def test_last_wait_includes_claimed_reply_transport_and_latest_progress( + self, + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + client = FakeOpencodeClient() + entered, release = asyncio.Event(), asyncio.Event() + + async def abort() -> None: + await client.abort_session("session") + + approvals = RoomApprovals( + OpencodeAdapterConfig(), + ApprovalPorts( + room_id="room", + client=lambda: client, + tools=lambda: tools, + turn_mentions=list, + release_turn_wait=lambda: None, + fail_turn=lambda message: None, + abort_session=abort, + is_own_band_tool=lambda permission: False, + ), + ) + asks = AskFactory() + await approvals.on_permission_asked(asks.permission("first")) + await approvals.on_permission_asked(asks.permission("second")) + original = client.reply_permission + + async def held_reply( + permission_id: str, *, reply: str, message: str | None = None + ) -> None: + if permission_id == "first": + entered.set() + await release.wait() + await original(permission_id, reply=reply, message=message) + + with patch.object(client, "reply_permission", held_reply): + first = asyncio.create_task( + approvals.try_handle_reply("approve first", "user") + ) + try: + await entered.wait() + assert await approvals.try_handle_reply("approve second", "user") + await report_tool_activity(tools, "ordinary-tool") + await end_tool_activity(tools) + assert approvals.awaiting_human() + assert recorder.waiting + finally: + release.set() + await first + approvals.cancel() + assert not approvals.awaiting_human() + assert recorder.events[-1] == {"waiting": False} diff --git a/tests/adapters/activity/test_permission_delivery.py b/tests/adapters/activity/test_permission_delivery.py new file mode 100644 index 000000000..b87588c1a --- /dev/null +++ b/tests/adapters/activity/test_permission_delivery.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock + +from band.adapters.codex import CodexAdapter, CodexAdapterConfig, CodexCommand +from band.integrations.codex import CodexApprovalMethod +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.codex.kit import make_platform_message, wire_codex_room +from tests.adapters.codexturns import event_request +from tests.adapters.opencode.helpers import wait_for + + +class TestPermissionDelivery: + async def test_codex_retains_claimed_permission_until_provider_response_finishes( + self, + ) -> None: + adapter = CodexAdapter(CodexAdapterConfig(approval_mode="manual"), emit=()) + await adapter.on_started("Test", "Test") + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + entered, release = asyncio.Event(), asyncio.Event() + + async def respond(request_id: int, result: dict[str, Any]) -> None: + entered.set() + await release.wait() + + client = AsyncMock() + client.respond.side_effect = respond + wire_codex_room(adapter, client) + params = {"command": "private-command", "approvalId": "approval"} + pending = asyncio.create_task( + adapter._handle_approval_request( + client=client, + tools=tools, + msg=make_platform_message(), + room_id="room-1", + event=event_request(1, CodexApprovalMethod.COMMAND_EXECUTION, params), + params=params, + ) + ) + try: + await wait_for(lambda: bool(tools.messages_sent)) + await adapter._handle_approval_command( + tools=FakeAgentTools(), + msg=make_platform_message(), + room_id="room-1", + command=CodexCommand.APPROVE, + args="", + ) + await entered.wait() + assert recorder.waiting + assert adapter._pending_approvals["room-1"].has_claimed() + finally: + release.set() + await pending + assert not recorder.waiting + assert not adapter._pending_approvals["room-1"] diff --git a/tests/adapters/agno/adapter/__init__.py b/tests/adapters/agno/adapter/__init__.py index e69de29bb..9d48db4f9 100644 --- a/tests/adapters/agno/adapter/__init__.py +++ b/tests/adapters/agno/adapter/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/agno/adapter/test_activity.py b/tests/adapters/agno/adapter/test_activity.py new file mode 100644 index 000000000..dd1396649 --- /dev/null +++ b/tests/adapters/agno/adapter/test_activity.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import pytest +from agno.models.message import Message +from agno.run.agent import RunOutput + +from band.core.types import Emit +from band.testing.activity import ActivityRecorder +from tests.adapters.agno.helpers import tool_events, tool_execution + + +class TestAgnoStreamActivity: + @pytest.mark.parametrize("narrated", [False, True]) + async def test_streamed_tool_activity_preserves_final_response_with_or_without_narration( + self, make_started_adapter, sample_platform_message, tools, narrated: bool + ) -> None: + recorder = ActivityRecorder() + tools._activity_observer = recorder + response = RunOutput( + content="final reply", + messages=[Message(role="assistant", content="final reply")], + ) + adapter, agent = await make_started_adapter( + response, + emit=(Emit.TOOL_CALLS,) if narrated else (), + events=tool_events( + tool_execution( + "shell", + args={"command": "private-argument"}, + result="private-result", + ) + ), + ) + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + recorder.assert_completed_tool("shell") + assert not tools.messages_sent + assert adapter._message_history["room-1"][-1].content == "final reply" + assert bool(tools.events_sent) is narrated + assert agent.arun.call_args.kwargs["stream"] is True diff --git a/tests/adapters/agno/adapter/test_emit_execution.py b/tests/adapters/agno/adapter/test_emit_execution.py index 20bcac251..4874ad68e 100644 --- a/tests/adapters/agno/adapter/test_emit_execution.py +++ b/tests/adapters/agno/adapter/test_emit_execution.py @@ -113,5 +113,4 @@ async def test_no_events_without_execution_emit( ) assert tools.events_sent == [] - # Confirms the non-streaming path: a single awaited arun, not a stream. - agent.arun.assert_awaited_once() + agent.arun.assert_called_once() diff --git a/tests/adapters/agno/adapter/test_failed_run_does_not_contaminate_next_turn.py b/tests/adapters/agno/adapter/test_failed_run_does_not_contaminate_next_turn.py index 71b05969c..c04393f08 100644 --- a/tests/adapters/agno/adapter/test_failed_run_does_not_contaminate_next_turn.py +++ b/tests/adapters/agno/adapter/test_failed_run_does_not_contaminate_next_turn.py @@ -1,9 +1,8 @@ from __future__ import annotations -from unittest.mock import AsyncMock +from unittest.mock import MagicMock import pytest -from agno.run.agent import RunOutput from band.adapters.agno import ( AgnoAdapter, @@ -21,9 +20,8 @@ async def test_failed_turn_leaves_no_residue_in_next_run_input( # Turn 1 raises mid-run; turn 2 succeeds. The injected system/user # messages from the failed turn must not survive into turn 2's input. agent = make_agno_agent() - agent.arun = AsyncMock( - side_effect=[RuntimeError("boom"), RunOutput(content="ok")] - ) + succeeding = agent.arun.side_effect + agent.arun = MagicMock(side_effect=[RuntimeError("boom"), succeeding()]) adapter = AgnoAdapter(agent=agent, emit=()) await adapter.on_started("TestBot", "desc") diff --git a/tests/adapters/agno/adapter/test_on_started.py b/tests/adapters/agno/adapter/test_on_started.py index 39c3f3b2b..1e8cd7cc4 100644 --- a/tests/adapters/agno/adapter/test_on_started.py +++ b/tests/adapters/agno/adapter/test_on_started.py @@ -41,5 +41,5 @@ async def test_runs_the_given_agent_on_a_message(self, make_agno_agent, tools): room_id="room-1", ) - agent.arun.assert_awaited_once() + agent.arun.assert_called_once() tools.assert_no_messages_sent() diff --git a/tests/adapters/agno/adapter/test_session_isolation.py b/tests/adapters/agno/adapter/test_session_isolation.py index 2a2613d84..59f5a850a 100644 --- a/tests/adapters/agno/adapter/test_session_isolation.py +++ b/tests/adapters/agno/adapter/test_session_isolation.py @@ -21,7 +21,7 @@ async def test_arun_uses_room_id_as_session_id(self, make_started_adapter): room_id="room-A", ) - assert agent.arun.await_args.kwargs["session_id"] == "room-A" + assert agent.arun.call_args.kwargs["session_id"] == "room-A" async def test_custom_session_id_factory_is_used(self, make_agno_agent): agent = make_agno_agent() @@ -40,7 +40,7 @@ async def test_custom_session_id_factory_is_used(self, make_agno_agent): room_id="room-A", ) - assert agent.arun.await_args.kwargs["session_id"] == "sess::room-A" + assert agent.arun.call_args.kwargs["session_id"] == "sess::room-A" async def test_two_rooms_get_isolated_sessions_and_inputs( self, make_started_adapter @@ -66,7 +66,7 @@ async def test_two_rooms_get_isolated_sessions_and_inputs( room_id="room-B", ) - calls = agent.arun.await_args_list + calls = agent.arun.call_args_list assert calls[0].kwargs["session_id"] == "room-A" assert calls[1].kwargs["session_id"] == "room-B" diff --git a/tests/adapters/agno/helpers.py b/tests/adapters/agno/helpers.py index 074600451..9025e6685 100644 --- a/tests/adapters/agno/helpers.py +++ b/tests/adapters/agno/helpers.py @@ -12,7 +12,7 @@ from collections.abc import Awaitable, Callable from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock from agno.models.base import Model from agno.models.message import Message @@ -61,30 +61,17 @@ def fake_agno_agent( agent.tools = [] resp = response if response is not None else RunOutput() - if events is None: - # Non-streaming path (Emit.TOOL_CALLS off): `await agent.arun(...)`. - async def _arun_once(*args: Any, **kwargs: Any) -> RunOutput: + def arun(*args: Any, **kwargs: Any) -> Any: + async def stream() -> Any: if model_turn is not None: await model_turn() - return resp - - agent.arun = AsyncMock(side_effect=_arun_once) - else: - # Streaming path (Emit.TOOL_CALLS on): the adapter iterates - # `agent.arun(stream=True, ...)`, which yields the run events then the - # final RunOutput. A bare MagicMock returns an async iterator without - # awaiting, matching how the adapter consumes the stream. - def _arun(*args: Any, **kwargs: Any) -> Any: - async def _stream() -> Any: - if model_turn is not None: - await model_turn() - for event in events: - yield event - yield resp - - return _stream() - - agent.arun = MagicMock(side_effect=_arun) + for event in events or []: + yield event + yield resp + + return stream() + + agent.arun = MagicMock(side_effect=arun) return agent @@ -156,6 +143,9 @@ async def aresponse(self, messages: list[Message], **kwargs: Any) -> ModelRespon await send.entrypoint(content=self._content, mentions=["@Alice"]) return ModelResponse(content=self._content) + async def aresponse_stream(self, messages: list[Message], **kwargs: Any) -> Any: + yield await self.aresponse(messages, **kwargs) + @property def captured_system_prompt(self) -> str: """The concatenated system message(s) Agno sent to the model.""" @@ -256,7 +246,7 @@ def make_agent_input( def run_input(copy: MagicMock) -> list[Message]: """The exact list[Message] the faked Agno agent received via arun(input=...).""" - return copy.arun.await_args.kwargs["input"] + return copy.arun.call_args.kwargs["input"] def platform_msg( diff --git a/tests/adapters/agno/rehydration/__init__.py b/tests/adapters/agno/rehydration/__init__.py index e69de29bb..9d48db4f9 100644 --- a/tests/adapters/agno/rehydration/__init__.py +++ b/tests/adapters/agno/rehydration/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/agno/rehydration/test_unanswered_message.py b/tests/adapters/agno/rehydration/test_unanswered_message.py index 4ab01c3ce..1072ca746 100644 --- a/tests/adapters/agno/rehydration/test_unanswered_message.py +++ b/tests/adapters/agno/rehydration/test_unanswered_message.py @@ -60,7 +60,7 @@ async def test_answers_unanswered_message_on_restart_bootstrap( ) ) - agent.arun.assert_awaited_once() + agent.arun.assert_called_once() assert run_input(agent)[-1].content == sample_platform_message.format_for_llm() async def test_trailing_unanswered_user_turns_are_preserved( diff --git a/tests/adapters/claude_sdk/activity/__init__.py b/tests/adapters/claude_sdk/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/claude_sdk/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/claude_sdk/activity/test_permission.py b/tests/adapters/claude_sdk/activity/test_permission.py new file mode 100644 index 000000000..9fbcae179 --- /dev/null +++ b/tests/adapters/claude_sdk/activity/test_permission.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from band.runtime.working_state import report_tool_activity +from band.testing.activity import ActivityRecorder +from tests.adapters.claude_sdk.helpers import OpenRoom, with_approvals +from tests.adapters.opencode.helpers import wait_for +from tests.baseline.decisions import ModelDecision, ToolCall + + +class TestClaudePermissionActivity: + async def test_overlapping_native_waits_include_claimed_notice_delivery( + self, claude_room: OpenRoom + ) -> None: + room = await claude_room(with_approvals(), emit=()) + recorder = ActivityRecorder() + room.tools._activity_observer = recorder + room.claude.script( + [ + ModelDecision( + tool_calls=( + ToolCall("Bash", {"command": "private-command"}), + ToolCall( + "Write", + {"file_path": "private-path", "content": "private-content"}, + ), + ) + ), + room.model_reply("Done"), + ] + ) + held = room.tools.hold_message("resolved") + await room.send("Do both") + await room.until_said("Approval requested", times=2) + await room.send("/approve a-1") + async with held: + assert recorder.waiting + await report_tool_activity(room.tools, "ordinary-tool") + await room.send("/approve a-2") + await wait_for( + lambda: len(room.adapter._pending_approvals[room.room_id]) == 1 + ) + assert recorder.waiting + await room.settled() + assert not recorder.waiting + assert not room.tool_call_names diff --git a/tests/adapters/claude_sdk/approvals/test_manual_approval.py b/tests/adapters/claude_sdk/approvals/test_manual_approval.py index 44ddfe6e0..d5afafecb 100644 --- a/tests/adapters/claude_sdk/approvals/test_manual_approval.py +++ b/tests/adapters/claude_sdk/approvals/test_manual_approval.py @@ -240,7 +240,7 @@ async def test_an_unanswered_ask_times_out_to_the_configured_decision( self, claude_room: OpenRoom, decision: str, bash_output: str ) -> None: """A reply landing while the timeout notice is still being sent is told - the token is gone, not "resolved".""" + the token is no longer pending, not "resolved".""" room = await claude_room( with_approvals(wait_timeout_s=60, timeout_decision=decision) ) @@ -254,7 +254,7 @@ async def test_an_unanswered_ask_times_out_to_the_configured_decision( assert room.chat == [ prompt("a-1", "Bash: `ls`"), - "Unknown approval token `a-1`. Available: none.", + "Approval `a-1` is no longer pending.", APPROVAL_TIMED_OUT_TEMPLATE.format(token="a-1", decision=decision), "Done waiting.", ] diff --git a/tests/adapters/codex/conftest.py b/tests/adapters/codex/conftest.py index bb9c8e8ee..4b5001f84 100644 --- a/tests/adapters/codex/conftest.py +++ b/tests/adapters/codex/conftest.py @@ -4,6 +4,7 @@ import asyncio from collections.abc import AsyncIterator, Awaitable, Callable +from pathlib import Path from typing import Any import looptime @@ -28,6 +29,7 @@ patch_codex_clients_by_room, send_bootstrap, ) +from tests.adapters.codex.probes import ProbeHost from tests.adapters.codexturns import ( FakeCodexClient, make_codex_adapter, @@ -129,3 +131,14 @@ async def start( yield start for adapter in adapters: await adapter.cleanup_all() + + +@pytest_asyncio.fixture(loop_scope="function") +async def probe_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[ProbeHost]: + host = ProbeHost(tmp_path, monkeypatch) + try: + yield host + finally: + await host.close() diff --git a/tests/adapters/codex/kit.py b/tests/adapters/codex/kit.py index d6b19736f..49f701de7 100644 --- a/tests/adapters/codex/kit.py +++ b/tests/adapters/codex/kit.py @@ -446,20 +446,21 @@ def ask( self, approval_id: str, *, request_id: int ) -> asyncio.Task[ApprovalDecision]: params = {"approvalId": approval_id, "command": "npm test"} - self.asks.append( - ask := asyncio.create_task( - self.adapter._resolve_manual_approval( - tools=self.tools, - msg=make_platform_message(room_id=ROOM_ID), - room_id=ROOM_ID, - event=event_request( - request_id, CodexApprovalMethod.COMMAND_EXECUTION, params - ), - summary="npm test", - params=params, - ) - ) - ) + + async def resolve() -> ApprovalDecision: + async with self.adapter._manual_approval( + tools=self.tools, + msg=make_platform_message(room_id=ROOM_ID), + room_id=ROOM_ID, + event=event_request( + request_id, CodexApprovalMethod.COMMAND_EXECUTION, params + ), + summary="npm test", + params=params, + ) as decision: + return decision + + self.asks.append(ask := asyncio.create_task(resolve())) return ask @asynccontextmanager diff --git a/tests/adapters/codex/probepeer.py b/tests/adapters/codex/probepeer.py new file mode 100644 index 000000000..18fd20790 --- /dev/null +++ b/tests/adapters/codex/probepeer.py @@ -0,0 +1,69 @@ +"""A real stdio app-server boundary for host probes without provider turns.""" + +from __future__ import annotations + +import json +import os +import sys +import threading +from pathlib import Path +from typing import Any + + +def emit(message: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def main() -> None: + record = Path(sys.argv[1]) + settings = json.loads(sys.argv[2]) + if diagnostic := settings.get("diagnostic"): + sys.stdout.write( + diagnostic + + "\n" + + json.dumps([diagnostic]) + + "\n" + + json.dumps({"diagnostic": diagnostic}) + + "\n" + ) + sys.stdout.flush() + sys.stderr.write(diagnostic + "\n") + sys.stderr.flush() + if settings.get("startup_hang"): + threading.Event().wait() + for line in sys.stdin: + message = json.loads(line) + with record.open("a", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + **message, + "pid": os.getpid(), + "marker": os.environ["CODEX_PROBE_MARKER"], + } + ) + + "\n" + ) + if "id" not in message: + continue + method = message["method"] + if method == "model/list": + cursor = message["params"].get("cursor") + pages = settings.get("pages", {"": {"data": [], "nextCursor": None}}) + reply = pages.get(cursor or "", {}) + if settings.get("endless_pages"): + page = int(cursor or "0") + reply = {"data": [], "nextCursor": str(page + 1)} + else: + reply = settings.get("replies", {}).get(method, {}) + if reply.get("hang"): + threading.Event().wait() + elif "error" in reply: + emit({"id": message["id"], "error": reply["error"]}) + else: + emit({"id": message["id"], "result": reply}) + + +if __name__ == "__main__": + main() diff --git a/tests/adapters/codex/probes.py b/tests/adapters/codex/probes.py new file mode 100644 index 000000000..45cfbbfb2 --- /dev/null +++ b/tests/adapters/codex/probes.py @@ -0,0 +1,121 @@ +"""Real Codex clients with controllable cleanup and a recorded wire boundary.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import pytest + +from band.adapters import codex +from band.adapters.codex import CodexAdapter, CodexAdapterConfig +from band.integrations.codex.stdio_client import CodexStdioClient +from tests.adapters.waiting import until + +PROBE_MARKER = "configured-host-environment" +CLOSE_FAILURE = "probe process cleanup refused" + + +class ProbeClient(CodexStdioClient): + refuse_close = False + + async def _close_process(self) -> None: + if self.refuse_close: + raise RuntimeError(CLOSE_FAILURE) + await super()._close_process() + + +class ProbeHost: + def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self.record = root / "probe.jsonl" + self.clients: list[ProbeClient] = [] + self.adapters: list[CodexAdapter] = [] + self.root = root + + def build( + *, + command: Sequence[str] | None = None, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + **transport: Any, + ) -> ProbeClient: + client = ProbeClient( + command=command, + cwd=cwd, + env=env, + **transport, + ) + self.clients.append(client) + return client + + monkeypatch.setattr(codex, "CodexStdioClient", build) + + def adapter( + self, + *, + replies: dict[str, Any] | None = None, + pages: dict[str, Any] | None = None, + endless_pages: bool = False, + diagnostic: str | None = None, + startup_hang: bool = False, + **config: Any, + ) -> CodexAdapter: + settings = { + "replies": replies or {}, + "endless_pages": endless_pages, + "diagnostic": diagnostic, + "startup_hang": startup_hang, + } + if pages is not None: + settings["pages"] = pages + adapter = CodexAdapter( + CodexAdapterConfig( + codex_command=( + sys.executable, + "-u", + str(Path(__file__).with_name("probepeer.py")), + str(self.record), + json.dumps(settings), + ), + codex_env={"CODEX_PROBE_MARKER": PROBE_MARKER}, + workspace_for_room=lambda room: str(self.root / room), + **config, + ) + ) + self.adapters.append(adapter) + return adapter + + @property + def messages(self) -> list[dict[str, Any]]: + if not self.record.exists(): + return [] + return [json.loads(line) for line in self.record.read_text().splitlines()] + + @property + def methods(self) -> list[str]: + return [message["method"] for message in self.messages] + + async def received(self, method: str) -> None: + await until( + lambda: method in self.methods, + failure=f"The app-server never received {method}", + ) + + def assert_exited(self) -> None: + assert self.clients, "The configured app-server never started" + for client in self.clients: + assert client._proc is not None + assert client._proc.returncode is not None + assert client._windows_process is None + assert not client._connected + + async def close(self) -> None: + for client in self.clients: + client.refuse_close = False + for adapter in self.adapters: + await adapter.cleanup_all() + for client in self.clients: + await client.close() diff --git a/tests/adapters/codex/test_callback_shutdown.py b/tests/adapters/codex/test_callback_shutdown.py new file mode 100644 index 000000000..258e8c336 --- /dev/null +++ b/tests/adapters/codex/test_callback_shutdown.py @@ -0,0 +1,73 @@ +"""A custom tool can request shutdown without being cancelled by its own stop.""" + +from __future__ import annotations + +import asyncio + +from pydantic import BaseModel + +from band.adapters.codex import CodexAdapterConfig +from band.core.exceptions import CleanupIncompleteError +from band.core.retention import RoomRetentionConfig +from tests.adapters.codex.kit import ToolSchemaFakeTools, send_bootstrap +from tests.adapters.codexturns import ( + FakeCodexClient, + event_request, + make_codex_adapter, + turn_completed, +) + + +class TestCallbackShutdown: + async def test_enrolled_tool_gets_pending_cleanup_and_unwinds_without_cancellation( + self, + ) -> None: + class ShutdownInput(BaseModel): + """Request orderly host shutdown.""" + + observed: list[str] = [] + + async def shutdown(_input: ShutdownInput) -> str: + try: + await adapter.cleanup_all() + except CleanupIncompleteError: + observed.append("pending") + except asyncio.CancelledError: + observed.append("cancelled") + raise + return "shutdown requested" + + client = FakeCodexClient( + events=[ + event_request( + 99, + "item/tool/call", + {"tool": "shutdown", "arguments": {}, "callId": "shutdown-99"}, + ), + turn_completed(), + ] + ) + adapter = make_codex_adapter( + client, + config=CodexAdapterConfig( + turn_settle_timeout_s=0.05, + retention=RoomRetentionConfig(transition_timeout_s=1), + ), + additional_tools=[(ShutdownInput, shutdown)], + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Codex Agent", "A coding agent") + try: + results = await asyncio.gather( + send_bootstrap(adapter, tools=tools), return_exceptions=True + ) + assert observed == ["pending"] + assert results == [None] + assert client.responses[0][1]["success"] is True + assert ( + client.responses[0][1]["contentItems"][0]["text"] + == "shutdown requested" + ) + finally: + await adapter.cleanup_all() + assert client.closed diff --git a/tests/adapters/codex/test_model_pagination.py b/tests/adapters/codex/test_model_pagination.py new file mode 100644 index 000000000..efbed3225 --- /dev/null +++ b/tests/adapters/codex/test_model_pagination.py @@ -0,0 +1,88 @@ +"""Room startup, model listing and effort lookup share complete pagination.""" + +from __future__ import annotations + +import pytest + +from band.core.defaultmodels import OPENAI_MODEL +from tests.adapters.codex.kit import ROOM_ID, ToolSchemaFakeTools, send_bootstrap +from tests.adapters.codex.probes import ProbeHost + + +class TestModelPagination: + async def test_startup_and_room_listing_find_visible_later_page_models( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + pages={ + "": { + "data": [{"id": "hidden-first", "hidden": True}], + "nextCursor": "next", + }, + "next": {"data": [{"id": "visible-second"}], "nextCursor": None}, + } + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/model list", tools=tools) + + assert adapter._room_clients[ROOM_ID].selected_model == "visible-second" + assert "visible-second" in tools.messages_sent[0]["content"] + assert "hidden-first" not in tools.messages_sent[0]["content"] + assert "thread/start" not in probe_host.methods + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_configured_model_efforts_on_later_page_validate_room_command( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-model", + pages={ + "": {"data": [{"id": "other-model"}], "nextCursor": "next"}, + "next": { + "data": [ + { + "id": "configured-model", + "supportedReasoningEfforts": [ + {"reasoningEffort": "low"}, + {"reasoningEffort": "ultra"}, + ], + } + ], + "nextCursor": None, + }, + }, + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/reasoning typo", tools=tools) + + assert adapter._room_clients[ROOM_ID].reasoning_effort is None + assert adapter.config.model == "configured-model" + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + @pytest.mark.parametrize("endless", [False, True], ids=["repeated", "page-ceiling"]) + async def test_bad_pagination_retains_the_existing_startup_default_policy( + self, probe_host: ProbeHost, endless: bool + ) -> None: + adapter = probe_host.adapter( + pages={ + "": {"data": [{"id": "incomplete-catalog"}], "nextCursor": "same"}, + "same": {"data": [], "nextCursor": "same"}, + }, + endless_pages=endless, + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/model", tools=tools) + + assert adapter._room_clients[ROOM_ID].selected_model == OPENAI_MODEL + await adapter.cleanup_all() + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_model_probe.py b/tests/adapters/codex/test_model_probe.py new file mode 100644 index 000000000..0d6808607 --- /dev/null +++ b/tests/adapters/codex/test_model_probe.py @@ -0,0 +1,175 @@ +"""Model probes return the visible paginated catalog without provider turns.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.model_catalog import ModelCatalog +from band.integrations.codex.rpc_base import CodexJsonRpcError +from tests.adapters.codex.kit import ROOM_ID, send_bootstrap +from tests.adapters.codex.probes import ProbeHost + + +class TestModelProbe: + async def test_visible_paginated_metadata_uses_the_canonical_catalog( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-room-model", + pages={ + "": { + "data": [{"id": "internal-preview", "hidden": True}], + "nextCursor": "page-two", + }, + "page-two": { + "data": [ + { + "id": "gpt-6-sol", + "model": "gpt-6-sol", + "displayName": "GPT-6 Sol", + "hidden": False, + "isDefault": True, + "defaultReasoningEffort": "medium", + "supportedReasoningEfforts": [ + { + "reasoningEffort": "medium", + "description": "Balanced", + }, + {"reasoningEffort": "max", "description": "Thorough"}, + ], + }, + {"id": "no-effort", "supportedReasoningEfforts": []}, + {"id": "unknown-efforts"}, + {"displayName": "Not a selectable model"}, + ], + "nextCursor": None, + }, + }, + ) + + catalog = await adapter.probe_models(timeout_s=5) + + assert isinstance(catalog, ModelCatalog) + assert catalog.model_ids == ("gpt-6-sol", "no-effort", "unknown-efforts") + assert catalog.current_model is None + model = catalog.choice("gpt-6-sol") + assert model is not None + assert model.label == "GPT-6 Sol" + assert model.provider is None + assert model.is_default + assert model.efforts == ("medium", "max") + assert model.default_effort == "medium" + assert catalog.choice("no-effort").efforts == () + assert catalog.choice("unknown-efforts").efforts is None + assert [ + message["params"] + for message in probe_host.messages + if message["method"] == "model/list" + ] == [{}, {"cursor": "page-two"}] + assert probe_host.methods == [ + "initialize", + "initialized", + "model/list", + "model/list", + ] + assert adapter._room_clients == {} + assert adapter._workspaces.rooms == () + assert adapter.config.model == "configured-room-model" + probe_host.assert_exited() + + async def test_empty_and_unsupported_are_not_conflated( + self, probe_host: ProbeHost + ) -> None: + empty = probe_host.adapter() + unsupported = probe_host.adapter( + pages={"": {"error": {"code": -32601, "message": "model/list unsupported"}}} + ) + + assert (await empty.probe_models(timeout_s=5)).models == () + with pytest.raises(CodexJsonRpcError, match="unsupported"): + await unsupported.probe_models(timeout_s=5) + probe_host.assert_exited() + + async def test_cancellation_during_model_listing_reaps_the_probe( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(pages={"": {"hang": True}}) + probing = asyncio.create_task(adapter.probe_models(timeout_s=5)) + try: + await probe_host.received("model/list") + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + + async def test_second_probe_does_not_reuse_a_closed_transport( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter() + assert (await adapter.probe_models(timeout_s=5)).models == () + assert (await adapter.probe_models(timeout_s=5)).models == () + assert len(probe_host.clients) == 2 + probe_host.assert_exited() + + async def test_probe_does_not_change_or_close_an_existing_rooms_selection( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-model", + pages={"": {"data": [{"id": "catalog-default", "isDefault": True}]}}, + ) + await adapter.on_started("Agent", "A coding agent") + await send_bootstrap(adapter, content="/model room-override") + [room_client] = probe_host.clients + + catalog = await adapter.probe_models(timeout_s=5) + + assert catalog.model_ids == ("catalog-default",) + assert catalog.current_model is None + assert adapter.config.model == "configured-model" + assert adapter._room_clients[ROOM_ID].selected_model == "room-override" + assert adapter._workspaces.rooms == (ROOM_ID,) + assert room_client._proc is not None + assert room_client._proc.returncode is None + probe_client = probe_host.clients[-1] + assert probe_client is not room_client + assert probe_client._proc is not None + assert probe_client._proc.returncode is not None + assert "thread/start" not in probe_host.methods + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + @pytest.mark.parametrize("method", ["initialize", "model/list"]) + async def test_probe_rpc_timeout_propagates_after_reaping( + self, probe_host: ProbeHost, method: str + ) -> None: + adapter = probe_host.adapter( + replies={method: {"hang": True}}, pages={"": {"hang": True}} + ) + with pytest.raises(TimeoutError): + await adapter.probe_models(timeout_s=1) + probe_host.assert_exited() + + @pytest.mark.parametrize("endless", [False, True], ids=["repeated", "page-ceiling"]) + async def test_broken_pagination_is_bounded_and_closes_the_probe( + self, probe_host: ProbeHost, endless: bool + ) -> None: + adapter = probe_host.adapter( + pages={ + "": {"data": [], "nextCursor": "same"}, + "same": {"data": [], "nextCursor": "same"}, + }, + endless_pages=endless, + ) + expected_error = "more than 50 pages" if endless else "repeated cursor" + with pytest.raises(RuntimeError, match=expected_error): + await adapter.probe_models(timeout_s=5) + assert probe_host.methods.count("model/list") == (50 if endless else 2) + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_model_provider.py b/tests/adapters/codex/test_model_provider.py new file mode 100644 index 000000000..8fd2eecda --- /dev/null +++ b/tests/adapters/codex/test_model_provider.py @@ -0,0 +1,36 @@ +"""Native Codex catalogs do not establish the configured provider's identity.""" + +from __future__ import annotations + +from tests.adapters.codex.probes import ProbeHost + + +class TestModelProvider: + async def test_unreported_provider_is_unknown_even_for_a_configured_model( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="private-provider-model", + pages={ + "": { + "data": [ + { + "id": "private-provider-model", + "model": "private-provider-model", + "displayName": "Private model", + "hidden": False, + "isDefault": True, + "supportedReasoningEfforts": [], + } + ], + "nextCursor": None, + } + }, + ) + + catalog = await adapter.probe_models(timeout_s=5) + + assert catalog.model_ids == ("private-provider-model",) + assert catalog.models[0].provider is None + assert adapter.config.model == "private-provider-model" + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_preflight.py b/tests/adapters/codex/test_preflight.py new file mode 100644 index 000000000..45be4ad1e --- /dev/null +++ b/tests/adapters/codex/test_preflight.py @@ -0,0 +1,168 @@ +"""Host preflight performs configured RPCs without claiming a room or turn.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from band.adapters.codex import CodexAdapter, CodexAdapterConfig +from band.core.exceptions import CleanupIncompleteError +from band.integrations.codex.stdio_client import CodexStdioClient +from tests.adapters.codex.probes import CLOSE_FAILURE, PROBE_MARKER, ProbeHost + + +class TestPreflight: + async def test_configured_handshake_checks_login_without_room_or_turn( + self, probe_host: ProbeHost, tmp_path: Path + ) -> None: + skill_root = str(tmp_path / "skills") + adapter = probe_host.adapter( + replies={"account/read": {"account": {"type": "chatgpt"}}}, + client_name="host_name", + client_title="Host title", + client_version="2.3.4", + experimental_api=False, + skill_roots=[skill_root], + ) + + result = await adapter.preflight(timeout_s=5) + + assert result.ok + assert probe_host.methods == [ + "initialize", + "initialized", + "skills/extraRoots/set", + "account/read", + ] + initialize = probe_host.messages[0] + assert initialize["params"] == { + "clientInfo": { + "name": "host_name", + "title": "Host title", + "version": "2.3.4", + }, + "capabilities": {"experimentalApi": False}, + } + assert initialize["marker"] == PROBE_MARKER + assert probe_host.messages[2]["params"] == {"extraRoots": [skill_root]} + assert adapter._room_clients == {} + assert adapter._workspaces.rooms == () + probe_host.assert_exited() + + @pytest.mark.parametrize( + ("account", "ok"), + [ + ({"account": None, "requiresOpenaiAuth": True}, False), + ({"account": None, "requiresOpenaiAuth": False}, True), + ({"account": {"type": "apiKey"}, "requiresOpenaiAuth": True}, True), + ({"error": {"code": -32601, "message": "unsupported"}}, True), + ({"error": {"code": -32000, "message": "refused"}}, True), + ], + ids=[ + "missing-required-login", + "custom-provider", + "api-key", + "unsupported", + "refused", + ], + ) + async def test_only_positively_missing_required_auth_fails( + self, probe_host: ProbeHost, account: dict, ok: bool + ) -> None: + adapter = probe_host.adapter(replies={"account/read": account}) + + result = await adapter.preflight(timeout_s=5) + + assert result.ok is ok + if not ok: + assert "not logged in" in result.reason + assert "codex login" in result.remedy + probe_host.assert_exited() + + async def test_missing_executable_names_install_remedy( + self, tmp_path: Path + ) -> None: + adapter = CodexAdapter( + CodexAdapterConfig(codex_command=(str(tmp_path / "absent-codex"),)) + ) + + result = await adapter.preflight(timeout_s=5) + + assert not result.ok + assert "binary not found" in result.reason + assert "codex_command" in result.remedy + await adapter.cleanup_all() + + @pytest.mark.parametrize("method", ["initialize", "account/read"]) + async def test_hung_rpc_has_a_budget_and_reaps_the_owned_process( + self, probe_host: ProbeHost, method: str + ) -> None: + adapter = probe_host.adapter(replies={method: {"hang": True}}) + + result = await adapter.preflight(timeout_s=1) + + assert not result.ok + assert result.reason + probe_host.assert_exited() + + async def test_initialize_error_is_a_failed_result_after_cleanup( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + replies={"initialize": {"error": {"code": -32000, "message": "bad frame"}}} + ) + + result = await adapter.preflight(timeout_s=5) + + assert not result.ok + assert "handshake" in result.reason + assert result.remedy + probe_host.assert_exited() + + async def test_cancellation_propagates_and_reaps_the_process( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await probe_host.received("account/read") + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + + async def test_cleanup_failure_remains_owned_for_adapter_retry( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + + async def refuse(_client: CodexStdioClient) -> None: + raise RuntimeError(CLOSE_FAILURE) + + with monkeypatch.context() as patch: + patch.setattr(CodexStdioClient, "_close_process", refuse) + with pytest.raises(CleanupIncompleteError): + await adapter.preflight(timeout_s=5) + [client] = probe_host.clients + assert client._proc is not None and client._proc.returncode is None + + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_second_preflight_uses_a_fresh_process( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter() + + first = await adapter.preflight(timeout_s=5) + second = await adapter.preflight(timeout_s=5) + + assert first.ok and second.ok + assert len(probe_host.clients) == 2 + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_probe_cleanup.py b/tests/adapters/codex/test_probe_cleanup.py new file mode 100644 index 000000000..53efbe180 --- /dev/null +++ b/tests/adapters/codex/test_probe_cleanup.py @@ -0,0 +1,76 @@ +"""Incomplete process and reader cleanup cannot become a successful preflight.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.exceptions import CleanupIncompleteError +from band.integrations.codex import stdio_client, winspawn +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + + +class TestProbeCleanup: + async def test_unconfirmed_process_exit_stays_owned_for_retry( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + + async def not_observed( + _process: asyncio.subprocess.Process, _timeout: float + ) -> bool: + return False + + with monkeypatch.context() as patch: + patch.setattr(stdio_client, "await_returncode", not_observed) + patch.setattr(winspawn, "await_returncode", not_observed) + with pytest.raises(CleanupIncompleteError): + await adapter.preflight(timeout_s=5) + + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_unsettled_reader_prevents_success_and_is_rejoined( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + finish_reader = asyncio.Event() + reader_started = asyncio.Event() + + async def held_reader() -> None: + reader_started.set() + while not finish_reader.is_set(): + try: + await finish_reader.wait() + except asyncio.CancelledError: + continue + + reader = asyncio.create_task(held_reader()) + try: + await reader_started.wait() + await probe_host.received("account/read") + [client] = probe_host.clients + assert client._stderr_task is not None + client._stderr_task.cancel() + await asyncio.gather(client._stderr_task, return_exceptions=True) + client._stderr_task = reader + probing.cancel() + await spin() + with pytest.raises(asyncio.CancelledError): + await probing + with monkeypatch.context() as patch: + patch.setattr(stdio_client, "_EXIT_OBSERVE_S", 0) + with pytest.raises(CleanupIncompleteError): + await adapter._harness_probe.close(timeout_s=0.1) + finish_reader.set() + await reader + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + finish_reader.set() + await asyncio.gather(reader, return_exceptions=True) + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) diff --git a/tests/adapters/codex/test_probe_diagnostics.py b/tests/adapters/codex/test_probe_diagnostics.py new file mode 100644 index 000000000..b0900f2a9 --- /dev/null +++ b/tests/adapters/codex/test_probe_diagnostics.py @@ -0,0 +1,108 @@ +"""Standalone probes retain failures without copying raw transport diagnostics.""" + +from __future__ import annotations + +import asyncio +import json +import logging + +import pytest + +from band.integrations.codex.stdio_client import CodexStdioClient +from tests.adapters.codex.kit import send_bootstrap +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + +SENSITIVE_DIAGNOSTIC = "configured-token-that-must-not-reach-probe-logs" + + +class TestProbeDiagnostics: + async def test_real_probe_wire_and_stderr_are_metadata_only( + self, probe_host: ProbeHost, caplog: pytest.LogCaptureFixture + ) -> None: + adapter = probe_host.adapter(diagnostic=SENSITIVE_DIAGNOSTIC) + + with caplog.at_level(logging.DEBUG): + result = await adapter.preflight(timeout_s=5) + + assert result.ok + assert "initialize" in probe_host.methods + assert "account/read" in probe_host.methods + probe_host.assert_exited() + assert SENSITIVE_DIAGNOSTIC not in caplog.text + + async def test_room_wire_and_stderr_are_also_metadata_only( + self, probe_host: ProbeHost, caplog: pytest.LogCaptureFixture + ) -> None: + adapter = probe_host.adapter( + diagnostic=SENSITIVE_DIAGNOSTIC, model="configured-model" + ) + await adapter.on_started("Codex Agent", "A coding agent") + with caplog.at_level(logging.DEBUG): + await send_bootstrap(adapter, content="/model") + await adapter.cleanup_all() + + assert "initialize" in probe_host.methods + probe_host.assert_exited() + assert SENSITIVE_DIAGNOSTIC not in caplog.text + + @pytest.mark.parametrize("request_id", [None, 7], ids=["notification", "request"]) + async def test_full_transport_queue_logs_only_the_dropped_event_kind( + self, caplog: pytest.LogCaptureFixture, request_id: int | None + ) -> None: + client = CodexStdioClient() + client._events = asyncio.Queue(maxsize=1) + payload = {"method": SENSITIVE_DIAGNOSTIC, "id": request_id} + + with caplog.at_level(logging.WARNING): + await client._dispatch_rpc_message(json.dumps(payload)) + await client._dispatch_rpc_message(json.dumps(payload)) + + event = await client.recv_event(timeout_s=1) + assert event.method == SENSITIVE_DIAGNOSTIC + assert event.kind == ("notification" if request_id is None else "request") + assert any(record.levelno == logging.WARNING for record in caplog.records) + assert event.kind in caplog.text + assert SENSITIVE_DIAGNOSTIC not in caplog.text + + async def test_late_close_failure_keeps_the_original_error_and_retry_without_raw_log( + self, + probe_host: ProbeHost, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + close_started = asyncio.Event() + finish_close = asyncio.Event() + + async def refused_close() -> None: + close_started.set() + await finish_close.wait() + raise RuntimeError(SENSITIVE_DIAGNOSTIC) + + try: + with caplog.at_level(logging.DEBUG): + await probe_host.received("account/read") + [client] = probe_host.clients + with monkeypatch.context() as patch: + patch.setattr(client, "_close_process", refused_close) + probing.cancel() + await asyncio.wait_for(close_started.wait(), timeout=5) + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + finish_close.set() + assert client._close_task is not None + with pytest.raises(RuntimeError, match=SENSITIVE_DIAGNOSTIC): + await client._close_task + await spin() + assert client._proc is not None + assert client._proc.returncode is None + assert SENSITIVE_DIAGNOSTIC not in caplog.text + finally: + finish_close.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_probe_ownership.py b/tests/adapters/codex/test_probe_ownership.py new file mode 100644 index 000000000..c6648357a --- /dev/null +++ b/tests/adapters/codex/test_probe_ownership.py @@ -0,0 +1,138 @@ +"""A probe keeps partial spawns and failed cleanup reachable through the adapter.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from band.core.exceptions import CleanupIncompleteError +from tests.adapters.codex.kit import ROOM_ID, send_bootstrap +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + + +class TestProbeOwnership: + async def test_one_second_probe_reaps_a_child_hung_before_handshake( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(startup_hang=True) + + result = await adapter.preflight(timeout_s=1) + + assert not result.ok + probe_host.assert_exited() + + async def test_one_second_probe_reaps_a_late_published_hung_child_on_cancel( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter(startup_hang=True) + create_process = asyncio.create_subprocess_exec + created = asyncio.Event() + return_process = asyncio.Event() + processes: list[asyncio.subprocess.Process] = [] + + async def late_creation( + *args: str, **kwargs: Any + ) -> asyncio.subprocess.Process: + process = await create_process(*args, **kwargs) + processes.append(process) + created.set() + await return_process.wait() + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", late_creation) + probing = asyncio.create_task(adapter.preflight(timeout_s=1)) + try: + await asyncio.wait_for(created.wait(), timeout=5) + probing.cancel() + await spin() + return_process.set() + with pytest.raises(asyncio.CancelledError) as stopped: + await probing + assert stopped.value.__cause__ is None + assert processes + assert all(process.returncode is not None for process in processes) + assert all(client._windows_process is None for client in probe_host.clients) + assert adapter._room_clients == {} + finally: + return_process.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + for process in processes: + if process.returncode is None: + process.kill() + await process.wait() + + async def test_cancel_during_late_subprocess_creation_reaps_the_eventual_child( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + create_process = asyncio.create_subprocess_exec + created = asyncio.Event() + return_process = asyncio.Event() + processes: list[asyncio.subprocess.Process] = [] + + async def late_creation( + *args: str, **kwargs: Any + ) -> asyncio.subprocess.Process: + process = await create_process(*args, **kwargs) + processes.append(process) + created.set() + await return_process.wait() + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", late_creation) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await asyncio.wait_for(created.wait(), timeout=5) + probing.cancel() + await spin() + return_process.set() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + assert processes + assert all(process.returncode is not None for process in processes) + assert adapter._room_clients == {} + finally: + return_process.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + for process in processes: + if process.returncode is None: + process.kill() + await process.wait() + + async def test_probe_failure_does_not_skip_existing_room_cleanup( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + await adapter.on_started("Agent", "A coding agent") + await send_bootstrap(adapter, content="/model") + [room_client] = probe_host.clients + assert ROOM_ID in adapter._room_clients + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await probe_host.received("account/read") + probe_client = probe_host.clients[-1] + probe_client.refuse_close = True + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + with pytest.raises(CleanupIncompleteError): + await adapter.cleanup_all() + assert room_client._proc is not None + assert room_client._proc.returncode is not None + assert adapter._workspaces.rooms == () + assert probe_client._proc is not None + assert probe_client._proc.returncode is None + probe_client.refuse_close = False + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) diff --git a/tests/adapters/copilot_sdk/test_session_configuration.py b/tests/adapters/copilot_sdk/test_session_configuration.py index 2c7391ae6..79e67dd7a 100644 --- a/tests/adapters/copilot_sdk/test_session_configuration.py +++ b/tests/adapters/copilot_sdk/test_session_configuration.py @@ -90,7 +90,8 @@ async def ask_operator(request, context): await run_message(adapter, tools) kwargs = client.sessions[0].kwargs - assert kwargs["on_user_input_request"] is ask_operator + response = await kwargs["on_user_input_request"]({"question": "Question?"}, {}) + assert response == {"answer": "beta", "wasFreeform": False} # ask_user must be allowlisted alongside the bridged Band tools, # or the built-in tool stays filtered out and the handler is dead. assert "ask_user" in kwargs["available_tools"] diff --git a/tests/adapters/letta/__init__.py b/tests/adapters/letta/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/letta/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/letta/kit.py b/tests/adapters/letta/kit.py new file mode 100644 index 000000000..990a45d37 --- /dev/null +++ b/tests/adapters/letta/kit.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json + +import pytest + +from band.core.types import ToolEventKey +from band.runtime.tools import BandTool +from band.testing import ( + FakeAgentTools, + events_of_type, +) + +REPLY_SETTLING_CALLS = [ + pytest.param( + BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@alice"]}, id="message" + ), + pytest.param( + BandTool.SEND_ROOM_FILE, + {"content": "notes", "filename": "notes.txt", "mentions": ["@alice"]}, + id="room-file", + ), + pytest.param(BandTool.NO_REPLY, {"reason": "FYI only"}, id="no-reply"), +] + + +def sent_contents(tools: FakeAgentTools) -> list[str]: + return [message["content"] for message in tools.messages_sent] + + +def reported_tool_names(tools: FakeAgentTools, event_type: str) -> list[str]: + return [ + json.loads(event["content"])[ToolEventKey.NAME] + for event in events_of_type(tools, event_type) + ] diff --git a/tests/adapters/letta/test_agentcreateoptions.py b/tests/adapters/letta/test_agentcreateoptions.py new file mode 100644 index 000000000..ca27c331f --- /dev/null +++ b/tests/adapters/letta/test_agentcreateoptions.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) +from tests.adapters.lettakit import ( + make_mock_agent, +) + + +class TestAgentCreateOptions: + @pytest.mark.asyncio + async def test_embedding_passed_when_configured(self) -> None: + adapter = LettaAdapter( + config=LettaAdapterConfig(embedding="openai/text-embedding-3-small") + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-1") + + create_kwargs = mock_client.agents.create.call_args.kwargs + assert create_kwargs["embedding"] == "openai/text-embedding-3-small" + + @pytest.mark.asyncio + async def test_embedding_omitted_by_default(self) -> None: + """Letta Cloud picks its own default — None must not be sent.""" + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-1") + + assert "embedding" not in mock_client.agents.create.call_args.kwargs + + @pytest.mark.asyncio + async def test_per_room_persona_carries_room_id(self) -> None: + """Self-hosted MCP schemas require room_id per call; a per_room agent + gets it in its persona block.""" + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-42") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + persona = next(b for b in blocks if b["label"] == "persona") + assert "room-42" in persona["value"] + + @pytest.mark.asyncio + async def test_external_mode_persona_omits_room_id(self) -> None: + """External band-mcp tool schemas carry no room_id argument.""" + adapter = LettaAdapter( + config=LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-42") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + persona = next(b for b in blocks if b["label"] == "persona") + assert "room-42" not in persona["value"] diff --git a/tests/adapters/letta/test_autorelaydisabled.py b/tests/adapters/letta/test_autorelaydisabled.py new file mode 100644 index 000000000..7a5bd09a7 --- /dev/null +++ b/tests/adapters/letta/test_autorelaydisabled.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from band.adapters.letta import ( + LettaAdapterConfig, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.core.types import Capability +from band.runtime.tools import BandTool +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, + reported_failures, +) +from tests.adapters.letta.kit import REPLY_SETTLING_CALLS, sent_contents +from tests.adapters.lettakit import ( + make_platform_message, + ready_letta_adapter, + scripted_letta_turn, +) +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools + + +class TestAutoRelayDisabled: + @pytest.mark.asyncio + async def test_disabled_relay_drops_the_text_and_the_verdict_reports_it( + self, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, room_id=ROOM_ID, tool_calls=[], final_text="I'll help you!" + ) + + tools = turn_tools() + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + async def test_disabled_relay_completes_a_turn_that_did_real_work(self) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id=ROOM_ID, + tool_calls=[(BandTool.CREATE_CHATROOM, {})], + final_text="Created it.", + ) + + tools = turn_tools() + await adapter.on_event(turn_input(tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) + async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( + self, reply_tool: str, arguments: dict[str, Any] + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), capabilities={Capability.FILES} + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert "Done!" not in sent_contents(tools) + assert not reported_failures(tools) diff --git a/tests/adapters/letta/test_coldbootseeding.py b/tests/adapters/letta/test_coldbootseeding.py new file mode 100644 index 000000000..175edb21d --- /dev/null +++ b/tests/adapters/letta/test_coldbootseeding.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_tool_page, + make_platform_message, +) + + +class TestColdBootSeeding: + @pytest.fixture + def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.server_id = "mcp-server-1" + adapter._mcp.tool_ids = [] + return adapter, mock_client + + @pytest.mark.asyncio + async def test_new_agent_in_room_with_history_is_seeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi again!") + ) + + history = LettaSessionState( + replay_messages=["[Alice]: The secret word is kumquat.", "[Bot]: Noted!"] + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(content="what was the secret word?"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ + "content" + ] + assert "joining an ongoing conversation" in content + assert "[Alice]: The secret word is kumquat." in content + assert "what was the secret word?" in content + # Seed is delivered exactly once + assert adapter._rooms["room-1"].pending_seed == [] + + @pytest.mark.asyncio + async def test_reported_turn_consumes_delivered_seed(self) -> None: + """A completed provider call consumes its seed before response handling.""" + adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.server_id = "mcp-server-1" + adapter._mcp.tool_ids = [] + room_ctx = RoomContext( + agent_id="agent-1", pending_seed=["[Alice]: Earlier context"] + ) + adapter._rooms["room-1"] = room_ctx + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("The response was not sent through the tool.") + ) + + await adapter.on_message( + make_platform_message(), + FakeAgentTools(), + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert room_ctx.pending_seed == [] + assert room_ctx.last_interaction is not None + + @pytest.mark.asyncio + async def test_failed_first_turn_preserves_pending_seed( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Cold-boot history must survive a failed first turn — clearing the + seed at compose time would drop replay context forever.""" + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.side_effect = TimeoutError("slow letta") + + history = LettaSessionState( + replay_messages=["[Alice]: The secret word is kumquat."] + ) + tools = FakeAgentTools() + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_message( + make_platform_message(content="what was the secret word?"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert adapter._rooms["room-1"].pending_seed == [ + "[Alice]: The secret word is kumquat." + ] + + @pytest.mark.asyncio + async def test_per_room_does_not_resume_config_agent_id( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """config.agent_id is not a per-room resume target — each room without + persisted history must get its own Letta agent.""" + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy( + update={"agent_id": "shared-bootstrap-id"} + ) + mock_client.agents.create.return_value = make_mock_agent("room-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + await adapter.on_message( + make_platform_message(room_id="room-1"), + FakeAgentTools(), + LettaSessionState(), + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.agents.retrieve.assert_not_called() + mock_client.agents.create.assert_awaited_once() + + @pytest.mark.asyncio + async def test_second_message_is_not_reseeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + history = LettaSessionState(replay_messages=["[Alice]: earlier message"]) + tools = FakeAgentTools() + for _ in range(2): + await adapter.on_message( + make_platform_message(), + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + second_content = mock_client.agents.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: earlier message" not in second_content + + @pytest.mark.asyncio + async def test_resumed_agent_is_not_seeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Resume-by-id stays the fast path — the live agent already carries + its own context, so replaying history would duplicate it.""" + adapter, mock_client = adapter_with_client + mock_client.agents.retrieve.return_value = make_mock_agent("live-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Resumed!") + ) + + history = LettaSessionState( + agent_id="live-agent", + replay_messages=["[Alice]: earlier message"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ + "content" + ] + assert "[Alice]: earlier message" not in content + mock_client.agents.create.assert_not_called() diff --git a/tests/adapters/letta/test_configenvsourcing.py b/tests/adapters/letta/test_configenvsourcing.py new file mode 100644 index 000000000..40233104c --- /dev/null +++ b/tests/adapters/letta/test_configenvsourcing.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import pytest + +from band.adapters.letta import ( + LettaAdapterConfig, +) + + +class TestConfigEnvSourcing: + """provider_key env sourcing: LETTA_* names only, never bare vars.""" + + @pytest.fixture(autouse=True) + def clean_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + for var in ("API_KEY", "PROVIDER_KEY", "LETTA_API_KEY", "LETTA_PROVIDER_KEY"): + monkeypatch.delenv(var, raising=False) + + def test_bare_env_vars_never_populate_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A foreign secret in API_KEY/PROVIDER_KEY must not reach Letta.""" + monkeypatch.setenv("API_KEY", "foreign-secret") + monkeypatch.setenv("PROVIDER_KEY", "foreign-secret") + + assert LettaAdapterConfig().provider_key is None + + def test_letta_api_key_env_populates_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_API_KEY", "cloud-key") + + assert LettaAdapterConfig().provider_key == "cloud-key" + + def test_prefixed_field_name_env_populates_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_PROVIDER_KEY", "prefixed-key") + + assert LettaAdapterConfig().provider_key == "prefixed-key" + + def test_explicit_kwarg_wins_over_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_API_KEY", "env-key") + + assert LettaAdapterConfig(provider_key="kwarg-key").provider_key == "kwarg-key" + + def test_prefixed_env_fills_a_field_and_an_explicit_value_beats_it( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_MODEL", "openai/env-model") + + assert LettaAdapterConfig().model == "openai/env-model" + assert LettaAdapterConfig(model="openai/explicit").model == "openai/explicit" diff --git a/tests/adapters/letta/test_deleteagentsoncleanup.py b/tests/adapters/letta/test_deleteagentsoncleanup.py new file mode 100644 index 000000000..e5a2cf5b3 --- /dev/null +++ b/tests/adapters/letta/test_deleteagentsoncleanup.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) + + +class TestDeleteAgentsOnCleanup: + @pytest.mark.asyncio + async def test_opt_in_deletes_agent_instead_of_consolidating(self) -> None: + adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_awaited_once_with("agent-1") + # No consolidation prompt when the agent is being deleted + mock_client.agents.messages.create.assert_not_called() + + @pytest.mark.asyncio + async def test_default_keeps_agent(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_not_called() + + @pytest.mark.asyncio + async def test_can_skip_consolidation_on_cleanup(self) -> None: + adapter = LettaAdapter( + config=LettaAdapterConfig(consolidate_memory_on_cleanup=False) + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_not_called() + mock_client.agents.messages.create.assert_not_called() + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_delete_failure_does_not_propagate(self) -> None: + adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) + mock_client = AsyncMock() + adapter._client = mock_client + mock_client.agents.delete.side_effect = Exception("gone already") + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") # should not raise + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_executionreporting.py b/tests/adapters/letta/test_executionreporting.py new file mode 100644 index 000000000..6ae667c07 --- /dev/null +++ b/tests/adapters/letta/test_executionreporting.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import pytest +from letta_client.types.agents.text_content import TextContent +from letta_client.types.agents.tool_call import ToolCall +from letta_client.types.agents.tool_return import ToolReturn + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.types import Emit +from band.runtime.tools import BandTool +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.letta.kit import reported_tool_names +from tests.adapters.lettakit import ( + make_assistant_message, + make_grouped_call_message, + make_grouped_return_message, + make_letta_response, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) + + +class TestExecutionReporting: + @pytest.mark.asyncio + async def test_reports_non_silent_tool_calls(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_lookup_peers", "{}"), + make_tool_return_message("band_lookup_peers", '{"peers": []}'), + make_assistant_message("Done"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_call_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_call" + ] + tool_result_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_result" + ] + assert len(tool_call_events) == 1 + assert len(tool_result_events) == 1 + + @pytest.mark.asyncio + async def test_reports_every_parallel_call_and_its_result(self) -> None: + adapter, mock_client = ready_letta_adapter(emit=Emit.TOOL_CALLS) + calls = [ + ToolCall(name=BandTool.GET_PARTICIPANTS, arguments="{}", tool_call_id="a"), + ToolCall(name=BandTool.LOOKUP_PEERS, arguments="{}", tool_call_id="b"), + ] + multimodal_output = [TextContent(type="text", text="no peers")] + returns = [ + ToolReturn( + type="tool", + status="success", + tool_call_id="b", + tool_return=multimodal_output, + ), + ToolReturn( + type="tool", status="success", tool_call_id="a", tool_return="[]" + ), + ] + mock_client.agents.messages.create.return_value = make_letta_response( + make_grouped_call_message(calls), + make_grouped_return_message(returns, name=calls[0].name), + make_assistant_message("Done"), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert reported_tool_names(tools, "tool_call") == [ + BandTool.GET_PARTICIPANTS, + BandTool.LOOKUP_PEERS, + ] + assert reported_tool_names(tools, "tool_result") == [ + BandTool.LOOKUP_PEERS, + BandTool.GET_PARTICIPANTS, + ] + + @pytest.mark.asyncio + async def test_silent_tools_not_reported(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_send_message"), + make_tool_return_message("band_send_message"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_events = [ + e + for e in tools.events_sent + if e["message_type"] in ("tool_call", "tool_result") + ] + assert len(tool_events) == 0 + + @pytest.mark.asyncio + async def test_send_room_file_reports_content_placeholder_not_raw_bytes( + self, + ) -> None: + """ToolCall.arguments is a raw JSON string (letta_client's own wire + shape); the tool_call event must still redact band_send_room_file's + content field, not embed the real file bytes verbatim.""" + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message( + "band_send_room_file", + '{"content": "SECRET FILE BYTES", "filename": "f.txt"}', + ), + make_tool_return_message("band_send_room_file", '{"status": "ok"}'), + make_assistant_message("Done"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_call_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_call" + ] + assert len(tool_call_events) == 1 + reported_args = json.loads(tool_call_events[0]["content"])["args"] + assert "SECRET FILE BYTES" not in json.dumps(reported_args) + assert "byte file content" in reported_args["content"] + assert reported_args["filename"] == "f.txt" diff --git a/tests/adapters/letta/test_externaltoolrecording.py b/tests/adapters/letta/test_externaltoolrecording.py new file mode 100644 index 000000000..238fdcd86 --- /dev/null +++ b/tests/adapters/letta/test_externaltoolrecording.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Literal + +import pytest +from letta_client.types.tool_return_message import ToolReturnMessage + +from band.adapters.letta import ( + LettaAdapterConfig, + LettaMCPConfig, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.runtime.tools import BandTool +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, +) +from tests.adapters.letta.kit import sent_contents +from tests.adapters.lettakit import ( + call_id, + make_assistant_message, + make_letta_response, + make_parallel_tool_messages, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools + + +class TestExternalToolRecording: + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("mcp_mode", "status", "replied"), + [ + pytest.param("external", "success", True, id="external-success"), + pytest.param("external", "error", False, id="external-error"), + # Self-hosted calls run on the room's own tools, which record them. + pytest.param("self_host", "success", False, id="self-host"), + ], + ) + async def test_streamed_tool_return_records_only_for_external_band_mcp( + self, + mcp_mode: Literal["self_host", "external"], + status: Literal["success", "error"], + replied: bool, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), + make_tool_return_message(BandTool.SEND_MESSAGE, status=status), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied is replied + + @pytest.mark.asyncio + async def test_ungrouped_tool_return_is_matched_to_its_call(self) -> None: + """A server that sends no ``tool_returns`` puts the one result on the + message itself.""" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), + ToolReturnMessage( + id="message-return", + date=datetime.now(UTC), + message_type="tool_return_message", + status="success", + tool_call_id=call_id(BandTool.SEND_MESSAGE), + tool_return="result", + ), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("effect_tool", "replied"), + [ + pytest.param(BandTool.ADD_PARTICIPANT, False, id="act"), + pytest.param(BandTool.SEND_MESSAGE, True, id="reply"), + pytest.param(BandTool.NO_REPLY, True, id="decline"), + ], + ) + @pytest.mark.parametrize("reverse_calls", [False, True]) + async def test_parallel_success_completes_turn_independent_of_order( + self, + effect_tool: str, + replied: bool, + reverse_calls: bool, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + results: list[tuple[str, Literal["success", "error"]]] = [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "success"), + ] + if reverse_calls: + results.reverse() + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages(results, reverse_returns=False) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert tools.turn.replied is replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "effect_tool", + [BandTool.ADD_PARTICIPANT, BandTool.SEND_MESSAGE, BandTool.NO_REPLY], + ) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_effect_does_not_complete_turn( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "error"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + @pytest.mark.parametrize("effect_tool", [BandTool.SEND_MESSAGE, BandTool.NO_REPLY]) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_reply_does_not_suppress_fallback( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [(effect_tool, "error"), (BandTool.GET_PARTICIPANTS, "success")], + reverse_returns=reverse_returns, + ), + make_assistant_message("Fallback reply"), + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert sent_contents(tools) == ["Fallback reply"] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_results_match_call_ids_not_positions( + self, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.SEND_MESSAGE, "error"), + (BandTool.ADD_PARTICIPANT, "success"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] diff --git a/tests/adapters/letta/test_extractsummary.py b/tests/adapters/letta/test_extractsummary.py new file mode 100644 index 000000000..500c2d986 --- /dev/null +++ b/tests/adapters/letta/test_extractsummary.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from band.adapters.letta import ( + LettaAdapter, +) + + +class TestExtractSummary: + def test_short_text_kept_whole(self) -> None: + parts = ["Hello there. This is more text."] + assert "Hello there. This is more text." == LettaAdapter._extract_summary(parts) + + def test_truncates_on_word_boundary(self) -> None: + parts = ["alpha beta " * 30] + result = LettaAdapter._extract_summary(parts, max_length=50) + assert len(result) <= 53 # 50 + "..." + assert result.endswith("...") + assert not result.removesuffix("...").endswith("alph") # no mid-word cut + + def test_no_sentence_heuristics(self) -> None: + """Decimals must not be mistaken for sentence ends.""" + parts = ["pi is 3.14 and e is 2.71"] + assert "pi is 3.14 and e is 2.71" == LettaAdapter._extract_summary(parts) + + def test_empty(self) -> None: + assert "" == LettaAdapter._extract_summary([]) + + def test_multiple_parts_joined(self) -> None: + parts = ["First part.", "Second part."] + assert "First part. Second part." == LettaAdapter._extract_summary(parts) diff --git a/tests/adapters/letta/test_formattimeago.py b/tests/adapters/letta/test_formattimeago.py new file mode 100644 index 000000000..d1965a3c4 --- /dev/null +++ b/tests/adapters/letta/test_formattimeago.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from band.adapters.letta import ( + LettaAdapter, +) + + +class TestFormatTimeAgo: + def test_seconds(self) -> None: + dt = datetime.now(UTC) - timedelta(seconds=30) + assert "30s" == LettaAdapter._format_time_ago(dt) + + def test_minutes(self) -> None: + dt = datetime.now(UTC) - timedelta(minutes=5) + assert "5m" == LettaAdapter._format_time_ago(dt) + + def test_hours(self) -> None: + dt = datetime.now(UTC) - timedelta(hours=3) + assert "3h" == LettaAdapter._format_time_ago(dt) + + def test_one_hour(self) -> None: + dt = datetime.now(UTC) - timedelta(hours=1) + assert "1 hour" == LettaAdapter._format_time_ago(dt) + + def test_days(self) -> None: + dt = datetime.now(UTC) - timedelta(days=5) + assert "5d" == LettaAdapter._format_time_ago(dt) + + def test_one_day(self) -> None: + dt = datetime.now(UTC) - timedelta(days=1) + assert "1 day" == LettaAdapter._format_time_ago(dt) + + def test_naive_datetime_treated_as_utc(self) -> None: + dt = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=10) + result = LettaAdapter._format_time_ago(dt) + assert "10m" == result diff --git a/tests/adapters/letta/test_instructionblockupdate.py b/tests/adapters/letta/test_instructionblockupdate.py new file mode 100644 index 000000000..f3f0cb1f2 --- /dev/null +++ b/tests/adapters/letta/test_instructionblockupdate.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, +) +from tests.adapters.lettakit import ( + default_enforcement, +) + + +class TestInstructionBlockUpdate: + @pytest.mark.asyncio + async def test_updates_persona_block(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test system prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + await adapter._update_instruction_block("agent-1", "room-1") + + # Default config self-hosts the MCP server in per_room mode, so the + # enforcement carries the room id the tool schemas require. + mock_client.agents.blocks.update.assert_called_once_with( + "persona", + agent_id="agent-1", + value=default_enforcement(room_id="room-1") + "Test system prompt", + ) + + @pytest.mark.asyncio + async def test_falls_back_to_next_label(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + # First label fails, second succeeds + mock_client.agents.blocks.update.side_effect = [ + Exception("not found"), + None, + None, + ] + + await adapter._update_instruction_block("agent-1", "room-1") + + assert mock_client.agents.blocks.update.call_count == 2 + second_call = mock_client.agents.blocks.update.call_args_list[1] + assert second_call.args[0] == "custom_instructions" + + @pytest.mark.asyncio + async def test_creates_persona_when_all_labels_fail(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + mock_client.agents.blocks.update.side_effect = Exception("not found") + mock_block = MagicMock() + mock_block.id = "block-1" + mock_client.blocks.create.return_value = mock_block + + await adapter._update_instruction_block("agent-1", "room-1") + + mock_client.blocks.create.assert_called_once() + assert mock_client.blocks.create.call_args.kwargs["label"] == "persona" + mock_client.agents.blocks.attach.assert_called_once_with( + "block-1", + agent_id="agent-1", + ) diff --git a/tests/adapters/letta/test_lettaadapterinit.py b/tests/adapters/letta/test_lettaadapterinit.py new file mode 100644 index 000000000..99a5b535e --- /dev/null +++ b/tests/adapters/letta/test_lettaadapterinit.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) + + +class TestLettaAdapterInit: + # Default values are not asserted here: that would only restate the + # dataclass definition, and the frozen-config conformance suite already + # pins ``LettaAdapter().config == LettaAdapterConfig()``. + + def test_custom_config_reaches_adapter(self) -> None: + config = LettaAdapterConfig( + base_url="http://custom:8283", + provider_key="sk-test", + mode="shared", + mcp=LettaMCPConfig(mode="external", server_url="http://mcp:9000/sse"), + ) + adapter = LettaAdapter(config=config) + assert adapter.config is config + + def test_no_leaked_adapter_config_env_vars( + self, assert_no_leaked_adapter_config_env: None + ) -> None: + """Requesting the fixture is the assertion — see its docstring.""" diff --git a/tests/adapters/letta/test_lettaadapteroncleanup.py b/tests/adapters/letta/test_lettaadapteroncleanup.py new file mode 100644 index 000000000..b2ffff17b --- /dev/null +++ b/tests/adapters/letta/test_lettaadapteroncleanup.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) + + +class TestLettaAdapterOnCleanup: + @pytest.mark.asyncio + async def test_cleanup_removes_room_state(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + assert "room-1" not in adapter._rooms + # Memory consolidation was attempted + mock_client.agents.messages.create.assert_called_once() + + @pytest.mark.asyncio + async def test_cleanup_idempotent(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + + # Room doesn't exist + await adapter.on_cleanup("nonexistent") + mock_client.agents.messages.create.assert_not_called() + + @pytest.mark.asyncio + async def test_cleanup_twice(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + await adapter.on_cleanup("room-1") # Should not raise + + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_multi_room(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + adapter._rooms["room-2"] = RoomContext(agent_id="agent-2") + + await adapter.on_cleanup("room-1") + + assert "room-1" not in adapter._rooms + assert "room-2" in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_without_client(self) -> None: + adapter = LettaAdapter() + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # No client — should not raise + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_before_started(self) -> None: + adapter = LettaAdapter() + # Completely uninitialized — no client, no rooms + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_lettaadapteronmessageperroom.py b/tests/adapters/letta/test_lettaadapteronmessageperroom.py new file mode 100644 index 000000000..ea6b9c0a4 --- /dev/null +++ b/tests/adapters/letta/test_lettaadapteronmessageperroom.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + GENERIC_PROVIDER_FAILURE_MESSAGE, + TurnResultAlreadyReported, +) +from band.core.types import Capability, Emit +from band.testing import ( + FakeAgentTools, + reported_failures, +) +from tests.adapters.letta.kit import REPLY_SETTLING_CALLS, sent_contents +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_tool_page, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, + scripted_letta_turn, +) + + +class TestLettaAdapterOnMessagePerRoom: + """Tests for per_room mode message handling.""" + + @pytest.fixture + def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.tool_ids = ["t1", "t2"] + adapter._mcp.server_id = "mcp-server-1" + return adapter, mock_client + + @pytest.mark.asyncio + async def test_basic_message_creates_agent_and_sends( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + mock_agent = make_mock_agent("agent-1") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # Agent was created + mock_client.agents.create.assert_called_once() + # MCP tools were attached + assert mock_client.agents.tools.attach.call_count == 2 + # Message was sent via direct agent API (not conversations) + mock_client.agents.messages.create.assert_called_once() + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + assert call_kwargs["agent_id"] == "agent-1" + assert "conversation_id" not in call_kwargs + + @pytest.mark.asyncio + async def test_configured_memory_blocks_follow_the_persona_block( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + human = {"label": "human", "value": "Prefers short answers."} + adapter.config = LettaAdapterConfig(memory_blocks=(human,)) + mock_client.agents.create.return_value = make_mock_agent("agent-1") + + await adapter._create_agent("room-1") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + assert [block["label"] for block in blocks] == ["persona", "human"] + assert blocks[1] == human + + @pytest.mark.asyncio + async def test_auto_relay_when_no_send_message( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + # Setup room with agent + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + # Auto-relay should have sent the message + assert len(tools.messages_sent) == 1 + assert tools.messages_sent[0]["content"] == "I'll help you!" + + @pytest.mark.asyncio + async def test_send_message_failure_is_not_reported_as_provider_failure( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """The Letta agent answered fine; the room POST is what failed. That + must not surface as a Letta AgentFailure -- deliver_reply's + DeliveryFailedError must be recognized and left unreported here.""" + adapter, mock_client = adapter_with_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + + async def _raise(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("platform rejected the message") + + tools.send_message = _raise # type: ignore[method-assign] + + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(RuntimeError, match="platform rejected the message"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert not reported_failures(tools) + + @pytest.mark.asyncio + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) + async def test_skip_auto_relay_when_a_reply_tool_settled_the_turn( + self, reply_tool: str, arguments: dict[str, Any] + ) -> None: + adapter, mock_client = ready_letta_adapter(capabilities={Capability.FILES}) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert "Done!" not in sent_contents(tools) + + @pytest.mark.asyncio + async def test_timeout_reports_error( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.01}) + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + async def slow_response(**kwargs: Any) -> MagicMock: + await asyncio.sleep(1) + return make_letta_response() + + mock_client.agents.messages.create.side_effect = slow_response + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert "timed out" in failures[0]["message"] + assert failures[0]["provider"] == "letta" + assert failures[0]["code"] == "timeout" + + @pytest.mark.asyncio + async def test_slow_delivery_is_not_misreported_as_provider_timeout( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """turn_timeout_s bounds only the Letta round-trip. A room POST that + is merely slow (not the Letta call) must not be misreported as a + Letta provider timeout -- it must be given time to complete.""" + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.05}) + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # The Letta call itself responds instantly, well inside turn_timeout_s. + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + real_send_message = tools.send_message + + async def _slow_send_message(*args: Any, **kwargs: Any) -> Any: + # Longer than turn_timeout_s: only the provider call may race it. + await asyncio.sleep(0.15) + return await real_send_message(*args, **kwargs) + + tools.send_message = _slow_send_message # type: ignore[method-assign] + + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert len(tools.messages_sent) == 1 + assert not reported_failures(tools) + + @pytest.mark.asyncio + async def test_send_event_timeout_is_not_misreported_as_provider_timeout( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A TimeoutError from tool-event reporting (send_event) -- not the + Letta round-trip itself -- must fall through to the generic failure + path, never be mislabeled as a Letta provider timeout. (send_failure + itself best-effort-swallows the same broken channel here, same as + production, so the room never receives a failure event either way -- + what this guards is which branch is taken/logged.)""" + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # The Letta round-trip itself returns instantly -- well inside + # turn_timeout_s -- so any TimeoutError must come from elsewhere. + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_lookup_peers", "{}"), + make_tool_return_message("band_lookup_peers", '{"peers": []}'), + ) + + tools = FakeAgentTools() + tools.send_event_error = TimeoutError("event POST hiccup") + msg = make_platform_message() + history = LettaSessionState() + + with ( + caplog.at_level(logging.ERROR, logger="band.adapters.letta"), + pytest.raises(TimeoutError), + ): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert not any("timed out" in r.message for r in caplog.records) + assert any("Error during Letta turn" in r.message for r in caplog.records) + + @pytest.mark.asyncio + async def test_generic_exception_reports_and_propagates( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A bare exception from the Letta client (not a timeout, not a + delivery failure) must still surface via the generic fallback.""" + adapter, mock_client = adapter_with_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + mock_client.agents.messages.create.side_effect = ConnectionError( + "letta connection reset" + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(ConnectionError, match="letta connection reset"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + failure = reported_failures(tools)[0] + assert failure["provider"] == "letta" + assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE + assert "letta connection reset" not in failure["message"] + + @pytest.mark.asyncio + async def test_participants_and_contacts_injected( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Got it.") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + "Participants: Alice, Bob", + "Contacts: Charlie", + is_session_bootstrap=False, + room_id="room-1", + ) + + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + content = call_kwargs["messages"][0]["content"] + assert "[System]: Participants: Alice, Bob" in content + assert "[System]: Contacts: Charlie" in content + + @pytest.mark.asyncio + async def test_agent_resume_from_history( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + mock_client.agents.retrieve.return_value = make_mock_agent("history-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Resumed!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState(agent_id="history-agent") + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # Agent was retrieved, not created + mock_client.agents.retrieve.assert_called_once_with("history-agent") + mock_client.agents.create.assert_not_called() + + @pytest.mark.asyncio + async def test_uninitialized_client_reports_error(self) -> None: + adapter = LettaAdapter() + # _client is None + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(RuntimeError, match="not initialized"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert failures[0]["provider"] == "letta" + assert "not initialized" in failures[0]["message"] diff --git a/tests/adapters/letta/test_lettaadaptersharedmode.py b/tests/adapters/letta/test_lettaadaptersharedmode.py new file mode 100644 index 000000000..f8323225c --- /dev/null +++ b/tests/adapters/letta/test_lettaadaptersharedmode.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_mock_agent, + make_mock_async_stream, + make_mock_conversation, + make_mock_tool_page, + make_platform_message, +) + + +class TestLettaAdapterSharedMode: + """Tests for shared mode with Conversations API.""" + + @pytest.fixture + def shared_adapter(self) -> tuple[LettaAdapter, AsyncMock]: + config = LettaAdapterConfig(mode="shared") + adapter = LettaAdapter(config=config) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.tool_ids = ["t1", "t2"] + adapter._mcp.server_id = "mcp-server-1" + return adapter, mock_client + + @pytest.mark.asyncio + async def test_shared_mode_creates_agent_and_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + + mock_agent = make_mock_agent("shared-agent") + mock_client.agents.create.return_value = mock_agent + mock_conv = make_mock_conversation("conv-room1") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi from shared!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-1") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # One agent created + mock_client.agents.create.assert_called_once() + # Conversation created for this room + mock_client.conversations.create.assert_called_once_with( + agent_id="shared-agent", + ) + # Message sent through room-scoped conversation endpoint + call_kwargs = mock_client.conversations.messages.create.call_args.kwargs + assert call_kwargs["conversation_id"] == "conv-room1" + assert adapter._shared_agent_id == "shared-agent" + + @pytest.mark.asyncio + async def test_shared_mode_reuses_agent_for_second_room( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + + # Setup: first room already connected + adapter._shared_agent_id = "shared-agent" + adapter._rooms["room-1"] = RoomContext( + agent_id="shared-agent", conversation_id="conv-1" + ) + + mock_conv2 = make_mock_conversation("conv-room2") + mock_client.conversations.create.return_value = mock_conv2 + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi room 2!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-2") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-2", + ) + + # Agent was NOT created again + mock_client.agents.create.assert_not_called() + # But a new conversation was created + mock_client.conversations.create.assert_called_once_with( + agent_id="shared-agent", + ) + assert adapter._rooms["room-2"].conversation_id == "conv-room2" + + @pytest.mark.asyncio + async def test_shared_mode_resumes_existing_agent( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + adapter.config = adapter.config.model_copy( + update={"agent_id": "pre-existing-agent"} + ) + + mock_client.agents.retrieve.return_value = make_mock_agent("pre-existing-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_conv = make_mock_conversation("conv-1") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Resumed shared!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-1") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.agents.retrieve.assert_called_once_with("pre-existing-agent") + mock_client.agents.create.assert_not_called() + assert adapter._shared_agent_id == "pre-existing-agent" + + @pytest.mark.asyncio + async def test_shared_mode_resumes_persisted_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A restart with a persisted conversation reuses it — a fresh one + would silently drop the room's conversational context.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.return_value = make_mock_conversation( + "conv-persisted", agent_id="shared-agent" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Still here!") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-persisted", + replay_messages=["[Alice]: earlier"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-1"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.conversations.retrieve.assert_awaited_once_with("conv-persisted") + mock_client.conversations.create.assert_not_called() + assert adapter._rooms["room-1"].conversation_id == "conv-persisted" + # A resumed conversation already has its context — no seeding. + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: earlier" not in content + + @pytest.mark.asyncio + async def test_shared_mode_rejects_foreign_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A persisted conversation must belong to the current shared agent.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.return_value = make_mock_conversation( + "conv-other", agent_id="other-agent" + ) + mock_client.conversations.create.return_value = make_mock_conversation( + "conv-fresh", agent_id="shared-agent" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Fresh thread") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-other", + ) + await adapter.on_message( + make_platform_message(room_id="room-1"), + FakeAgentTools(), + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.conversations.create.assert_awaited_once_with( + agent_id="shared-agent" + ) + assert adapter._rooms["room-1"].conversation_id == "conv-fresh" + + @pytest.mark.asyncio + async def test_shared_mode_seeds_fresh_conversation_from_history( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A dead persisted conversation falls back to create + seed.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.side_effect = Exception("gone") + mock_client.conversations.create.return_value = make_mock_conversation( + "conv-fresh" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Rebuilt!") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-dead", + replay_messages=["[Alice]: the secret is kumquat"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-1"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert adapter._rooms["room-1"].conversation_id == "conv-fresh" + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: the secret is kumquat" in content + + @pytest.mark.asyncio + async def test_shared_mode_injects_room_id_per_message( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Self-host + shared: the (shared) persona cannot carry a room id, so + every message reminds the agent which room_id to pass to tools.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + adapter._rooms["room-42"] = RoomContext( + agent_id="shared-agent", conversation_id="conv-1" + ) + + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-42"), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-42", + ) + + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "Current chat_id: room-42" in content diff --git a/tests/adapters/letta/test_memoryconsolidation.py b/tests/adapters/letta/test_memoryconsolidation.py new file mode 100644 index 000000000..b49139c80 --- /dev/null +++ b/tests/adapters/letta/test_memoryconsolidation.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) + + +class TestMemoryConsolidation: + @pytest.mark.asyncio + async def test_consolidation_on_cleanup(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.messages.create.assert_called_once() + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + assert "Consolidate" in call_kwargs["messages"][0]["content"] + + @pytest.mark.asyncio + async def test_consolidation_failure_does_not_propagate(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.side_effect = Exception("API error") + + # Should not raise + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_rejoincontext.py b/tests/adapters/letta/test_rejoincontext.py new file mode 100644 index 000000000..4c564cae4 --- /dev/null +++ b/tests/adapters/letta/test_rejoincontext.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, +) + + +class TestRejoinContext: + @pytest.mark.asyncio + async def test_rejoin_injects_time_away(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + last_time = datetime.now(UTC) - timedelta(hours=2) + adapter._rooms["room-1"] = RoomContext( + agent_id="agent-1", + last_interaction=last_time, + summary="Discussed project plan", + ) + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'm back!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + content = call_kwargs["messages"][0]["content"] + assert "rejoined" in content + assert "2h" in content + assert "Discussed project plan" in content diff --git a/tests/adapters/letta/test_sendtoolresolution.py b/tests/adapters/letta/test_sendtoolresolution.py new file mode 100644 index 000000000..0ab2d06fe --- /dev/null +++ b/tests/adapters/letta/test_sendtoolresolution.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) + + +class TestSendToolResolution: + def test_resolves_external_band_mcp_names(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event", "health_check"] + ) + assert adapter._mcp.send_message_tool == "create_agent_chat_message" + assert adapter._mcp.send_event_tool == "create_agent_chat_event" + assert adapter._mcp.silent_reporting_tools == { + "create_agent_chat_message", + "create_agent_chat_event", + } + + def test_prefers_band_names_when_both_present(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools( + ["band_send_message", "create_agent_chat_message", "band_send_event"] + ) + assert adapter._mcp.send_message_tool == "band_send_message" + assert adapter._mcp.send_event_tool == "band_send_event" + + def test_falls_back_to_band_names_when_none_discovered(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools(["some_unrelated_tool"]) + assert adapter._mcp.send_message_tool == "band_send_message" + assert adapter._mcp.send_event_tool == "band_send_event" + + def test_enforcement_prompt_uses_resolved_names(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Base prompt" + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event"] + ) + text = adapter._instruction_text("room-1") + assert "create_agent_chat_message" in text + assert "band_send_message" not in text + + @pytest.mark.asyncio + async def test_relay_detection_uses_resolved_name(self) -> None: + """A send via the resolved (external) tool name suppresses auto-relay.""" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event"] + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("create_agent_chat_message"), + make_tool_return_message("create_agent_chat_message"), + make_assistant_message("Done!"), + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert len(tools.messages_sent) == 0 diff --git a/tests/adapters/letta/test_summarystorage.py b/tests/adapters/letta/test_summarystorage.py new file mode 100644 index 000000000..7a5d2f7b3 --- /dev/null +++ b/tests/adapters/letta/test_summarystorage.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, +) + + +class TestSummaryStorage: + @pytest.mark.asyncio + async def test_summary_stored_after_turn(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("The weather is sunny. More details follow.") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + room_ctx = adapter._rooms["room-1"] + assert room_ctx.summary == "The weather is sunny. More details follow." + assert room_ctx.last_interaction is not None diff --git a/tests/adapters/letta/test_taskevents.py b/tests/adapters/letta/test_taskevents.py new file mode 100644 index 000000000..93206615f --- /dev/null +++ b/tests/adapters/letta/test_taskevents.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_async_stream, + make_mock_conversation, + make_platform_message, +) + + +class TestTaskEvents: + @pytest.mark.asyncio + async def test_emits_task_event_on_agent_creation(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("new-agent") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 1 + metadata = task_events[0]["metadata"] + assert metadata["letta_agent_id"] == "new-agent" + assert metadata["letta_room_id"] == "room-1" + + @pytest.mark.asyncio + async def test_task_events_disabled(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=()) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("new-agent") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 0 + + @pytest.mark.asyncio + async def test_shared_mode_emits_conversation_id(self) -> None: + config = LettaAdapterConfig(mode="shared") + adapter = LettaAdapter(config=config) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("shared-agent") + mock_client.agents.create.return_value = mock_agent + mock_conv = make_mock_conversation("conv-123") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 1 + assert task_events[0]["metadata"]["letta_conversation_id"] == "conv-123" diff --git a/tests/adapters/lettakit.py b/tests/adapters/lettakit.py index 5ee801992..883240207 100644 --- a/tests/adapters/lettakit.py +++ b/tests/adapters/lettakit.py @@ -2,8 +2,7 @@ Mock builders for Letta API objects (messages, responses, agents, conversations, MCP servers/tools, async streams) and platform messages, used -by ``test_letta_adapter.py``, ``test_letta_mcp.py``, and -``test_letta_orgscope.py``. +by the Letta adapter and MCP suites. """ from __future__ import annotations @@ -18,6 +17,7 @@ from uuid import uuid4 from letta_client import AsyncLetta +from letta_client.types.agents.letta_streaming_response import LettaUsageStatistics from letta_client.types.agents.tool_call import ToolCall from letta_client.types.agents.tool_call_message import ToolCallMessage from letta_client.types.agents.tool_return import ToolReturn @@ -151,6 +151,21 @@ def make_letta_response(*messages: Any) -> MagicMock: """Create a fake Letta API response.""" resp = MagicMock() resp.messages = list(messages) + + async def stream() -> Any: + for message in resp.messages: + yield message + if usage := resp.__dict__.get("usage"): + values = { + name: value + for name in LettaUsageStatistics.model_fields + if isinstance(value := getattr(usage, name, None), int) + } + yield LettaUsageStatistics(message_type="usage_statistics", **values) + + resp.__aiter__.side_effect = stream + resp.__aenter__.return_value = resp + resp.__aexit__.return_value = None return resp @@ -331,15 +346,7 @@ def mock_org_user_provisioned( def make_mock_async_stream(*messages: MagicMock) -> Any: """Create a mock async stream yielding Letta messages.""" - class _AsyncStream: - def __init__(self, stream_messages: list[MagicMock]) -> None: - self._messages = stream_messages - - async def __aiter__(self) -> Any: - for stream_message in self._messages: - yield stream_message - - return _AsyncStream(list(messages)) + return make_letta_response(*messages) class RecordingLettaClient: @@ -386,7 +393,7 @@ def cancels(self) -> list[str]: return [entry for entry in self.journal if entry.startswith("cancel ")] async def _create_agent_turn( - self, *, agent_id: str, messages: list[dict[str, str]] + self, *, agent_id: str, messages: list[dict[str, str]], **stream_options: Any ) -> MagicMock: self.journal.append(f"send agent {agent_id}") await self.release.wait() diff --git a/tests/adapters/strands/__init__.py b/tests/adapters/strands/__init__.py index e69de29bb..9d48db4f9 100644 --- a/tests/adapters/strands/__init__.py +++ b/tests/adapters/strands/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/strands/activity/__init__.py b/tests/adapters/strands/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/strands/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/strands/activity/test_hooks.py b/tests/adapters/strands/activity/test_hooks.py new file mode 100644 index 000000000..daeec28c1 --- /dev/null +++ b/tests/adapters/strands/activity/test_hooks.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import pytest + +pytest.importorskip("strands") + +from strands.hooks.events import AfterToolCallEvent, BeforeToolCallEvent + +from band.adapters.strands import BandTurnHooks +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder + + +class TestStrandsActivity: + @pytest.mark.parametrize("status", ["success", "error"]) + async def test_native_hook_reports_without_narration(self, status: str) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + hooks = BandTurnHooks(tools, emit_execution=False, custom_effects={}) + call = { + "name": "shell", + "input": {"command": "private-argument"}, + "toolUseId": "call", + } + await hooks._on_before_tool( + BeforeToolCallEvent( + agent=None, invocation_state={}, tool_use=call, selected_tool=None + ) + ) + assert recorder.events == [{"kind": "tool_call", "name": "shell"}] + await hooks._on_after_tool( + AfterToolCallEvent( + agent=None, + selected_tool=None, + invocation_state={}, + tool_use=call, + result={ + "toolUseId": "call", + "status": status, + "content": [{"text": "private-result"}], + }, + ) + ) + recorder.assert_completed_tool("shell") + assert not tools.events_sent diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py deleted file mode 100644 index 5b0c07cf0..000000000 --- a/tests/adapters/test_letta_adapter.py +++ /dev/null @@ -1,2123 +0,0 @@ -"""Tests for LettaAdapter. - -Message-path and agent-lifecycle coverage. MCP wiring/lifecycle tests live -in ``test_letta_mcp.py``; the shared mock factories in ``lettakit.py``. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import UTC, datetime, timedelta -from typing import Any, Literal -from unittest.mock import AsyncMock, MagicMock - -import pytest -from letta_client.types.agents.text_content import TextContent -from letta_client.types.agents.tool_call import ToolCall -from letta_client.types.agents.tool_return import ToolReturn -from letta_client.types.tool_return_message import ToolReturnMessage - -from band.adapters.letta import ( - LettaAdapter, - LettaAdapterConfig, - LettaMCPConfig, - RoomContext, -) -from band.converters.letta import LettaSessionState -from band.core.protocols import ( - GENERIC_PROVIDER_FAILURE_MESSAGE, - TurnResultAlreadyReported, -) -from band.core.types import Capability, Emit, ToolEventKey -from band.runtime.tools import BandTool -from band.testing import ( - MISSING_REPLY_FAILURE, - FakeAgentTools, - events_of_type, - failure_reports, - reported_failures, -) -from tests.adapters.lettakit import ( - call_id, - default_enforcement, - make_assistant_message, - make_grouped_call_message, - make_grouped_return_message, - make_letta_response, - make_mock_agent, - make_mock_async_stream, - make_mock_conversation, - make_mock_tool_page, - make_parallel_tool_messages, - make_platform_message, - make_tool_call_message, - make_tool_return_message, - ready_letta_adapter, - scripted_letta_turn, -) -from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools - -# Band tool calls that settle the turn's reply, so no assistant text is relayed. -REPLY_SETTLING_CALLS = [ - pytest.param( - BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@alice"]}, id="message" - ), - pytest.param( - BandTool.SEND_ROOM_FILE, - {"content": "notes", "filename": "notes.txt", "mentions": ["@alice"]}, - id="room-file", - ), - pytest.param(BandTool.NO_REPLY, {"reason": "FYI only"}, id="no-reply"), -] - - -def sent_contents(tools: FakeAgentTools) -> list[str]: - return [message["content"] for message in tools.messages_sent] - - -def reported_tool_names(tools: FakeAgentTools, event_type: str) -> list[str]: - return [ - json.loads(event["content"])[ToolEventKey.NAME] - for event in events_of_type(tools, event_type) - ] - - -# ────────────────────────────────────────────────────────────────────── -# Initialization -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterInit: - # Default values are not asserted here: that would only restate the - # dataclass definition, and the frozen-config conformance suite already - # pins ``LettaAdapter().config == LettaAdapterConfig()``. - - def test_custom_config_reaches_adapter(self) -> None: - config = LettaAdapterConfig( - base_url="http://custom:8283", - provider_key="sk-test", - mode="shared", - mcp=LettaMCPConfig(mode="external", server_url="http://mcp:9000/sse"), - ) - adapter = LettaAdapter(config=config) - assert adapter.config is config - - def test_no_leaked_adapter_config_env_vars( - self, assert_no_leaked_adapter_config_env: None - ) -> None: - """Requesting the fixture is the assertion — see its docstring.""" - - -# ────────────────────────────────────────────────────────────────────── -# on_message (per_room mode) -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterOnMessagePerRoom: - """Tests for per_room mode message handling.""" - - @pytest.fixture - def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.tool_ids = ["t1", "t2"] - adapter._mcp.server_id = "mcp-server-1" - return adapter, mock_client - - @pytest.mark.asyncio - async def test_basic_message_creates_agent_and_sends( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - mock_agent = make_mock_agent("agent-1") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # Agent was created - mock_client.agents.create.assert_called_once() - # MCP tools were attached - assert mock_client.agents.tools.attach.call_count == 2 - # Message was sent via direct agent API (not conversations) - mock_client.agents.messages.create.assert_called_once() - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - assert call_kwargs["agent_id"] == "agent-1" - assert "conversation_id" not in call_kwargs - - @pytest.mark.asyncio - async def test_configured_memory_blocks_follow_the_persona_block( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - human = {"label": "human", "value": "Prefers short answers."} - adapter.config = LettaAdapterConfig(memory_blocks=(human,)) - mock_client.agents.create.return_value = make_mock_agent("agent-1") - - await adapter._create_agent("room-1") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - assert [block["label"] for block in blocks] == ["persona", "human"] - assert blocks[1] == human - - @pytest.mark.asyncio - async def test_auto_relay_when_no_send_message( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - # Setup room with agent - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - # Auto-relay should have sent the message - assert len(tools.messages_sent) == 1 - assert tools.messages_sent[0]["content"] == "I'll help you!" - - @pytest.mark.asyncio - async def test_send_message_failure_is_not_reported_as_provider_failure( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """The Letta agent answered fine; the room POST is what failed. That - must not surface as a Letta AgentFailure -- deliver_reply's - DeliveryFailedError must be recognized and left unreported here.""" - adapter, mock_client = adapter_with_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - - async def _raise(*args: Any, **kwargs: Any) -> None: - raise RuntimeError("platform rejected the message") - - tools.send_message = _raise # type: ignore[method-assign] - - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(RuntimeError, match="platform rejected the message"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert not reported_failures(tools) - - @pytest.mark.asyncio - @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) - async def test_skip_auto_relay_when_a_reply_tool_settled_the_turn( - self, reply_tool: str, arguments: dict[str, Any] - ) -> None: - adapter, mock_client = ready_letta_adapter(capabilities={Capability.FILES}) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id="room-1", - tool_calls=[(reply_tool, arguments)], - final_text="Done!", - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert "Done!" not in sent_contents(tools) - - @pytest.mark.asyncio - async def test_timeout_reports_error( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.01}) - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - async def slow_response(**kwargs: Any) -> MagicMock: - await asyncio.sleep(1) - return make_letta_response() - - mock_client.agents.messages.create.side_effect = slow_response - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert "timed out" in failures[0]["message"] - assert failures[0]["provider"] == "letta" - assert failures[0]["code"] == "timeout" - - @pytest.mark.asyncio - async def test_slow_delivery_is_not_misreported_as_provider_timeout( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """turn_timeout_s bounds only the Letta round-trip. A room POST that - is merely slow (not the Letta call) must not be misreported as a - Letta provider timeout -- it must be given time to complete.""" - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.05}) - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # The Letta call itself responds instantly, well inside turn_timeout_s. - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - real_send_message = tools.send_message - - async def _slow_send_message(*args: Any, **kwargs: Any) -> Any: - # Longer than turn_timeout_s: only the provider call may race it. - await asyncio.sleep(0.15) - return await real_send_message(*args, **kwargs) - - tools.send_message = _slow_send_message # type: ignore[method-assign] - - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert len(tools.messages_sent) == 1 - assert not reported_failures(tools) - - @pytest.mark.asyncio - async def test_send_event_timeout_is_not_misreported_as_provider_timeout( - self, caplog: pytest.LogCaptureFixture - ) -> None: - """A TimeoutError from tool-event reporting (send_event) -- not the - Letta round-trip itself -- must fall through to the generic failure - path, never be mislabeled as a Letta provider timeout. (send_failure - itself best-effort-swallows the same broken channel here, same as - production, so the room never receives a failure event either way -- - what this guards is which branch is taken/logged.)""" - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # The Letta round-trip itself returns instantly -- well inside - # turn_timeout_s -- so any TimeoutError must come from elsewhere. - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_lookup_peers", "{}"), - make_tool_return_message("band_lookup_peers", '{"peers": []}'), - ) - - tools = FakeAgentTools() - tools.send_event_error = TimeoutError("event POST hiccup") - msg = make_platform_message() - history = LettaSessionState() - - with ( - caplog.at_level(logging.ERROR, logger="band.adapters.letta"), - pytest.raises(TimeoutError), - ): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert not any("timed out" in r.message for r in caplog.records) - assert any("Error during Letta turn" in r.message for r in caplog.records) - - @pytest.mark.asyncio - async def test_generic_exception_reports_and_propagates( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A bare exception from the Letta client (not a timeout, not a - delivery failure) must still surface via the generic fallback.""" - adapter, mock_client = adapter_with_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - mock_client.agents.messages.create.side_effect = ConnectionError( - "letta connection reset" - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(ConnectionError, match="letta connection reset"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - failure = reported_failures(tools)[0] - assert failure["provider"] == "letta" - assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE - assert "letta connection reset" not in failure["message"] - - @pytest.mark.asyncio - async def test_participants_and_contacts_injected( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Got it.") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - "Participants: Alice, Bob", - "Contacts: Charlie", - is_session_bootstrap=False, - room_id="room-1", - ) - - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - content = call_kwargs["messages"][0]["content"] - assert "[System]: Participants: Alice, Bob" in content - assert "[System]: Contacts: Charlie" in content - - @pytest.mark.asyncio - async def test_agent_resume_from_history( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - mock_client.agents.retrieve.return_value = make_mock_agent("history-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Resumed!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState(agent_id="history-agent") - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # Agent was retrieved, not created - mock_client.agents.retrieve.assert_called_once_with("history-agent") - mock_client.agents.create.assert_not_called() - - @pytest.mark.asyncio - async def test_uninitialized_client_reports_error(self) -> None: - adapter = LettaAdapter() - # _client is None - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(RuntimeError, match="not initialized"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert failures[0]["provider"] == "letta" - assert "not initialized" in failures[0]["message"] - - -# ────────────────────────────────────────────────────────────────────── -# on_message (shared mode) -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterSharedMode: - """Tests for shared mode with Conversations API.""" - - @pytest.fixture - def shared_adapter(self) -> tuple[LettaAdapter, AsyncMock]: - config = LettaAdapterConfig(mode="shared") - adapter = LettaAdapter(config=config) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.tool_ids = ["t1", "t2"] - adapter._mcp.server_id = "mcp-server-1" - return adapter, mock_client - - @pytest.mark.asyncio - async def test_shared_mode_creates_agent_and_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - - mock_agent = make_mock_agent("shared-agent") - mock_client.agents.create.return_value = mock_agent - mock_conv = make_mock_conversation("conv-room1") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi from shared!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-1") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # One agent created - mock_client.agents.create.assert_called_once() - # Conversation created for this room - mock_client.conversations.create.assert_called_once_with( - agent_id="shared-agent", - ) - # Message sent through room-scoped conversation endpoint - call_kwargs = mock_client.conversations.messages.create.call_args.kwargs - assert call_kwargs["conversation_id"] == "conv-room1" - assert adapter._shared_agent_id == "shared-agent" - - @pytest.mark.asyncio - async def test_shared_mode_reuses_agent_for_second_room( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - - # Setup: first room already connected - adapter._shared_agent_id = "shared-agent" - adapter._rooms["room-1"] = RoomContext( - agent_id="shared-agent", conversation_id="conv-1" - ) - - mock_conv2 = make_mock_conversation("conv-room2") - mock_client.conversations.create.return_value = mock_conv2 - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi room 2!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-2") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-2", - ) - - # Agent was NOT created again - mock_client.agents.create.assert_not_called() - # But a new conversation was created - mock_client.conversations.create.assert_called_once_with( - agent_id="shared-agent", - ) - assert adapter._rooms["room-2"].conversation_id == "conv-room2" - - @pytest.mark.asyncio - async def test_shared_mode_resumes_existing_agent( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - adapter.config = adapter.config.model_copy( - update={"agent_id": "pre-existing-agent"} - ) - - mock_client.agents.retrieve.return_value = make_mock_agent("pre-existing-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_conv = make_mock_conversation("conv-1") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Resumed shared!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-1") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.agents.retrieve.assert_called_once_with("pre-existing-agent") - mock_client.agents.create.assert_not_called() - assert adapter._shared_agent_id == "pre-existing-agent" - - @pytest.mark.asyncio - async def test_shared_mode_resumes_persisted_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A restart with a persisted conversation reuses it — a fresh one - would silently drop the room's conversational context.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.return_value = make_mock_conversation( - "conv-persisted", agent_id="shared-agent" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Still here!") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-persisted", - replay_messages=["[Alice]: earlier"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-1"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.conversations.retrieve.assert_awaited_once_with("conv-persisted") - mock_client.conversations.create.assert_not_called() - assert adapter._rooms["room-1"].conversation_id == "conv-persisted" - # A resumed conversation already has its context — no seeding. - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: earlier" not in content - - @pytest.mark.asyncio - async def test_shared_mode_rejects_foreign_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A persisted conversation must belong to the current shared agent.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.return_value = make_mock_conversation( - "conv-other", agent_id="other-agent" - ) - mock_client.conversations.create.return_value = make_mock_conversation( - "conv-fresh", agent_id="shared-agent" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Fresh thread") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-other", - ) - await adapter.on_message( - make_platform_message(room_id="room-1"), - FakeAgentTools(), - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.conversations.create.assert_awaited_once_with( - agent_id="shared-agent" - ) - assert adapter._rooms["room-1"].conversation_id == "conv-fresh" - - @pytest.mark.asyncio - async def test_shared_mode_seeds_fresh_conversation_from_history( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A dead persisted conversation falls back to create + seed.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.side_effect = Exception("gone") - mock_client.conversations.create.return_value = make_mock_conversation( - "conv-fresh" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Rebuilt!") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-dead", - replay_messages=["[Alice]: the secret is kumquat"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-1"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert adapter._rooms["room-1"].conversation_id == "conv-fresh" - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: the secret is kumquat" in content - - @pytest.mark.asyncio - async def test_shared_mode_injects_room_id_per_message( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Self-host + shared: the (shared) persona cannot carry a room id, so - every message reminds the agent which room_id to pass to tools.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - adapter._rooms["room-42"] = RoomContext( - agent_id="shared-agent", conversation_id="conv-1" - ) - - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-42"), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-42", - ) - - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "Current chat_id: room-42" in content - - -# ────────────────────────────────────────────────────────────────────── -# Execution reporting (observation only) -# ────────────────────────────────────────────────────────────────────── - - -class TestExecutionReporting: - @pytest.mark.asyncio - async def test_reports_non_silent_tool_calls(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_lookup_peers", "{}"), - make_tool_return_message("band_lookup_peers", '{"peers": []}'), - make_assistant_message("Done"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_call_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_call" - ] - tool_result_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_result" - ] - assert len(tool_call_events) == 1 - assert len(tool_result_events) == 1 - - @pytest.mark.asyncio - async def test_reports_every_parallel_call_and_its_result(self) -> None: - adapter, mock_client = ready_letta_adapter(emit=Emit.TOOL_CALLS) - calls = [ - ToolCall(name=BandTool.GET_PARTICIPANTS, arguments="{}", tool_call_id="a"), - ToolCall(name=BandTool.LOOKUP_PEERS, arguments="{}", tool_call_id="b"), - ] - multimodal_output = [TextContent(type="text", text="no peers")] - returns = [ - ToolReturn( - type="tool", - status="success", - tool_call_id="b", - tool_return=multimodal_output, - ), - ToolReturn( - type="tool", status="success", tool_call_id="a", tool_return="[]" - ), - ] - mock_client.agents.messages.create.return_value = make_letta_response( - make_grouped_call_message(calls), - make_grouped_return_message(returns, name=calls[0].name), - make_assistant_message("Done"), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert reported_tool_names(tools, "tool_call") == [ - BandTool.GET_PARTICIPANTS, - BandTool.LOOKUP_PEERS, - ] - assert reported_tool_names(tools, "tool_result") == [ - BandTool.LOOKUP_PEERS, - BandTool.GET_PARTICIPANTS, - ] - - @pytest.mark.asyncio - async def test_silent_tools_not_reported(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_send_message"), - make_tool_return_message("band_send_message"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_events = [ - e - for e in tools.events_sent - if e["message_type"] in ("tool_call", "tool_result") - ] - assert len(tool_events) == 0 - - @pytest.mark.asyncio - async def test_send_room_file_reports_content_placeholder_not_raw_bytes( - self, - ) -> None: - """ToolCall.arguments is a raw JSON string (letta_client's own wire - shape); the tool_call event must still redact band_send_room_file's - content field, not embed the real file bytes verbatim.""" - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message( - "band_send_room_file", - '{"content": "SECRET FILE BYTES", "filename": "f.txt"}', - ), - make_tool_return_message("band_send_room_file", '{"status": "ok"}'), - make_assistant_message("Done"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_call_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_call" - ] - assert len(tool_call_events) == 1 - reported_args = json.loads(tool_call_events[0]["content"])["args"] - assert "SECRET FILE BYTES" not in json.dumps(reported_args) - assert "byte file content" in reported_args["content"] - assert reported_args["filename"] == "f.txt" - - -# ────────────────────────────────────────────────────────────────────── -# on_cleanup -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterOnCleanup: - @pytest.mark.asyncio - async def test_cleanup_removes_room_state(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - assert "room-1" not in adapter._rooms - # Memory consolidation was attempted - mock_client.agents.messages.create.assert_called_once() - - @pytest.mark.asyncio - async def test_cleanup_idempotent(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - - # Room doesn't exist - await adapter.on_cleanup("nonexistent") - mock_client.agents.messages.create.assert_not_called() - - @pytest.mark.asyncio - async def test_cleanup_twice(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - await adapter.on_cleanup("room-1") # Should not raise - - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_multi_room(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - adapter._rooms["room-2"] = RoomContext(agent_id="agent-2") - - await adapter.on_cleanup("room-1") - - assert "room-1" not in adapter._rooms - assert "room-2" in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_without_client(self) -> None: - adapter = LettaAdapter() - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # No client — should not raise - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_before_started(self) -> None: - adapter = LettaAdapter() - # Completely uninitialized — no client, no rooms - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - -# ────────────────────────────────────────────────────────────────────── -# Instruction block update -# ────────────────────────────────────────────────────────────────────── - - -class TestInstructionBlockUpdate: - @pytest.mark.asyncio - async def test_updates_persona_block(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test system prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - await adapter._update_instruction_block("agent-1", "room-1") - - # Default config self-hosts the MCP server in per_room mode, so the - # enforcement carries the room id the tool schemas require. - mock_client.agents.blocks.update.assert_called_once_with( - "persona", - agent_id="agent-1", - value=default_enforcement(room_id="room-1") + "Test system prompt", - ) - - @pytest.mark.asyncio - async def test_falls_back_to_next_label(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - # First label fails, second succeeds - mock_client.agents.blocks.update.side_effect = [ - Exception("not found"), - None, - None, - ] - - await adapter._update_instruction_block("agent-1", "room-1") - - assert mock_client.agents.blocks.update.call_count == 2 - second_call = mock_client.agents.blocks.update.call_args_list[1] - assert second_call.args[0] == "custom_instructions" - - @pytest.mark.asyncio - async def test_creates_persona_when_all_labels_fail(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - mock_client.agents.blocks.update.side_effect = Exception("not found") - mock_block = MagicMock() - mock_block.id = "block-1" - mock_client.blocks.create.return_value = mock_block - - await adapter._update_instruction_block("agent-1", "room-1") - - mock_client.blocks.create.assert_called_once() - assert mock_client.blocks.create.call_args.kwargs["label"] == "persona" - mock_client.agents.blocks.attach.assert_called_once_with( - "block-1", - agent_id="agent-1", - ) - - -# ────────────────────────────────────────────────────────────────────── -# Rejoin context -# ────────────────────────────────────────────────────────────────────── - - -class TestRejoinContext: - @pytest.mark.asyncio - async def test_rejoin_injects_time_away(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - last_time = datetime.now(UTC) - timedelta(hours=2) - adapter._rooms["room-1"] = RoomContext( - agent_id="agent-1", - last_interaction=last_time, - summary="Discussed project plan", - ) - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'm back!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - content = call_kwargs["messages"][0]["content"] - assert "rejoined" in content - assert "2h" in content - assert "Discussed project plan" in content - - -# ────────────────────────────────────────────────────────────────────── -# Task events -# ────────────────────────────────────────────────────────────────────── - - -class TestTaskEvents: - @pytest.mark.asyncio - async def test_emits_task_event_on_agent_creation(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("new-agent") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 1 - metadata = task_events[0]["metadata"] - assert metadata["letta_agent_id"] == "new-agent" - assert metadata["letta_room_id"] == "room-1" - - @pytest.mark.asyncio - async def test_task_events_disabled(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=()) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("new-agent") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 0 - - @pytest.mark.asyncio - async def test_shared_mode_emits_conversation_id(self) -> None: - config = LettaAdapterConfig(mode="shared") - adapter = LettaAdapter(config=config) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("shared-agent") - mock_client.agents.create.return_value = mock_agent - mock_conv = make_mock_conversation("conv-123") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 1 - assert task_events[0]["metadata"]["letta_conversation_id"] == "conv-123" - - -# ────────────────────────────────────────────────────────────────────── -# Memory consolidation -# ────────────────────────────────────────────────────────────────────── - - -class TestMemoryConsolidation: - @pytest.mark.asyncio - async def test_consolidation_on_cleanup(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.messages.create.assert_called_once() - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - assert "Consolidate" in call_kwargs["messages"][0]["content"] - - @pytest.mark.asyncio - async def test_consolidation_failure_does_not_propagate(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.side_effect = Exception("API error") - - # Should not raise - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - -# ────────────────────────────────────────────────────────────────────── -# Static helpers -# ────────────────────────────────────────────────────────────────────── - - -class TestFormatTimeAgo: - def test_seconds(self) -> None: - dt = datetime.now(UTC) - timedelta(seconds=30) - assert "30s" == LettaAdapter._format_time_ago(dt) - - def test_minutes(self) -> None: - dt = datetime.now(UTC) - timedelta(minutes=5) - assert "5m" == LettaAdapter._format_time_ago(dt) - - def test_hours(self) -> None: - dt = datetime.now(UTC) - timedelta(hours=3) - assert "3h" == LettaAdapter._format_time_ago(dt) - - def test_one_hour(self) -> None: - dt = datetime.now(UTC) - timedelta(hours=1) - assert "1 hour" == LettaAdapter._format_time_ago(dt) - - def test_days(self) -> None: - dt = datetime.now(UTC) - timedelta(days=5) - assert "5d" == LettaAdapter._format_time_ago(dt) - - def test_one_day(self) -> None: - dt = datetime.now(UTC) - timedelta(days=1) - assert "1 day" == LettaAdapter._format_time_ago(dt) - - def test_naive_datetime_treated_as_utc(self) -> None: - dt = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=10) - result = LettaAdapter._format_time_ago(dt) - assert "10m" == result - - -class TestExtractSummary: - def test_short_text_kept_whole(self) -> None: - parts = ["Hello there. This is more text."] - assert "Hello there. This is more text." == LettaAdapter._extract_summary(parts) - - def test_truncates_on_word_boundary(self) -> None: - parts = ["alpha beta " * 30] - result = LettaAdapter._extract_summary(parts, max_length=50) - assert len(result) <= 53 # 50 + "..." - assert result.endswith("...") - assert not result.removesuffix("...").endswith("alph") # no mid-word cut - - def test_no_sentence_heuristics(self) -> None: - """Decimals must not be mistaken for sentence ends.""" - parts = ["pi is 3.14 and e is 2.71"] - assert "pi is 3.14 and e is 2.71" == LettaAdapter._extract_summary(parts) - - def test_empty(self) -> None: - assert "" == LettaAdapter._extract_summary([]) - - def test_multiple_parts_joined(self) -> None: - parts = ["First part.", "Second part."] - assert "First part. Second part." == LettaAdapter._extract_summary(parts) - - -# ────────────────────────────────────────────────────────────────────── -# Summary storage -# ────────────────────────────────────────────────────────────────────── - - -class TestSummaryStorage: - @pytest.mark.asyncio - async def test_summary_stored_after_turn(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("The weather is sunny. More details follow.") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - room_ctx = adapter._rooms["room-1"] - assert room_ctx.summary == "The weather is sunny. More details follow." - assert room_ctx.last_interaction is not None - - -# ────────────────────────────────────────────────────────────────────── -# Send-tool name resolution (derived from discovered MCP tools) -# ────────────────────────────────────────────────────────────────────── - - -class TestSendToolResolution: - def test_resolves_external_band_mcp_names(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event", "health_check"] - ) - assert adapter._mcp.send_message_tool == "create_agent_chat_message" - assert adapter._mcp.send_event_tool == "create_agent_chat_event" - assert adapter._mcp.silent_reporting_tools == { - "create_agent_chat_message", - "create_agent_chat_event", - } - - def test_prefers_band_names_when_both_present(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools( - ["band_send_message", "create_agent_chat_message", "band_send_event"] - ) - assert adapter._mcp.send_message_tool == "band_send_message" - assert adapter._mcp.send_event_tool == "band_send_event" - - def test_falls_back_to_band_names_when_none_discovered(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools(["some_unrelated_tool"]) - assert adapter._mcp.send_message_tool == "band_send_message" - assert adapter._mcp.send_event_tool == "band_send_event" - - def test_enforcement_prompt_uses_resolved_names(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Base prompt" - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event"] - ) - text = adapter._instruction_text("room-1") - assert "create_agent_chat_message" in text - assert "band_send_message" not in text - - @pytest.mark.asyncio - async def test_relay_detection_uses_resolved_name(self) -> None: - """A send via the resolved (external) tool name suppresses auto-relay.""" - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event"] - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("create_agent_chat_message"), - make_tool_return_message("create_agent_chat_message"), - make_assistant_message("Done!"), - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert len(tools.messages_sent) == 0 - - -# ────────────────────────────────────────────────────────────────────── -# Turn recording of out-of-process tool calls -# ────────────────────────────────────────────────────────────────────── - - -class TestExternalToolRecording: - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("mcp_mode", "status", "replied"), - [ - pytest.param("external", "success", True, id="external-success"), - pytest.param("external", "error", False, id="external-error"), - # Self-hosted calls run on the room's own tools, which record them. - pytest.param("self_host", "success", False, id="self-host"), - ], - ) - async def test_streamed_tool_return_records_only_for_external_band_mcp( - self, - mcp_mode: Literal["self_host", "external"], - status: Literal["success", "error"], - replied: bool, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(BandTool.SEND_MESSAGE), - make_tool_return_message(BandTool.SEND_MESSAGE, status=status), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert tools.turn.replied is replied - - @pytest.mark.asyncio - async def test_ungrouped_tool_return_is_matched_to_its_call(self) -> None: - """A server that sends no ``tool_returns`` puts the one result on the - message itself.""" - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(BandTool.SEND_MESSAGE), - ToolReturnMessage( - id="message-return", - date=datetime.now(UTC), - message_type="tool_return_message", - status="success", - tool_call_id=call_id(BandTool.SEND_MESSAGE), - tool_return="result", - ), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert tools.turn.replied - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("effect_tool", "replied"), - [ - pytest.param(BandTool.ADD_PARTICIPANT, False, id="act"), - pytest.param(BandTool.SEND_MESSAGE, True, id="reply"), - pytest.param(BandTool.NO_REPLY, True, id="decline"), - ], - ) - @pytest.mark.parametrize("reverse_calls", [False, True]) - async def test_parallel_success_completes_turn_independent_of_order( - self, - effect_tool: str, - replied: bool, - reverse_calls: bool, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - results: list[tuple[str, Literal["success", "error"]]] = [ - (BandTool.GET_PARTICIPANTS, "success"), - (effect_tool, "success"), - ] - if reverse_calls: - results.reverse() - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages(results, reverse_returns=False) - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert tools.turn.complete - assert tools.turn.replied is replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "effect_tool", - [BandTool.ADD_PARTICIPANT, BandTool.SEND_MESSAGE, BandTool.NO_REPLY], - ) - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_failed_effect_does_not_complete_turn( - self, effect_tool: str, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [ - (BandTool.GET_PARTICIPANTS, "success"), - (effect_tool, "error"), - ], - reverse_returns=reverse_returns, - ) - ) - tools = turn_tools() - - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_event(turn_input(tools)) - - assert not tools.turn.replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [MISSING_REPLY_FAILURE] - - @pytest.mark.asyncio - @pytest.mark.parametrize("effect_tool", [BandTool.SEND_MESSAGE, BandTool.NO_REPLY]) - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_failed_reply_does_not_suppress_fallback( - self, effect_tool: str, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [(effect_tool, "error"), (BandTool.GET_PARTICIPANTS, "success")], - reverse_returns=reverse_returns, - ), - make_assistant_message("Fallback reply"), - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert sent_contents(tools) == ["Fallback reply"] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_results_match_call_ids_not_positions( - self, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [ - (BandTool.SEND_MESSAGE, "error"), - (BandTool.ADD_PARTICIPANT, "success"), - ], - reverse_returns=reverse_returns, - ) - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert tools.turn.complete - assert not tools.turn.replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - -# ────────────────────────────────────────────────────────────────────── -# Auto-relay knob -# ────────────────────────────────────────────────────────────────────── - - -class TestAutoRelayDisabled: - @pytest.mark.asyncio - async def test_disabled_relay_drops_the_text_and_the_verdict_reports_it( - self, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, room_id=ROOM_ID, tool_calls=[], final_text="I'll help you!" - ) - - tools = turn_tools() - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_event(turn_input(tools)) - - assert tools.messages_sent == [] - assert failure_reports(tools) == [MISSING_REPLY_FAILURE] - - @pytest.mark.asyncio - async def test_disabled_relay_completes_a_turn_that_did_real_work(self) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id=ROOM_ID, - tool_calls=[(BandTool.CREATE_CHATROOM, {})], - final_text="Created it.", - ) - - tools = turn_tools() - await adapter.on_event(turn_input(tools)) - - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) - async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( - self, reply_tool: str, arguments: dict[str, Any] - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), capabilities={Capability.FILES} - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id="room-1", - tool_calls=[(reply_tool, arguments)], - final_text="Done!", - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert "Done!" not in sent_contents(tools) - assert not reported_failures(tools) - - -# ────────────────────────────────────────────────────────────────────── -# Cold-boot history seeding -# ────────────────────────────────────────────────────────────────────── - - -class TestColdBootSeeding: - @pytest.fixture - def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.server_id = "mcp-server-1" - adapter._mcp.tool_ids = [] - return adapter, mock_client - - @pytest.mark.asyncio - async def test_new_agent_in_room_with_history_is_seeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi again!") - ) - - history = LettaSessionState( - replay_messages=["[Alice]: The secret word is kumquat.", "[Bot]: Noted!"] - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(content="what was the secret word?"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ - "content" - ] - assert "joining an ongoing conversation" in content - assert "[Alice]: The secret word is kumquat." in content - assert "what was the secret word?" in content - # Seed is delivered exactly once - assert adapter._rooms["room-1"].pending_seed == [] - - @pytest.mark.asyncio - async def test_reported_turn_consumes_delivered_seed(self) -> None: - """A completed provider call consumes its seed before response handling.""" - adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.server_id = "mcp-server-1" - adapter._mcp.tool_ids = [] - room_ctx = RoomContext( - agent_id="agent-1", pending_seed=["[Alice]: Earlier context"] - ) - adapter._rooms["room-1"] = room_ctx - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("The response was not sent through the tool.") - ) - - await adapter.on_message( - make_platform_message(), - FakeAgentTools(), - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert room_ctx.pending_seed == [] - assert room_ctx.last_interaction is not None - - @pytest.mark.asyncio - async def test_failed_first_turn_preserves_pending_seed( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Cold-boot history must survive a failed first turn — clearing the - seed at compose time would drop replay context forever.""" - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.side_effect = TimeoutError("slow letta") - - history = LettaSessionState( - replay_messages=["[Alice]: The secret word is kumquat."] - ) - tools = FakeAgentTools() - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - make_platform_message(content="what was the secret word?"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert adapter._rooms["room-1"].pending_seed == [ - "[Alice]: The secret word is kumquat." - ] - - @pytest.mark.asyncio - async def test_per_room_does_not_resume_config_agent_id( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """config.agent_id is not a per-room resume target — each room without - persisted history must get its own Letta agent.""" - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy( - update={"agent_id": "shared-bootstrap-id"} - ) - mock_client.agents.create.return_value = make_mock_agent("room-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - await adapter.on_message( - make_platform_message(room_id="room-1"), - FakeAgentTools(), - LettaSessionState(), - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.agents.retrieve.assert_not_called() - mock_client.agents.create.assert_awaited_once() - - @pytest.mark.asyncio - async def test_second_message_is_not_reseeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - history = LettaSessionState(replay_messages=["[Alice]: earlier message"]) - tools = FakeAgentTools() - for _ in range(2): - await adapter.on_message( - make_platform_message(), - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - second_content = mock_client.agents.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: earlier message" not in second_content - - @pytest.mark.asyncio - async def test_resumed_agent_is_not_seeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Resume-by-id stays the fast path — the live agent already carries - its own context, so replaying history would duplicate it.""" - adapter, mock_client = adapter_with_client - mock_client.agents.retrieve.return_value = make_mock_agent("live-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Resumed!") - ) - - history = LettaSessionState( - agent_id="live-agent", - replay_messages=["[Alice]: earlier message"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ - "content" - ] - assert "[Alice]: earlier message" not in content - mock_client.agents.create.assert_not_called() - - -# ────────────────────────────────────────────────────────────────────── -# Agent-create options -# ────────────────────────────────────────────────────────────────────── - - -class TestAgentCreateOptions: - @pytest.mark.asyncio - async def test_embedding_passed_when_configured(self) -> None: - adapter = LettaAdapter( - config=LettaAdapterConfig(embedding="openai/text-embedding-3-small") - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-1") - - create_kwargs = mock_client.agents.create.call_args.kwargs - assert create_kwargs["embedding"] == "openai/text-embedding-3-small" - - @pytest.mark.asyncio - async def test_embedding_omitted_by_default(self) -> None: - """Letta Cloud picks its own default — None must not be sent.""" - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-1") - - assert "embedding" not in mock_client.agents.create.call_args.kwargs - - @pytest.mark.asyncio - async def test_per_room_persona_carries_room_id(self) -> None: - """Self-hosted MCP schemas require room_id per call; a per_room agent - gets it in its persona block.""" - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-42") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - persona = next(b for b in blocks if b["label"] == "persona") - assert "room-42" in persona["value"] - - @pytest.mark.asyncio - async def test_external_mode_persona_omits_room_id(self) -> None: - """External band-mcp tool schemas carry no room_id argument.""" - adapter = LettaAdapter( - config=LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-42") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - persona = next(b for b in blocks if b["label"] == "persona") - assert "room-42" not in persona["value"] - - -# ────────────────────────────────────────────────────────────────────── -# delete_agents_on_cleanup -# ────────────────────────────────────────────────────────────────────── - - -class TestDeleteAgentsOnCleanup: - @pytest.mark.asyncio - async def test_opt_in_deletes_agent_instead_of_consolidating(self) -> None: - adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_awaited_once_with("agent-1") - # No consolidation prompt when the agent is being deleted - mock_client.agents.messages.create.assert_not_called() - - @pytest.mark.asyncio - async def test_default_keeps_agent(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_not_called() - - @pytest.mark.asyncio - async def test_can_skip_consolidation_on_cleanup(self) -> None: - adapter = LettaAdapter( - config=LettaAdapterConfig(consolidate_memory_on_cleanup=False) - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_not_called() - mock_client.agents.messages.create.assert_not_called() - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_delete_failure_does_not_propagate(self) -> None: - adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) - mock_client = AsyncMock() - adapter._client = mock_client - mock_client.agents.delete.side_effect = Exception("gone already") - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") # should not raise - assert "room-1" not in adapter._rooms - - -class TestConfigEnvSourcing: - """provider_key env sourcing: LETTA_* names only, never bare vars.""" - - @pytest.fixture(autouse=True) - def clean_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - for var in ("API_KEY", "PROVIDER_KEY", "LETTA_API_KEY", "LETTA_PROVIDER_KEY"): - monkeypatch.delenv(var, raising=False) - - def test_bare_env_vars_never_populate_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """A foreign secret in API_KEY/PROVIDER_KEY must not reach Letta.""" - monkeypatch.setenv("API_KEY", "foreign-secret") - monkeypatch.setenv("PROVIDER_KEY", "foreign-secret") - - assert LettaAdapterConfig().provider_key is None - - def test_letta_api_key_env_populates_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_API_KEY", "cloud-key") - - assert LettaAdapterConfig().provider_key == "cloud-key" - - def test_prefixed_field_name_env_populates_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_PROVIDER_KEY", "prefixed-key") - - assert LettaAdapterConfig().provider_key == "prefixed-key" - - def test_explicit_kwarg_wins_over_env( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_API_KEY", "env-key") - - assert LettaAdapterConfig(provider_key="kwarg-key").provider_key == "kwarg-key" - - def test_prefixed_env_fills_a_field_and_an_explicit_value_beats_it( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_MODEL", "openai/env-model") - - assert LettaAdapterConfig().model == "openai/env-model" - assert LettaAdapterConfig(model="openai/explicit").model == "openai/explicit" diff --git a/tests/adapters/test_usage_mapping.py b/tests/adapters/test_usage_mapping.py deleted file mode 100644 index 9d837c76c..000000000 --- a/tests/adapters/test_usage_mapping.py +++ /dev/null @@ -1,270 +0,0 @@ -"""Unit tests for the Emit.USAGE per-adapter usage-mapping helpers. - -Each adapter maps its framework's response/usage object onto the shared -``TurnUsage``. These are pure static methods, so they're tested directly with -lightweight stand-ins (SimpleNamespace / dict) — no live LLM, no full adapter -construction. The emission path itself (gating on Emit.USAGE, empty-skip, -best-effort) lives on ``SimpleAdapter.emit_usage`` and is covered in -``test_anthropic_adapter.py``. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -import pytest -from band_rest.types.chat_message_metadata import ChatMessageMetadata - -from band.adapters.agno import AgnoAdapter -from band.adapters.claude_sdk import ClaudeSDKAdapter -from band.adapters.codex import CodexAdapter -from band.adapters.gemini import GeminiAdapter -from band.adapters.google_adk import GoogleADKAdapter -from band.adapters.letta import LettaAdapter -from band.core.types import USAGE_METADATA_KEY, TurnUsage, is_usage_event -from band.integrations.opencode import OpencodeMessageInfo - - -class TestTurnUsageConstructors: - """The shared from_object / from_mapping constructors the adapters build on.""" - - def test_from_object_reads_named_attrs(self): - src = SimpleNamespace(inp=100, out=20, cr=5, cw=3) - assert TurnUsage.from_object( - src, input="inp", output="out", cache_read="cr", cache_write="cw" - ) == TurnUsage(100, 20, 5, 3) - - def test_from_object_none_source_is_empty(self): - assert TurnUsage.from_object(None, input="i", output="o") == TurnUsage() - - def test_from_object_omitted_cache_keys_stay_zero(self): - src = SimpleNamespace(i=100, o=20, cache_read_tokens=999) - # cache_read/cache_write not passed → not read even if present on src - assert TurnUsage.from_object(src, input="i", output="o") == TurnUsage(100, 20) - - def test_from_mapping_reads_named_keys(self): - data = {"i": 100, "o": 20, "cr": 5, "cw": 3} - assert TurnUsage.from_mapping( - data, input="i", output="o", cache_read="cr", cache_write="cw" - ) == TurnUsage(100, 20, 5, 3) - - @pytest.mark.parametrize("bad", [None, "not a mapping", 42, ["list"]]) - def test_from_mapping_non_mapping_is_empty(self, bad): - assert TurnUsage.from_mapping(bad, input="i", output="o") == TurnUsage() - - def test_missing_and_non_int_fields_default_to_zero(self): - data = {"i": "oops", "o": None} # non-int / missing - assert TurnUsage.from_mapping(data, input="i", output="o") == TurnUsage() - - def test_fields_are_raw_no_cache_folding(self): - """Per the convention, values pass through raw — cache is never folded - into input regardless of provider.""" - data = {"i": 100, "o": 20, "cr": 5, "cw": 3} - assert TurnUsage.from_mapping( - data, input="i", output="o", cache_read="cr", cache_write="cw" - ) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - -class TestIsUsageEvent: - """The shared discriminator task-event consumers use to skip usage records.""" - - def test_true_when_band_usage_present(self): - assert is_usage_event({USAGE_METADATA_KEY: {"input_tokens": 1}}) is True - - def test_false_for_lifecycle_task_or_non_mapping(self): - assert is_usage_event({"codex_thread_id": "x"}) is False - assert is_usage_event(None) is False - assert is_usage_event("nope") is False - - def test_true_and_false_for_chat_message_metadata_model(self): - """band-client-rest 0.0.38 types ``ChatMessage.metadata`` as a model, - not a dict — the discriminator must dump it, not require ``Mapping``.""" - assert ( - is_usage_event(ChatMessageMetadata(band_usage={"input_tokens": 1})) is True - ) - assert is_usage_event(ChatMessageMetadata(claude_sdk_session_id="s")) is False - - -class TestClaudeSDKUsageMapping: - def test_maps_usage_dict(self): - """ResultMessage.usage (raw API dict) maps onto TurnUsage raw — Claude's - input_tokens excludes cache, reported separately in the cache fields.""" - result = SimpleNamespace( - usage={ - "input_tokens": 100, - "output_tokens": 20, - "cache_read_input_tokens": 5, - "cache_creation_input_tokens": 3, - } - ) - assert ClaudeSDKAdapter._usage_from_result(result) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_usage_is_empty(self): - """A ResultMessage without a usage dict yields empty usage.""" - assert ( - ClaudeSDKAdapter._usage_from_result(SimpleNamespace(usage=None)) - == TurnUsage() - ) - - -class TestAgnoUsageMapping: - def test_maps_run_metrics(self): - """RunOutput.metrics (aggregated) maps onto TurnUsage; names line up.""" - metrics = SimpleNamespace( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - response = SimpleNamespace(metrics=metrics) - assert AgnoAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_metrics_is_empty(self): - """A RunOutput without metrics yields empty usage.""" - assert ( - AgnoAdapter._usage_from_response(SimpleNamespace(metrics=None)) - == TurnUsage() - ) - - -class TestGoogleADKUsageMapping: - def test_maps_event_usage_metadata(self): - """ADK event.usage_metadata maps onto TurnUsage (no cache-write dim).""" - event = SimpleNamespace( - usage_metadata=SimpleNamespace( - prompt_token_count=100, - candidates_token_count=20, - cached_content_token_count=5, - ) - ) - assert GoogleADKAdapter._usage_from_event(event) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=0, - ) - - def test_event_without_usage_is_empty(self): - """A non-model event (no usage_metadata) contributes empty usage.""" - assert ( - GoogleADKAdapter._usage_from_event(SimpleNamespace(usage_metadata=None)) - == TurnUsage() - ) - - -class TestGeminiUsageMapping: - def test_maps_response_usage_metadata(self): - """GenerateContentResponse.usage_metadata maps onto TurnUsage.""" - response = SimpleNamespace( - usage_metadata=SimpleNamespace( - prompt_token_count=100, - candidates_token_count=20, - cached_content_token_count=5, - ) - ) - assert GeminiAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=0, - ) - - def test_response_without_usage_is_empty(self): - """A response without usage_metadata yields empty usage.""" - assert ( - GeminiAdapter._usage_from_response(SimpleNamespace(usage_metadata=None)) - == TurnUsage() - ) - - -class TestLettaUsageMapping: - def test_maps_response_usage(self): - """LettaResponse.usage (per_room mode) maps onto TurnUsage.""" - response = SimpleNamespace( - usage=SimpleNamespace( - prompt_tokens=100, - completion_tokens=20, - cached_input_tokens=5, - cache_write_tokens=3, - ) - ) - assert LettaAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_response_without_usage_is_empty(self): - """A response with no usage (e.g. shared-mode stream) yields empty usage.""" - assert ( - LettaAdapter._usage_from_response(SimpleNamespace(usage=None)) - == TurnUsage() - ) - - -class TestOpencodeUsageMapping: - def test_maps_info_tokens_with_nested_cache(self): - """OpenCode assistant info.tokens (nested cache) maps onto TurnUsage. - - OpenCode reports reasoning disjointly from output (its own total is - input + output + reasoning + cache), so reasoning folds into - output_tokens: 20 + 7 = 27. - """ - info = OpencodeMessageInfo.model_validate( - { - "tokens": { - "input": 100, - "output": 20, - "reasoning": 7, - "cache": {"read": 5, "write": 3}, - } - } - ) - assert info.tokens is not None - assert info.tokens.to_turn_usage() == TurnUsage( - input_tokens=100, - output_tokens=27, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_or_malformed_tokens_is_empty(self): - """No tokens (or a non-dict tokens) parses as absent usage.""" - assert OpencodeMessageInfo.model_validate({}).tokens is None - assert OpencodeMessageInfo.model_validate({"tokens": "nope"}).tokens is None - - -class TestCodexUsageMapping: - def test_maps_per_turn_deltas(self): - """Codex's per-turn token deltas (not thread cumulatives) map to TurnUsage. - - Codex reports reasoning disjointly from output (its own total is - input + output + reasoning), so reasoning folds into output_tokens: - 42 + 8 = 50. - """ - usage = SimpleNamespace( - turn_input_tokens=130, turn_output_tokens=42, turn_reasoning_tokens=8 - ) - assert CodexAdapter._turn_usage(usage) == TurnUsage( - input_tokens=130, output_tokens=50 - ) - - def test_none_usage_is_empty(self): - """No usage object for the thread yields empty usage (no emit).""" - assert CodexAdapter._turn_usage(None) == TurnUsage() diff --git a/tests/adapters/usagemapping/__init__.py b/tests/adapters/usagemapping/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/usagemapping/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/usagemapping/test_agnousagemapping.py b/tests/adapters/usagemapping/test_agnousagemapping.py new file mode 100644 index 000000000..cdeefe9df --- /dev/null +++ b/tests/adapters/usagemapping/test_agnousagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.agno import AgnoAdapter +from band.core.types import TurnUsage + + +class TestAgnoUsageMapping: + def test_maps_run_metrics(self): + """RunOutput.metrics (aggregated) maps onto TurnUsage; names line up.""" + metrics = SimpleNamespace( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + response = SimpleNamespace(metrics=metrics) + assert AgnoAdapter._usage_from_response(response) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_metrics_is_empty(self): + """A RunOutput without metrics yields empty usage.""" + assert ( + AgnoAdapter._usage_from_response(SimpleNamespace(metrics=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_claudesdkusagemapping.py b/tests/adapters/usagemapping/test_claudesdkusagemapping.py new file mode 100644 index 000000000..48ffc4466 --- /dev/null +++ b/tests/adapters/usagemapping/test_claudesdkusagemapping.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.core.types import TurnUsage + + +class TestClaudeSDKUsageMapping: + def test_maps_usage_dict(self): + """ResultMessage.usage (raw API dict) maps onto TurnUsage raw — Claude's + input_tokens excludes cache, reported separately in the cache fields.""" + result = SimpleNamespace( + usage={ + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 3, + } + ) + assert ClaudeSDKAdapter._usage_from_result(result) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_usage_is_empty(self): + """A ResultMessage without a usage dict yields empty usage.""" + assert ( + ClaudeSDKAdapter._usage_from_result(SimpleNamespace(usage=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_codexusagemapping.py b/tests/adapters/usagemapping/test_codexusagemapping.py new file mode 100644 index 000000000..1d0d163e7 --- /dev/null +++ b/tests/adapters/usagemapping/test_codexusagemapping.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.codex import CodexAdapter +from band.core.types import TurnUsage + + +class TestCodexUsageMapping: + def test_maps_per_turn_deltas(self): + """Codex's per-turn token deltas (not thread cumulatives) map to TurnUsage. + + Codex reports reasoning disjointly from output (its own total is + input + output + reasoning), so reasoning folds into output_tokens: + 42 + 8 = 50. + """ + usage = SimpleNamespace( + turn_input_tokens=130, turn_output_tokens=42, turn_reasoning_tokens=8 + ) + assert CodexAdapter._turn_usage(usage) == TurnUsage( + input_tokens=130, output_tokens=50 + ) + + def test_none_usage_is_empty(self): + """No usage object for the thread yields empty usage (no emit).""" + assert CodexAdapter._turn_usage(None) == TurnUsage() diff --git a/tests/adapters/usagemapping/test_geminiusagemapping.py b/tests/adapters/usagemapping/test_geminiusagemapping.py new file mode 100644 index 000000000..ad6daf1ed --- /dev/null +++ b/tests/adapters/usagemapping/test_geminiusagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.gemini import GeminiAdapter +from band.core.types import TurnUsage + + +class TestGeminiUsageMapping: + def test_maps_response_usage_metadata(self): + """GenerateContentResponse.usage_metadata maps onto TurnUsage.""" + response = SimpleNamespace( + usage_metadata=SimpleNamespace( + prompt_token_count=100, + candidates_token_count=20, + cached_content_token_count=5, + ) + ) + assert GeminiAdapter._usage_from_response(response) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=0, + ) + + def test_response_without_usage_is_empty(self): + """A response without usage_metadata yields empty usage.""" + assert ( + GeminiAdapter._usage_from_response(SimpleNamespace(usage_metadata=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_googleadkusagemapping.py b/tests/adapters/usagemapping/test_googleadkusagemapping.py new file mode 100644 index 000000000..76e8367a2 --- /dev/null +++ b/tests/adapters/usagemapping/test_googleadkusagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.google_adk import GoogleADKAdapter +from band.core.types import TurnUsage + + +class TestGoogleADKUsageMapping: + def test_maps_event_usage_metadata(self): + """ADK event.usage_metadata maps onto TurnUsage (no cache-write dim).""" + event = SimpleNamespace( + usage_metadata=SimpleNamespace( + prompt_token_count=100, + candidates_token_count=20, + cached_content_token_count=5, + ) + ) + assert GoogleADKAdapter._usage_from_event(event) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=0, + ) + + def test_event_without_usage_is_empty(self): + """A non-model event (no usage_metadata) contributes empty usage.""" + assert ( + GoogleADKAdapter._usage_from_event(SimpleNamespace(usage_metadata=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_isusageevent.py b/tests/adapters/usagemapping/test_isusageevent.py new file mode 100644 index 000000000..4f67775bd --- /dev/null +++ b/tests/adapters/usagemapping/test_isusageevent.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from band_rest.types.chat_message_metadata import ChatMessageMetadata + +from band.core.types import USAGE_METADATA_KEY, is_usage_event + + +class TestIsUsageEvent: + """The shared discriminator task-event consumers use to skip usage records.""" + + def test_true_when_band_usage_present(self): + assert is_usage_event({USAGE_METADATA_KEY: {"input_tokens": 1}}) is True + + def test_false_for_lifecycle_task_or_non_mapping(self): + assert is_usage_event({"codex_thread_id": "x"}) is False + assert is_usage_event(None) is False + assert is_usage_event("nope") is False + + def test_true_and_false_for_chat_message_metadata_model(self): + """band-client-rest 0.0.38 types ``ChatMessage.metadata`` as a model, + not a dict — the discriminator must dump it, not require ``Mapping``.""" + assert ( + is_usage_event(ChatMessageMetadata(band_usage={"input_tokens": 1})) is True + ) + assert is_usage_event(ChatMessageMetadata(claude_sdk_session_id="s")) is False diff --git a/tests/adapters/usagemapping/test_opencodeusagemapping.py b/tests/adapters/usagemapping/test_opencodeusagemapping.py new file mode 100644 index 000000000..99b8849b3 --- /dev/null +++ b/tests/adapters/usagemapping/test_opencodeusagemapping.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from band.core.types import TurnUsage +from band.integrations.opencode import OpencodeMessageInfo + + +class TestOpencodeUsageMapping: + def test_maps_info_tokens_with_nested_cache(self): + """OpenCode assistant info.tokens (nested cache) maps onto TurnUsage. + + OpenCode reports reasoning disjointly from output (its own total is + input + output + reasoning + cache), so reasoning folds into + output_tokens: 20 + 7 = 27. + """ + info = OpencodeMessageInfo.model_validate( + { + "tokens": { + "input": 100, + "output": 20, + "reasoning": 7, + "cache": {"read": 5, "write": 3}, + } + } + ) + assert info.tokens is not None + assert info.tokens.to_turn_usage() == TurnUsage( + input_tokens=100, + output_tokens=27, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_or_malformed_tokens_is_empty(self): + """No tokens (or a non-dict tokens) parses as absent usage.""" + assert OpencodeMessageInfo.model_validate({}).tokens is None + assert OpencodeMessageInfo.model_validate({"tokens": "nope"}).tokens is None diff --git a/tests/adapters/usagemapping/test_turnusageconstructors.py b/tests/adapters/usagemapping/test_turnusageconstructors.py new file mode 100644 index 000000000..c47b5a22a --- /dev/null +++ b/tests/adapters/usagemapping/test_turnusageconstructors.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from band.core.types import TurnUsage + + +class TestTurnUsageConstructors: + """The shared from_object / from_mapping constructors the adapters build on.""" + + def test_from_object_reads_named_attrs(self): + src = SimpleNamespace(inp=100, out=20, cr=5, cw=3) + assert TurnUsage.from_object( + src, input="inp", output="out", cache_read="cr", cache_write="cw" + ) == TurnUsage(100, 20, 5, 3) + + def test_from_object_none_source_is_empty(self): + assert TurnUsage.from_object(None, input="i", output="o") == TurnUsage() + + def test_from_object_omitted_cache_keys_stay_zero(self): + src = SimpleNamespace(i=100, o=20, cache_read_tokens=999) + # cache_read/cache_write not passed → not read even if present on src + assert TurnUsage.from_object(src, input="i", output="o") == TurnUsage(100, 20) + + def test_from_mapping_reads_named_keys(self): + data = {"i": 100, "o": 20, "cr": 5, "cw": 3} + assert TurnUsage.from_mapping( + data, input="i", output="o", cache_read="cr", cache_write="cw" + ) == TurnUsage(100, 20, 5, 3) + + @pytest.mark.parametrize("bad", [None, "not a mapping", 42, ["list"]]) + def test_from_mapping_non_mapping_is_empty(self, bad): + assert TurnUsage.from_mapping(bad, input="i", output="o") == TurnUsage() + + def test_missing_and_non_int_fields_default_to_zero(self): + data = {"i": "oops", "o": None} # non-int / missing + assert TurnUsage.from_mapping(data, input="i", output="o") == TurnUsage() + + def test_fields_are_raw_no_cache_folding(self): + """Per the convention, values pass through raw — cache is never folded + into input regardless of provider.""" + data = {"i": 100, "o": 20, "cr": 5, "cw": 3} + assert TurnUsage.from_mapping( + data, input="i", output="o", cache_read="cr", cache_write="cw" + ) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) diff --git a/tests/agent/test_startup_race_condition.py b/tests/agent/test_startup_race_condition.py index 36e01ff91..a493ea408 100644 --- a/tests/agent/test_startup_race_condition.py +++ b/tests/agent/test_startup_race_condition.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -14,7 +14,8 @@ ) from band.core.simple_adapter import SimpleAdapter from band.platform.event import MessageEvent -from band.runtime.context import ConversationContext +from band.runtime.execution import ExecutionContext +from band.runtime.sessionconfig import SessionConfig from band.testing.platform import platform_connection_stub @@ -26,7 +27,9 @@ class TestStartupRaceCondition: """ @pytest.mark.asyncio - async def test_adapter_on_started_before_first_message(self): + async def test_adapter_on_started_before_first_message( + self, mock_rest_client: MagicMock + ) -> None: """System prompt must be set before any message processing.""" # Track the order of calls @@ -93,20 +96,13 @@ async def start( self._on_execute = on_execute # Create a proper MessageEvent that passes through preprocessor - mock_ctx = MagicMock() - mock_ctx.is_llm_initialized = False - mock_ctx.mark_llm_initialized = MagicMock() - mock_ctx.get_context = AsyncMock( - return_value=ConversationContext( - room_id="room-123", - messages=[], - participants=[], - hydrated_at=datetime.now(UTC), - ) + mock_ctx = ExecutionContext( + room_id="room-123", + link=MagicMock(rest=mock_rest_client), + on_execute=on_execute, + agent_id="agent-123", + config=SessionConfig(enable_context_hydration=False), ) - mock_ctx.participants = [] - mock_ctx.participants_changed = MagicMock(return_value=False) - mock_ctx.room_id = "room-123" # Create a proper MessageEvent (not a MagicMock) message_event = MessageEvent( @@ -139,8 +135,4 @@ async def run_forever(self, *, install_signal_handlers: bool = True): # This will fail if on_message is called before on_started await agent.start() - # Verify on_started was called first - assert len(call_order) >= 1, "No calls were made" - assert call_order[0] == "on_started", ( - f"Expected on_started to be called first, but got: {call_order}" - ) + assert call_order == ["on_started", "on_message"] diff --git a/tests/core/test_harness_probe.py b/tests/core/test_harness_probe.py new file mode 100644 index 000000000..d2efa85be --- /dev/null +++ b/tests/core/test_harness_probe.py @@ -0,0 +1,216 @@ +"""Host probes keep resource custody through failed or interrupted cleanup.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import pytest + +from band.core.exceptions import BandConfigError, CleanupIncompleteError +from band.core.simple_adapter import SimpleAdapter +from band.core.types import PlatformMessage +from tests.conftest import eager_tasks, requires_eager_tasks + + +class ProbeAdapter(SimpleAdapter[None]): + async def on_message( + self, + msg: PlatformMessage, + tools: Any, + history: None, + participants_msg: str | None, + contacts_msg: str | None, + *, + is_session_bootstrap: bool, + room_id: str, + ) -> None: + await tools.no_reply() + + +async def test_an_unsupported_preflight_cannot_report_success() -> None: + result = await ProbeAdapter().preflight() + assert not result.ok + assert result.reason is not None + assert result.remedy is not None + + +async def test_an_unsupported_catalog_is_not_an_empty_success() -> None: + with pytest.raises(BandConfigError): + await ProbeAdapter().probe_models() + + +async def test_failed_close_keeps_the_resource_for_a_cleanup_retry( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "owned-resource" + refuse_close = True + + async def close() -> None: + if refuse_close: + raise PermissionError("resource still in use") + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + resource.write_text("private configuration", encoding="utf-8") + return "catalog" + + with pytest.raises(CleanupIncompleteError): + await adapter._harness_probe.run(operation, timeout_s=0.1) + assert resource.read_text(encoding="utf-8") == "private configuration" + refuse_close = False + await adapter.cleanup_all() + assert not resource.exists() + + async def next_operation() -> str: + return "next catalog" + + assert ( + await adapter._harness_probe.run(next_operation, timeout_s=0.1) + == "next catalog" + ) + + +async def test_timeout_retains_a_start_that_settles_after_cancellation( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "late-resource" + entered = asyncio.Event() + settle = asyncio.Event() + + async def close() -> None: + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + entered.set() + try: + await settle.wait() + except asyncio.CancelledError: + await settle.wait() + resource.write_text("late configuration", encoding="utf-8") + return "late catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=0.01)) + try: + await asyncio.wait_for(entered.wait(), 1) + with pytest.raises(CleanupIncompleteError): + await running + settle.set() + await adapter.cleanup_all() + assert not resource.exists() + finally: + settle.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() + + +async def test_caller_cancellation_does_not_abandon_late_startup( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "cancelled-resource" + entered = asyncio.Event() + settle = asyncio.Event() + + async def close() -> None: + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + entered.set() + try: + await settle.wait() + except asyncio.CancelledError: + await settle.wait() + resource.write_text("configuration", encoding="utf-8") + return "catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=0.01)) + try: + await asyncio.wait_for(entered.wait(), 1) + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + settle.set() + await adapter.cleanup_all() + assert not resource.exists() + finally: + settle.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() + + +async def test_cancelled_cleanup_reports_no_raw_error_but_retains_retry( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "failed-resource" + secret = "provider-token-in-transport-error" + entered = asyncio.Event() + release = asyncio.Event() + failed = asyncio.Event() + refuse_close = True + + async def close() -> None: + entered.set() + await release.wait() + if refuse_close: + failed.set() + raise PermissionError(secret) + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + resource.write_text("owned configuration", encoding="utf-8") + return "catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=1)) + try: + await asyncio.wait_for(entered.wait(), 1) + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + release.set() + await asyncio.wait_for(failed.wait(), 1) + await asyncio.sleep(0) + assert secret not in caplog.text + assert resource.exists() + with pytest.raises(CleanupIncompleteError): + await adapter.cleanup_all() + refuse_close = False + await adapter.cleanup_all() + assert not resource.exists() + finally: + refuse_close = False + release.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() + + +@requires_eager_tasks +async def test_eager_execution_enrolls_resources_before_operation_start( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "owned" + closed = tmp_path / "closed" + + async def close() -> None: + resource.rename(closed) + + async def operation() -> None: + adapter._harness_probe.own(close) + resource.write_text("owned payload", encoding="utf-8") + + try: + with eager_tasks(): + await adapter._harness_probe.run(operation, timeout_s=1) + assert closed.read_text(encoding="utf-8") == "owned payload" + assert not resource.exists() + finally: + await adapter.cleanup_all() diff --git a/tests/e2e/baseline/expected-cells.json b/tests/e2e/baseline/expected-cells.json index cca3b241e..bbc3654b8 100644 --- a/tests/e2e/baseline/expected-cells.json +++ b/tests/e2e/baseline/expected-cells.json @@ -617,6 +617,24 @@ "status": "pass", "reason": null }, + { + "test": "tests/e2e/baseline/smoke/activity/test_native_reporting.py::TestNativeActivity::test_native_shell_reports_privately_without_narration", + "adapter": "claude_sdk", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/activity/test_sdk_reporting.py::TestSDKActivity::test_held_sdk_tool_reports_privately_without_narration", + "adapter": "anthropic", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, { "test": "tests/e2e/baseline/smoke/adapters/copilotsdk/test_handler.py::TestAskUserHandler::test_copilot_ask_user_handler_round_trips_to_room_reply", "adapter": "suite", diff --git a/tests/e2e/baseline/smoke/activity/__init__.py b/tests/e2e/baseline/smoke/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/e2e/baseline/smoke/activity/conftest.py b/tests/e2e/baseline/smoke/activity/conftest.py new file mode 100644 index 000000000..2af5bb2f0 --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/conftest.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest + +from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.smoke.activity.kit import HeldLookup, HeldNativePeer + + +@pytest.fixture +async def held_lookup(baseline_settings: BaselineSettings) -> AsyncIterator[HeldLookup]: + held = HeldLookup(baseline_settings.e2e_timeout) + try: + yield held + finally: + held.release.set() + + +@pytest.fixture +async def native_peer( + baseline_settings: BaselineSettings, +) -> AsyncIterator[HeldNativePeer]: + async with HeldNativePeer(baseline_settings.e2e_timeout).running() as peer: + yield peer diff --git a/tests/e2e/baseline/smoke/activity/kit.py b/tests/e2e/baseline/smoke/activity/kit.py new file mode 100644 index 000000000..96f98f69f --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/kit.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import asyncio +import os +import shlex +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager, suppress +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from pydantic import BaseModel + +from band.adapters.claude_sdk import ( + ClaudeApprovalOptions, + ClaudeSDKAdapter, + ClaudeSDKAdapterConfig, +) +from band.core.types import MessageType +from tests.e2e.baseline.smoke.samples.sample_tools import LookupInput +from tests.e2e.baseline.toolkit.capture import ReplyCapture +from tests.e2e.baseline.toolkit.provisioning import ( + AdapterCell, + ProvisionedAgent, + running_provisioned_agent, +) +from tests.e2e.baseline.toolkit.tools import ToolSpec + + +class HeldLookup: + def __init__(self, timeout_s: float) -> None: + self.timeout_s = timeout_s + self.argument_sentinel = f"argument-{uuid4().hex}" + self.result_sentinel = f"result-{uuid4().hex}" + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.arguments: list[LookupInput] = [] + self.spec = ToolSpec(LookupInput, self.lookup) + + async def lookup(self, args: BaseModel) -> str: + assert isinstance(args, LookupInput) + self.arguments.append(args) + self.entered.set() + async with asyncio.timeout(self.timeout_s): + await self.release.wait() + return self.result_sentinel + + +class HeldNativePeer: + def __init__(self, timeout_s: float) -> None: + self.timeout_s = timeout_s + self.argument_sentinel = f"argument-{uuid4().hex}" + self.result_sentinel = f"result-{uuid4().hex}" + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.requests: list[str] = [] + self.writers: set[asyncio.StreamWriter] = set() + self.handlers: set[asyncio.Task[object]] = set() + self.url = "" + + async def handle( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + task = asyncio.current_task() + assert task is not None + self.handlers.add(task) + self.writers.add(writer) + try: + async with asyncio.timeout(self.timeout_s): + request = await reader.readuntil(b"\r\n\r\n") + self.requests.append(request.split(b"\r\n", 1)[0].decode()) + self.entered.set() + await self.release.wait() + body = self.result_sentinel.encode() + writer.write( + b"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + await writer.drain() + except (TimeoutError, ConnectionError, asyncio.IncompleteReadError): + self.entered.set() + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + self.writers.discard(writer) + self.handlers.discard(task) + + @property + def command(self) -> str: + script = f"import sys, urllib.request; sys.stdout.write(urllib.request.urlopen({self.url!r}, timeout={self.timeout_s!r}).read().decode())" + if os.name == "nt": + executable = sys.executable.replace("'", "''") + escaped_script = script.replace("'", "''") + return f"& '{executable}' -c '{escaped_script}'" + return f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}" + + @asynccontextmanager + async def running(self) -> AsyncIterator[HeldNativePeer]: + server = await asyncio.start_server(self.handle, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + self.url = f"http://127.0.0.1:{port}/{self.argument_sentinel}" + try: + yield self + finally: + self.release.set() + server.close() + await server.wait_closed() + for writer in tuple(self.writers): + writer.close() + await asyncio.gather(*self.handlers, return_exceptions=True) + + +@asynccontextmanager +async def native_activity_agent(cell: AdapterCell) -> AsyncIterator[ProvisionedAgent]: + with TemporaryDirectory(prefix="band-activity-native-") as directory: + adapter = ClaudeSDKAdapter( + ClaudeSDKAdapterConfig( + model=cell.settings.llm_models.anthropic_model, + cwd=str(Path(directory).resolve()), + approvals=ClaudeApprovalOptions( + mode="auto_accept", text_notifications=False + ), + custom_section="Use your native Bash or PowerShell tool to execute the user's exact test command. Report its actual output using band_send_message. Do not substitute a different tool or narrate the command.", + ), + emit=(), + ) + async with running_provisioned_agent( + adapter, cell.resources, label="activity-native" + ) as agent: + yield agent + + +async def assert_no_tool_narration(capture: ReplyCapture, agent_id: str) -> None: + for kind in (MessageType.TOOL_CALL, MessageType.TOOL_RESULT): + events = await capture.events(kind, sender_id=agent_id) + events.assert_none() diff --git a/tests/e2e/baseline/smoke/activity/test_native_reporting.py b/tests/e2e/baseline/smoke/activity/test_native_reporting.py new file mode 100644 index 000000000..e95ef9c7d --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/test_native_reporting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from tests.baseline.adapter import Adapter +from tests.e2e.baseline.agents import per_adapter +from tests.e2e.baseline.smoke.activity.kit import ( + HeldNativePeer, + assert_no_tool_narration, + native_activity_agent, +) +from tests.e2e.baseline.smoke.samples.approvals import CLAUDE_SHELL_TOOLS +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import AdapterCell +from tests.e2e.baseline.toolkit.user_ops import UserOps + + +class TestNativeActivity: + @per_adapter(Adapter.CLAUDE_SDK) + @pytest.mark.asyncio(loop_scope="session") + @pytest.mark.timeout(extra=120) + async def test_native_shell_reports_privately_without_narration( + self, + cell: AdapterCell, + native_peer: HeldNativePeer, + user_ops: UserOps, + reply_capture: CaptureFactory, + ) -> None: + async with native_activity_agent(cell) as agent: + try: + room_id = await cell.resources.provision_room( + title="activity-native", participants=[agent.id] + ) + async with reply_capture(room_id, observe_activity=True) as capture: + mid = await user_ops.send_message( + room_id, + f"Execute this exact command once through native Bash/PowerShell, then send me its actual output: {native_peer.command}", + mention_id=agent.id, + mention_name=agent.name, + ) + async with asyncio.timeout(cell.settings.e2e_timeout): + await native_peer.entered.wait() + await capture.wait_for_tool_activity(agent.id, CLAUDE_SHELL_TOOLS) + mark = capture.activities.snapshot() + native_peer.release.set() + replies = await capture.wait_for_reply(mid, agent.id) + await capture.wait_for_activity_stop(agent.id, since=mark) + assert ( + f"GET /{native_peer.argument_sentinel} HTTP/1.1" + in native_peer.requests + ) + replies.assert_contains_any([native_peer.result_sentinel]) + capture.activities.assert_names_only( + native_peer.argument_sentinel, native_peer.result_sentinel + ) + await assert_no_tool_narration(capture, agent.id) + finally: + native_peer.release.set() diff --git a/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py b/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py new file mode 100644 index 000000000..23aa76cba --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.types import AdapterFeatures +from tests.baseline.adapter import Adapter +from tests.e2e.baseline.agents import per_adapter +from tests.e2e.baseline.smoke.activity.kit import HeldLookup, assert_no_tool_narration +from tests.e2e.baseline.smoke.samples.sample_tools import ( + LOOKUP_PROMPT, + lookup_code_instruction, +) +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import AdapterCell +from tests.e2e.baseline.toolkit.user_ops import UserOps + + +class TestSDKActivity: + @per_adapter( + Adapter.ANTHROPIC, prompt=LOOKUP_PROMPT, features=AdapterFeatures(emit=()) + ) + @pytest.mark.asyncio(loop_scope="session") + @pytest.mark.timeout(extra=120) + async def test_held_sdk_tool_reports_privately_without_narration( + self, + cell: AdapterCell, + held_lookup: HeldLookup, + user_ops: UserOps, + reply_capture: CaptureFactory, + ) -> None: + async with cell.running(tools=[held_lookup.spec]) as agent: + try: + room_id = await cell.resources.provision_room( + title="activity-sdk", participants=[agent.id] + ) + async with reply_capture(room_id, observe_activity=True) as capture: + mid = await user_ops.send_message( + room_id, + f"{lookup_code_instruction('alpha')}. Pass note='{held_lookup.argument_sentinel}' exactly; relay the tool's returned value.", + mention_id=agent.id, + mention_name=agent.name, + ) + async with asyncio.timeout(cell.settings.e2e_timeout): + await held_lookup.entered.wait() + await capture.wait_for_tool_activity( + agent.id, {held_lookup.spec.name} + ) + mark = capture.activities.snapshot() + held_lookup.release.set() + replies = await capture.wait_for_reply(mid, agent.id) + await capture.wait_for_activity_stop(agent.id, since=mark) + replies.assert_contains_any([held_lookup.result_sentinel]) + assert ( + held_lookup.arguments[0].note == held_lookup.argument_sentinel + ) + capture.activities.assert_names_only( + held_lookup.argument_sentinel, held_lookup.result_sentinel + ) + await assert_no_tool_narration(capture, agent.id) + finally: + held_lookup.release.set() diff --git a/tests/e2e/baseline/toolkit/activity.py b/tests/e2e/baseline/toolkit/activity.py new file mode 100644 index 000000000..4c3fc872b --- /dev/null +++ b/tests/e2e/baseline/toolkit/activity.py @@ -0,0 +1,87 @@ +"""Opt-in test observation of ephemeral room activity, independent of SDK receiving.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator, Awaitable, Callable, Collection +from contextlib import asynccontextmanager +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +from phoenix_channels_python_client.phx_messages import PHXMessage + +from band.client.streaming import WebSocketClient +from tests.e2e.baseline.toolkit.ws import TrackingWebSocketClient + + +class ActivityEvent(StrEnum): + STARTED = "agent_activity_started" + CHANGED = "agent_activity_changed" + STOPPED = "agent_activity_stopped" + + +def activity_topic(room_id: str) -> str: + return f"room_activity:{room_id}" + + +@dataclass(frozen=True) +class ActivityFrame: + event: ActivityEvent + payload: dict[str, Any] + + @property + def agent_id(self) -> str: + return str(self.payload.get("agent_id", "")) + + @property + def detail(self) -> dict[str, Any]: + detail = self.payload.get("activity") + return detail if isinstance(detail, dict) else {} + + +class Activities(list[ActivityFrame]): + def has_tool(self, agent_id: str, names: Collection[str]) -> bool: + return any( + frame.agent_id == agent_id + and frame.detail.get("kind") == "tool_call" + and frame.detail.get("name") in names + for frame in self + ) + + def snapshot(self) -> int: + return len(self) + + def has_stopped(self, agent_id: str, *, since: int = 0) -> bool: + return any( + frame.agent_id == agent_id and frame.event == ActivityEvent.STOPPED + for frame in self[since:] + ) + + def assert_names_only(self, *private_values: str) -> None: + for frame in self: + assert set(frame.detail) <= {"kind", "name"}, frame.detail + serialized = json.dumps([frame.payload for frame in self]) + for value in private_values: + assert value not in serialized, ( + "Private tool data appeared in room activity" + ) + + +@asynccontextmanager +async def activity_subscription( + ws: WebSocketClient | TrackingWebSocketClient, + room_id: str, + handler: Callable[[PHXMessage], Awaitable[None]], +) -> AsyncIterator[None]: + raw = ws.ws if isinstance(ws, TrackingWebSocketClient) else ws + client = raw.client + if client is None: + raise RuntimeError("Activity observation requires a connected WebSocket") + topic = activity_topic(room_id) + try: + await client.subscribe_to_topic(topic, handler) + yield + finally: + if topic in raw.joined_topics(): + await client.unsubscribe_from_topic(topic) diff --git a/tests/e2e/baseline/toolkit/capture.py b/tests/e2e/baseline/toolkit/capture.py index 0402cb306..400cb3a97 100644 --- a/tests/e2e/baseline/toolkit/capture.py +++ b/tests/e2e/baseline/toolkit/capture.py @@ -39,12 +39,13 @@ import sys from collections import defaultdict from collections.abc import AsyncIterator, Callable, Iterable -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from datetime import UTC, datetime -from typing import Any +from typing import Any, Protocol from band_rest import AsyncRestClient from phoenix_channels_python_client.exceptions import PHXClientError +from phoenix_channels_python_client.phx_messages import PHXMessage from band.client.streaming import ( DeliveryStatus, @@ -53,6 +54,12 @@ ) from band.core.types import MessageType, metadata_to_dict from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.toolkit.activity import ( + Activities, + ActivityEvent, + ActivityFrame, + activity_subscription, +) from tests.e2e.baseline.toolkit.observations import ( Errors, Events, @@ -116,6 +123,7 @@ def __init__( # One agent-auth REST client per agent id, reused across memory reads. self._agent_clients: dict[str, AsyncRestClient] = {} self.messages: Replies = Replies() + self.activities = Activities() # message_id -> {recipient_id: delivery-state dict}, fed by # ``message_updated`` events. The authoritative "agent finished this # message" signal, independent of any LLM reply text. @@ -252,6 +260,21 @@ async def _log_delivery_timeout(self, message_id: str, recipient_id: str) -> Non error, ) + async def wait_for_tool_activity( + self, agent_id: str, names: Iterable[str] + ) -> Activities: + accepted = frozenset(names) + await self.wait_until(lambda _: self.activities.has_tool(agent_id, accepted)) + return self.activities + + async def wait_for_activity_stop( + self, agent_id: str, *, since: int = 0 + ) -> Activities: + await self.wait_until( + lambda _: self.activities.has_stopped(agent_id, since=since) + ) + return self.activities + async def wait_until( self, predicate: Callable[[list[MessageCreatedPayload]], bool], @@ -645,6 +668,7 @@ async def reply_capture( user_ops: UserOps | None = None, settings: BaselineSettings | None = None, deadline_s: float = DEFAULT_DEADLINE_S, + observe_activity: bool = False, ) -> AsyncIterator[ReplyCapture]: """Subscribe to a room before sending, yield a capture, leave on exit.""" capture = ReplyCapture( @@ -657,9 +681,22 @@ async def handler(payload: MessageCreatedPayload) -> None: async def updated_handler(payload: MessageCreatedPayload) -> None: capture._on_message_updated(payload) + async def activity_handler(message: PHXMessage) -> None: + try: + event = ActivityEvent(message.event) + except ValueError: + return + capture.activities.append(ActivityFrame(event, dict(message.payload))) + capture._nudge.set() + await ws.join_chat_room_channel(room_id, handler, updated_handler) try: - yield capture + async with ( + activity_subscription(ws, room_id, activity_handler) + if observe_activity + else nullcontext() + ): + yield capture finally: # Best-effort, but only for the transport: a leave the server never acks # (``PHXTopicError``) or one attempted while the socket is down ( @@ -680,4 +717,7 @@ async def updated_handler(payload: MessageCreatedPayload) -> None: # Type of the ``reply_capture`` fixture: call with a room id, get a # ``ReplyCapture`` async context manager. Shared so smokes type their fixture # parameter without each redefining the same alias. -CaptureFactory = Callable[[str], AbstractAsyncContextManager[ReplyCapture]] +class CaptureFactory(Protocol): + def __call__( + self, room_id: str, *, observe_activity: bool = False + ) -> AbstractAsyncContextManager[ReplyCapture]: ... diff --git a/tests/e2e/baseline/toolkit/control.py b/tests/e2e/baseline/toolkit/control.py index 70e5c1764..2b5087d92 100644 --- a/tests/e2e/baseline/toolkit/control.py +++ b/tests/e2e/baseline/toolkit/control.py @@ -7,6 +7,8 @@ from collections.abc import AsyncGenerator, Callable from contextlib import asynccontextmanager +from band_rest.types.agent_activity_detail import AgentActivityDetail + import band.runtime from band.client.streaming import ( AgentControlPayload, @@ -195,6 +197,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -205,7 +208,11 @@ def observe_stop() -> None: on_stopped() accepted = await super().report_activity( - room_id, working, timeout_seconds=timeout_seconds, on_stopped=observe_stop + room_id, + working, + activity=activity, + timeout_seconds=timeout_seconds, + on_stopped=observe_stop, ) if accepted and not working and room_id == self.control_room_id: self.control.activity_idle.set() diff --git a/tests/framework_conformance/crewaicoverage/__init__.py b/tests/framework_conformance/crewaicoverage/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/framework_conformance/crewaicoverage/kit.py b/tests/framework_conformance/crewaicoverage/kit.py new file mode 100644 index 000000000..218788b9e --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/kit.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc + +_CREWAI_ONLY_MODULES = { + "crewai": "crewai", + "nest-asyncio": "nest_asyncio", + "pillow": "PIL", +} + + +def _job_command() -> str: + return vjc.job_command("test-crewai", "Run crewai tests") + + +def activity_command() -> str: + return vjc.job_command("test-crewai", "Run CrewAI activity tests") diff --git a/tests/framework_conformance/crewaicoverage/test_collection.py b/tests/framework_conformance/crewaicoverage/test_collection.py new file mode 100644 index 000000000..b68ecb2b2 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_collection.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, + _job_command, + activity_command, +) + + +class TestCrewaiJobRunsEveryTestThatNeedsItsVenv: + def test_crewai_job_runs_every_test_that_needs_its_venv(self) -> None: + pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + command = _job_command() + listed = vjc.job_paths(command) + missing = sorted( + str(path) + for path in needed + if not vjc.covered_by(path, listed, command) + and not vjc.covered_by( + path, vjc.job_paths(activity_command()), activity_command() + ) + ) + assert not missing, ( + "these test files need a dev-crewai-only dependency but ci.yml's crewai " + f"job never collects them, so they run nowhere: {missing}" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_dependencies.py b/tests/framework_conformance/crewaicoverage/test_dependencies.py new file mode 100644 index 000000000..1bf150144 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_dependencies.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, +) + + +class TestCrewaiOnlyDependencySetMatchesPyproject: + def test_crewai_only_dependency_set_matches_pyproject(self) -> None: + """The detected module set is derived from a real dep list, not a guess.""" + assert vjc.only_distributions("dev-crewai") == set(_CREWAI_ONLY_MODULES), ( + "the dev-crewai-only dependencies changed; update _CREWAI_ONLY_MODULES so " + "this guard still detects every test that needs that venv" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_detection.py b/tests/framework_conformance/crewaicoverage/test_detection.py new file mode 100644 index 000000000..c6f6a812b --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_detection.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, +) + + +class TestDetectionFindsTheTestsThatOnlyTheCrewaiVenvCanRun: + def test_detection_finds_the_tests_that_only_the_crewai_venv_can_run(self) -> None: + """Guard the guard: if the pattern drifts, the tests above pass empty. + + These are the whole set today — `phase3` for its nest_asyncio cases, + `test_files_image_passthrough_matrix.py` for its guarded `import crewai` + (real-package probe, `_CREWAI_AVAILABLE` skip elsewhere), the rest for + importing crewai itself. + """ + pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + assert needed == { + Path("tests/integrations/crewai/activity/test_execution.py"), + Path("tests/adapters/test_crewai_flow_phase3.py"), + Path("tests/adapters/crewaiworker/test_native_settlement.py"), + Path("tests/adapters/crewaiworker/test_flow_admission.py"), + Path("tests/adapters/crewaiworker/test_parallel_settlement.py"), + Path("tests/integrations/test_crewai_flow_real_sdk.py"), + Path("tests/integrations/test_crewai_real_tools.py"), + Path("tests/integrations/test_crewai_completion.py"), + Path("tests/test_capability_gating_e2e.py"), + Path("tests/framework_conformance/test_files_image_passthrough_matrix.py"), + } diff --git a/tests/framework_conformance/crewaicoverage/test_paths.py b/tests/framework_conformance/crewaicoverage/test_paths.py new file mode 100644 index 000000000..2ab18018c --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_paths.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _job_command, + activity_command, +) +from tests.paths import REPO_ROOT + + +class TestCrewaiJobPathsAllExist: + def test_crewai_job_paths_all_exist(self) -> None: + """The other drift direction: a listed path that was renamed or deleted is a + silently empty target, since pytest is given the whole list at once.""" + listed = vjc.job_paths(_job_command()) | vjc.job_paths(activity_command()) + stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) + assert not stale, ( + f"ci.yml's crewai job lists paths that no longer exist: {stale}" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_strictness.py b/tests/framework_conformance/crewaicoverage/test_strictness.py new file mode 100644 index 000000000..1b2e069b7 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_strictness.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from tests.framework_configs.sentinel import StrictnessSettings + + +class TestMissingFrameworkOptoutIsParsedAsABoolean: + @pytest.mark.parametrize("flag", ["1", "0", ""]) + def test_missing_framework_optout_is_parsed_as_a_boolean( + self, flag: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``BAND_ALLOW_MISSING_FRAMEWORKS=0`` must keep strict mode ON. + + The flag is set explicitly rather than conditionally: e2e.yml sends "0" for every + lane that should stay strict, and only ci.yml's crewai/parlant jobs send "1". A + presence check reads that "0" as an opt-out — so the value has to be parsed as a + boolean, or a cell asking to stay strict silently disables the fail-loud guard + instead. + """ + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("BAND_ALLOW_MISSING_FRAMEWORKS", flag) + + settings = StrictnessSettings() + strict = settings.ci and not settings.band_allow_missing_frameworks + assert strict is (flag != "1") diff --git a/tests/framework_conformance/parlantcoverage/__init__.py b/tests/framework_conformance/parlantcoverage/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/framework_conformance/parlantcoverage/kit.py b/tests/framework_conformance/parlantcoverage/kit.py new file mode 100644 index 000000000..9c797de12 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/kit.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc + +_PARLANT_ONLY_MODULES = { + "parlant": "parlant", + "werkzeug": "werkzeug", +} + + +def _job_command() -> str: + return vjc.job_command("test-parlant", "Run parlant tests") + + +def activity_command() -> str: + return vjc.job_command("test-parlant", "Run Parlant activity tests") diff --git a/tests/framework_conformance/parlantcoverage/test_collection.py b/tests/framework_conformance/parlantcoverage/test_collection.py new file mode 100644 index 000000000..8155bd5d4 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_collection.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, + _job_command, + activity_command, +) + + +class TestParlantJobRunsEveryTestThatNeedsItsVenv: + def test_parlant_job_runs_every_test_that_needs_its_venv(self) -> None: + pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + command = _job_command() + listed = vjc.job_paths(command) + missing = sorted( + str(path) + for path in needed + if not vjc.covered_by(path, listed, command) + and not vjc.covered_by( + path, vjc.job_paths(activity_command()), activity_command() + ) + ) + assert not missing, ( + "these test files need a dev-parlant-only dependency but ci.yml's parlant " + f"job never collects them, so they run nowhere: {missing}" + ) diff --git a/tests/framework_conformance/parlantcoverage/test_dependencies.py b/tests/framework_conformance/parlantcoverage/test_dependencies.py new file mode 100644 index 000000000..465d6cbc7 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_dependencies.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, +) + + +class TestParlantOnlyDependencySetMatchesPyproject: + def test_parlant_only_dependency_set_matches_pyproject(self) -> None: + """The detected module set is derived from a real dep list, not a guess.""" + assert vjc.only_distributions("dev-parlant") == set(_PARLANT_ONLY_MODULES), ( + "the dev-parlant-only dependencies changed; update _PARLANT_ONLY_MODULES so " + "this guard still detects every test that needs that venv" + ) diff --git a/tests/framework_conformance/parlantcoverage/test_detection.py b/tests/framework_conformance/parlantcoverage/test_detection.py new file mode 100644 index 000000000..1666583a8 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_detection.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, +) + + +class TestDetectionFindsTheTestsThatOnlyTheParlantVenvCanRun: + def test_detection_finds_the_tests_that_only_the_parlant_venv_can_run(self) -> None: + """Guard the guard: if the pattern drifts, the tests above pass empty.""" + pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + assert needed == { + Path("tests/integrations/parlant/activity/test_execution.py"), + Path("tests/integrations/parlant/test_catalog.py"), + Path("tests/integrations/parlant/test_chattools.py"), + Path("tests/integrations/parlant/test_contacttools.py"), + Path("tests/integrations/parlant/test_customtools.py"), + Path("tests/integrations/parlant/test_filetools.py"), + Path("tests/integrations/parlant/test_guard.py"), + Path("tests/integrations/parlant/test_preambletask.py"), + Path("tests/integrations/parlant/test_selection.py"), + Path("tests/integrations/parlant/test_server.py"), + Path("tests/integrations/parlant/test_tasktools.py"), + Path("tests/integrations/parlant/test_vocabulary.py"), + } diff --git a/tests/framework_conformance/parlantcoverage/test_paths.py b/tests/framework_conformance/parlantcoverage/test_paths.py new file mode 100644 index 000000000..f2c960630 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_paths.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _job_command, + activity_command, +) +from tests.paths import REPO_ROOT + + +class TestParlantJobPathsAllExist: + def test_parlant_job_paths_all_exist(self) -> None: + """The other drift direction: a listed path that was renamed or deleted is a + silently empty target, since pytest is given the whole list at once.""" + listed = vjc.job_paths(_job_command()) | vjc.job_paths(activity_command()) + stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) + assert not stale, ( + f"ci.yml's parlant job lists paths that no longer exist: {stale}" + ) diff --git a/tests/framework_conformance/test_crewai_job_coverage.py b/tests/framework_conformance/test_crewai_job_coverage.py deleted file mode 100644 index 89f5314fc..000000000 --- a/tests/framework_conformance/test_crewai_job_coverage.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Guard against drift in ci.yml's hand-listed crewai test paths. - -See `venv_job_coverage`'s module docstring for the general shape of this guard. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.framework_configs.sentinel import StrictnessSettings -from tests.framework_conformance import venv_job_coverage as vjc -from tests.paths import REPO_ROOT - -# Import names of the distributions only `dev-crewai` installs. Kept as a map so -# the drift test below can prove it still matches pyproject: a new dev-crewai-only -# dep fails there rather than silently narrowing what this guard detects. -_CREWAI_ONLY_MODULES = { - "crewai": "crewai", - "nest-asyncio": "nest_asyncio", - "pillow": "PIL", -} - - -def _job_command() -> str: - return vjc.job_command("test-crewai", "Run crewai tests") - - -class TestCrewAIJobCoverage: - def test_crewai_job_runs_every_test_that_needs_its_venv(self) -> None: - pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - command = _job_command() - listed = vjc.job_paths(command) - missing = sorted( - str(path) for path in needed if not vjc.covered_by(path, listed, command) - ) - assert not missing, ( - "these test files need a dev-crewai-only dependency but ci.yml's crewai " - f"job never collects them, so they run nowhere: {missing}" - ) - - def test_crewai_job_paths_all_exist(self) -> None: - """The other drift direction: a listed path that was renamed or deleted is a - silently empty target, since pytest is given the whole list at once.""" - listed = vjc.job_paths(_job_command()) - stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) - assert not stale, ( - f"ci.yml's crewai job lists paths that no longer exist: {stale}" - ) - - def test_crewai_only_dependency_set_matches_pyproject(self) -> None: - """The detected module set is derived from a real dep list, not a guess.""" - assert vjc.only_distributions("dev-crewai") == set(_CREWAI_ONLY_MODULES), ( - "the dev-crewai-only dependencies changed; update _CREWAI_ONLY_MODULES so " - "this guard still detects every test that needs that venv" - ) - - def test_detection_finds_the_tests_that_only_the_crewai_venv_can_run(self) -> None: - """Guard the guard: if the pattern drifts, the tests above pass empty. - - These are the whole set today — `phase3` for its nest_asyncio cases, - `test_files_image_passthrough_matrix.py` for its guarded `import crewai` - (real-package probe, `_CREWAI_AVAILABLE` skip elsewhere), the rest for - importing crewai itself. - """ - pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - assert needed == { - Path("tests/adapters/test_crewai_flow_phase3.py"), - Path("tests/adapters/crewaiworker/test_native_settlement.py"), - Path("tests/adapters/crewaiworker/test_flow_admission.py"), - Path("tests/adapters/crewaiworker/test_parallel_settlement.py"), - Path("tests/integrations/test_crewai_flow_real_sdk.py"), - Path("tests/integrations/test_crewai_real_tools.py"), - Path("tests/integrations/test_crewai_completion.py"), - Path("tests/test_capability_gating_e2e.py"), - Path("tests/framework_conformance/test_files_image_passthrough_matrix.py"), - } - - @pytest.mark.parametrize("flag", ["1", "0", ""]) - def test_missing_framework_optout_is_parsed_as_a_boolean( - self, flag: str, monkeypatch: pytest.MonkeyPatch - ) -> None: - """``BAND_ALLOW_MISSING_FRAMEWORKS=0`` must keep strict mode ON. - - The flag is set explicitly rather than conditionally: e2e.yml sends "0" for every - lane that should stay strict, and only ci.yml's crewai/parlant jobs send "1". A - presence check reads that "0" as an opt-out — so the value has to be parsed as a - boolean, or a cell asking to stay strict silently disables the fail-loud guard - instead. - """ - monkeypatch.setenv("CI", "true") - monkeypatch.setenv("BAND_ALLOW_MISSING_FRAMEWORKS", flag) - - settings = StrictnessSettings() - strict = settings.ci and not settings.band_allow_missing_frameworks - assert strict is (flag != "1") diff --git a/tests/framework_conformance/test_parlant_job_coverage.py b/tests/framework_conformance/test_parlant_job_coverage.py deleted file mode 100644 index b0bf342e3..000000000 --- a/tests/framework_conformance/test_parlant_job_coverage.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Guard against drift in ci.yml's hand-listed parlant test paths. - -See `venv_job_coverage`'s module docstring for the general shape of this guard. -``BAND_ALLOW_MISSING_FRAMEWORKS`` boolean-parsing is shared, non-parlant-specific -machinery — covered once in ``test_crewai_job_coverage.py``, not duplicated here. -""" - -from __future__ import annotations - -from pathlib import Path - -from tests.framework_conformance import venv_job_coverage as vjc -from tests.paths import REPO_ROOT - -# Import names of the distributions only `dev-parlant` installs. Kept as a map so -# the drift test below can prove it still matches pyproject: a new dev-parlant-only -# dep fails there rather than silently narrowing what this guard detects. -_PARLANT_ONLY_MODULES = { - "parlant": "parlant", - "werkzeug": "werkzeug", -} - - -def _job_command() -> str: - return vjc.job_command("test-parlant", "Run parlant tests") - - -class TestParlantJobCoverage: - def test_parlant_job_runs_every_test_that_needs_its_venv(self) -> None: - pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - command = _job_command() - listed = vjc.job_paths(command) - missing = sorted( - str(path) for path in needed if not vjc.covered_by(path, listed, command) - ) - assert not missing, ( - "these test files need a dev-parlant-only dependency but ci.yml's parlant " - f"job never collects them, so they run nowhere: {missing}" - ) - - def test_parlant_job_paths_all_exist(self) -> None: - """The other drift direction: a listed path that was renamed or deleted is a - silently empty target, since pytest is given the whole list at once.""" - listed = vjc.job_paths(_job_command()) - stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) - assert not stale, ( - f"ci.yml's parlant job lists paths that no longer exist: {stale}" - ) - - def test_parlant_only_dependency_set_matches_pyproject(self) -> None: - """The detected module set is derived from a real dep list, not a guess.""" - assert vjc.only_distributions("dev-parlant") == set(_PARLANT_ONLY_MODULES), ( - "the dev-parlant-only dependencies changed; update _PARLANT_ONLY_MODULES so " - "this guard still detects every test that needs that venv" - ) - - def test_detection_finds_the_tests_that_only_the_parlant_venv_can_run(self) -> None: - """Guard the guard: if the pattern drifts, the tests above pass empty.""" - pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - assert needed == { - Path("tests/integrations/parlant/test_catalog.py"), - Path("tests/integrations/parlant/test_chattools.py"), - Path("tests/integrations/parlant/test_contacttools.py"), - Path("tests/integrations/parlant/test_customtools.py"), - Path("tests/integrations/parlant/test_filetools.py"), - Path("tests/integrations/parlant/test_guard.py"), - Path("tests/integrations/parlant/test_preambletask.py"), - Path("tests/integrations/parlant/test_selection.py"), - Path("tests/integrations/parlant/test_server.py"), - Path("tests/integrations/parlant/test_tasktools.py"), - Path("tests/integrations/parlant/test_vocabulary.py"), - } diff --git a/tests/framework_conformance/test_strands_nooa_job_coverage.py b/tests/framework_conformance/test_strands_nooa_job_coverage.py index d639dd29e..6f37efed9 100644 --- a/tests/framework_conformance/test_strands_nooa_job_coverage.py +++ b/tests/framework_conformance/test_strands_nooa_job_coverage.py @@ -32,6 +32,8 @@ def test_expected_configs_are_present(self, framework_environment: None) -> None assert expected <= {config.framework_id for config in ADAPTER_CONFIGS} assert expected <= {config.framework_id for config in CONVERTER_CONFIGS} + # Three collect-only subprocesses; each is slow to start on Windows runners. + @pytest.mark.timeout(120) def test_job_selects_every_required_node(self, framework_environment: None) -> None: needed = coverage.tests_needing_venv( coverage.needs_venv_pattern(frozenset({"strands", "nooa"})) @@ -42,6 +44,11 @@ def test_job_selects_every_required_node(self, framework_environment: None) -> N coverage.job_command("test-strands-nooa", "Run Strands and NOOA tests") ) selected = coverage.selected_nodes(arguments) + activity_arguments = coverage.pytest_arguments( + coverage.job_command("test-strands-nooa", "Run Strands activity tests") + ) + assert "-k" not in activity_arguments + selected |= coverage.selected_nodes(activity_arguments) # Directory collection honors NOOA's supported-platform conftest gate. nooa_tests = REPO_ROOT / "tests/adapters/nooa" targets = { diff --git a/tests/integrations/codex/conftest.py b/tests/integrations/codex/conftest.py index 5d3fc862c..d7ffc14f5 100644 --- a/tests/integrations/codex/conftest.py +++ b/tests/integrations/codex/conftest.py @@ -12,10 +12,20 @@ import pytest_asyncio from band.integrations.codex import stdio_client, winspawn +from tests.integrations.codex.jobpeer import NativeJobPeer from tests.integrations.codex.lifecycle import WindowsLifecycle from tests.integrations.codex.lookup import WindowsLookup +@pytest.fixture +def native_job(monkeypatch: pytest.MonkeyPatch) -> NativeJobPeer: + peer = NativeJobPeer() + monkeypatch.setattr(winspawn, "_kernel32", lambda: peer.kernel) + monkeypatch.setattr(winspawn, "_last_error", lambda: peer.error) + monkeypatch.setattr(winspawn, "_win_os_error", lambda code: OSError(code)) + return peer + + @pytest.fixture def windows_lookup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> WindowsLookup: peer = WindowsLookup(tmp_path) diff --git a/tests/integrations/codex/jobpeer.py b/tests/integrations/codex/jobpeer.py new file mode 100644 index 000000000..247eccad1 --- /dev/null +++ b/tests/integrations/codex/jobpeer.py @@ -0,0 +1,134 @@ +"""Native job boundary where accounting can finish before physical process exit.""" + +from __future__ import annotations + +import ctypes +from ctypes import wintypes +from types import SimpleNamespace + +from band.integrations.codex.winspawn import ( + BasicAccountingInformation, + ExtendedLimitInformation, +) + + +class NativeJobPeer: + def __init__(self) -> None: + self.handle = ctypes.c_void_p(1) + self.pids = [101] + self.exited: set[int] = set() + self.reused: set[int] = set() + self.opened: set[int] = set() + self.waited: list[int] = [] + self.active = 1 + self.error = 0 + self.open_error = 0 + self.wait_error = False + self.close_error = False + self.query_error = False + self.terminated = False + self.closed = False + self.sealed = False + self.kernel = SimpleNamespace( + SetInformationJobObject=self.set_limits, + QueryInformationJobObject=self.query, + TerminateJobObject=self.terminate, + OpenProcess=self.open_process, + IsProcessInJob=self.is_member, + WaitForSingleObject=self.wait, + CloseHandle=self.close, + ) + + def set_limits( + self, handle: ctypes.c_void_p, kind: int, data: object, size: int + ) -> bool: + assert handle is self.handle + assert kind == 9 + assert size == ctypes.sizeof(ExtendedLimitInformation) + limits = ctypes.cast(data, ctypes.POINTER(ExtendedLimitInformation)).contents + basic = limits.BasicLimitInformation + assert basic.LimitFlags & 0x2000, "sealing must preserve kill-on-close" + self.sealed = bool(basic.LimitFlags & 8 and basic.ActiveProcessLimit == 0) + return True + + def query( + self, + handle: ctypes.c_void_p, + kind: int, + data: object, + size: int, + returned: object, + ) -> bool: + assert handle is self.handle + if self.query_error: + self.error = 5 + return False + if kind == 1: + info = ctypes.cast( + data, ctypes.POINTER(BasicAccountingInformation) + ).contents + info.ActiveProcesses = self.active + return True + assert kind == 3 + assert self.sealed, "new descendants must be excluded before the snapshot" + assert not self.terminated, "termination can remove unsignaled members" + header_size = 2 * ctypes.sizeof(wintypes.DWORD) + header = ctypes.cast(data, ctypes.POINTER(wintypes.DWORD)) + capacity = (size - header_size) // ctypes.sizeof(ctypes.c_size_t) + header[0] = len(self.pids) + header[1] = min(capacity, len(self.pids)) + address = ctypes.cast(data, ctypes.c_void_p).value + assert address is not None + entries = (ctypes.c_size_t * header[1]).from_address(address + header_size) + entries[:] = self.pids[: header[1]] + if capacity < len(self.pids): + self.error = 234 + return False + return True + + def terminate(self, handle: ctypes.c_void_p, code: int) -> bool: + assert handle is self.handle + assert code == 1 + self.terminated = True + self.active = 0 + return True + + def open_process( + self, access: int, inherit: bool, pid: int + ) -> ctypes.c_void_p | None: + assert access == 0x101000 + assert not inherit + if self.open_error: + self.error = self.open_error + return None + self.opened.add(pid) + return ctypes.c_void_p(pid) + + def is_member( + self, process: ctypes.c_void_p, job: ctypes.c_void_p, result: object + ) -> bool: + assert job is self.handle + ctypes.cast(result, ctypes.POINTER(wintypes.BOOL))[0] = ( + process.value not in self.reused + ) + return True + + def wait(self, handle: ctypes.c_void_p, timeout: int) -> int: + assert timeout == 0, "process waits must not block the event loop" + assert handle.value is not None + self.waited.append(handle.value) + if self.wait_error: + self.error = 6 + return 0xFFFFFFFF + return 0 if handle.value in self.exited else 258 + + def close(self, handle: ctypes.c_void_p) -> bool: + if self.close_error: + self.error = 6 + return False + if handle is self.handle: + self.closed = True + else: + assert handle.value is not None + self.opened.remove(handle.value) + return True diff --git a/tests/integrations/codex/test_cleanup.py b/tests/integrations/codex/test_cleanup.py index b382d7644..f287f31c3 100644 --- a/tests/integrations/codex/test_cleanup.py +++ b/tests/integrations/codex/test_cleanup.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import ctypes import logging import os import shutil @@ -311,9 +312,7 @@ def deny(_name: str) -> object: async def test_idle_wait_preserves_ownership_until_explicit_release( self, active: int | None, drained: bool ) -> None: - job = Job.__new__(Job) - job._handle = object() # type: ignore[assignment] - job.name = "test" + job = Job(ctypes.c_void_p(1), "test") closed = False def active_processes() -> int | None: diff --git a/tests/integrations/codex/test_job.py b/tests/integrations/codex/test_job.py index c801264cf..d907231b2 100644 --- a/tests/integrations/codex/test_job.py +++ b/tests/integrations/codex/test_job.py @@ -8,6 +8,7 @@ import pytest from band.integrations.codex import winspawn +from tests.integrations.codex.jobpeer import NativeJobPeer class TestJobHandleOwnership: @@ -32,25 +33,167 @@ def close_handle(current: ctypes.c_void_p) -> bool: job.close() assert attempted == [handle, handle] - def test_failed_termination_is_reported_without_releasing_the_job( - self, monkeypatch: pytest.MonkeyPatch + async def test_failed_termination_is_reported_without_releasing_the_job( + self, native_job: NativeJobPeer ) -> None: - handle = ctypes.c_void_p(1) + handle = native_job.handle job = winspawn.Job(handle, "test") terminated: list[ctypes.c_void_p] = [] def terminate(current: ctypes.c_void_p, _code: int) -> bool: terminated.append(current) - return len(terminated) > 1 + if len(terminated) == 1: + native_job.error = 5 + return False + return native_job.terminate(current, _code) - monkeypatch.setattr( - winspawn, - "_kernel32", - lambda: SimpleNamespace(TerminateJobObject=terminate), - ) - monkeypatch.setattr(winspawn, "_last_error", lambda: 5) - monkeypatch.setattr(winspawn, "_win_os_error", lambda code: OSError(code)) + native_job.kernel.TerminateJobObject = terminate with pytest.raises(OSError): job.terminate() job.terminate() assert terminated == [handle, handle] + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + + async def test_zero_accounting_retains_unsignaled_member_until_retry( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + assert native_job.active == 0 + with pytest.raises(winspawn.JobNotDrained): + await job.wait_until_idle(0) + assert not native_job.closed + assert native_job.opened == {101} + native_job.exited.add(101) + job.terminate() + await job.wait_until_idle(0) + job.close() + assert native_job.closed + assert not native_job.opened + + async def test_all_members_must_exit_when_the_pid_buffer_grows( + self, native_job: NativeJobPeer + ) -> None: + native_job.pids = list(range(101, 166)) + native_job.active = len(native_job.pids) + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.exited.update(native_job.pids[:-1]) + with pytest.raises(winspawn.JobNotDrained): + await job.wait_until_idle(0) + assert native_job.opened == {165} + native_job.exited.add(165) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_reused_pid_is_not_waited_as_a_job_member( + self, native_job: NativeJobPeer + ) -> None: + native_job.reused.add(101) + job = winspawn.Job(native_job.handle, "test") + job.terminate() + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + assert not native_job.waited + + def test_denied_member_handle_preserves_job_custody( + self, native_job: NativeJobPeer + ) -> None: + native_job.open_error = 5 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert not native_job.terminated + assert not native_job.closed + + async def test_native_wait_failure_retains_the_member_handle( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.wait_error = True + with pytest.raises(OSError): + await job.wait_until_idle(0) + assert native_job.opened == {101} + assert not native_job.closed + native_job.wait_error = False + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_failed_member_handle_close_keeps_retryable_ownership( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.exited.add(101) + native_job.close_error = True + with pytest.raises(OSError): + await job.wait_until_idle(0) + assert native_job.opened == {101} + assert not native_job.closed + native_job.close_error = False + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_job_close_cannot_discard_pending_physical_exit( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + with pytest.raises(winspawn.JobNotDrained): + job.close() + assert not native_job.closed + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert native_job.closed + + async def test_failed_membership_query_keeps_the_handle_for_retry( + self, native_job: NativeJobPeer + ) -> None: + native_job.kernel.IsProcessInJob = lambda *_args: False + native_job.error = 5 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert native_job.opened == {101} + assert not native_job.terminated + native_job.kernel.IsProcessInJob = native_job.is_member + job.terminate() + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_vanished_member_pid_is_confirmed_against_the_job( + self, native_job: NativeJobPeer + ) -> None: + def gone(_access: int, _inherit: bool, _pid: int) -> None: + native_job.pids.clear() + native_job.active = 0 + native_job.error = 87 + + native_job.kernel.OpenProcess = gone + job = winspawn.Job(native_job.handle, "test") + job.terminate() + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + assert not native_job.waited + + def test_invalid_member_pid_still_in_the_job_is_not_suppressed( + self, native_job: NativeJobPeer + ) -> None: + native_job.open_error = 87 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert not native_job.terminated + assert not native_job.closed diff --git a/tests/integrations/crewai/__init__.py b/tests/integrations/crewai/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/__init__.py b/tests/integrations/crewai/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/kit.py b/tests/integrations/crewai/activity/kit.py new file mode 100644 index 000000000..77e0cfd42 --- /dev/null +++ b/tests/integrations/crewai/activity/kit.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LookupInput(BaseModel): + """Look up a private value.""" + + note: str diff --git a/tests/integrations/crewai/activity/test_execution.py b/tests/integrations/crewai/activity/test_execution.py new file mode 100644 index 000000000..6f532ac4c --- /dev/null +++ b/tests/integrations/crewai/activity/test_execution.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json + +import pytest + +from band.integrations.crewai.tools import ( + CrewAIToolContext, + NoopReporter, + build_band_crewai_tools, +) +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.integrations.crewai.activity.kit import LookupInput + +pytest.importorskip("crewai") + + +class TestCrewAIActivity: + @pytest.mark.parametrize("failed", [False, True]) + def test_real_base_tool_reports_without_narration_and_ends_on_error( + self, failed: bool + ) -> None: + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + + async def lookup(args: LookupInput) -> str: + assert recorder.events == [{"kind": "tool_call", "name": "lookup"}] + assert args.note == "private-argument" + if failed: + raise ValueError("private-error") + return "private-result" + + built = build_band_crewai_tools( + get_context=lambda: CrewAIToolContext(room_id="room", tools=tools), + reporter=NoopReporter(), + custom_tools=[(LookupInput, lookup)], + ) + lookup_tool = next(tool for tool in built if tool.name == "lookup") + result = json.loads(lookup_tool.run(note="private-argument")) + assert result["status"] == ("error" if failed else "success") + recorder.assert_completed_tool("lookup") + assert not tools.events_sent diff --git a/tests/integrations/parlant/activity/__init__.py b/tests/integrations/parlant/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/parlant/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/parlant/activity/kit.py b/tests/integrations/parlant/activity/kit.py new file mode 100644 index 000000000..77e0cfd42 --- /dev/null +++ b/tests/integrations/parlant/activity/kit.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LookupInput(BaseModel): + """Look up a private value.""" + + note: str diff --git a/tests/integrations/parlant/activity/test_execution.py b/tests/integrations/parlant/activity/test_execution.py new file mode 100644 index 000000000..1b2b5e1ed --- /dev/null +++ b/tests/integrations/parlant/activity/test_execution.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("parlant.sdk") + +from parlant.core.tools import ToolContext + +from band.integrations.parlant.customtools import build_custom_tool +from band.integrations.parlant.sessiontools import set_session_tools +from band.integrations.parlant.tools import create_parlant_tools +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.integrations.parlant.activity.kit import LookupInput +from tests.integrations.parlant.helpers import ToolServer + + +class TestParlantActivity: + @pytest.mark.parametrize("failed", [False, True]) + async def test_real_server_reports_only_calling_session_and_always_ends( + self, plugin_server: ToolServer, failed: bool + ) -> None: + tools, other = FakeAgentTools(), FakeAgentTools() + recorder, quiet = ActivityRecorder(), ActivityRecorder() + tools._activity_observer = recorder + other._activity_observer = quiet + set_session_tools("calling", tools) + set_session_tools("other", other) + entered, release = asyncio.Event(), asyncio.Event() + + async def lookup(args: LookupInput) -> str: + assert args.note == "private-argument" + entered.set() + await release.wait() + if failed: + raise ValueError("private-error") + return "private-result" + + await plugin_server.enable( + create_parlant_tools(custom_tools=[(LookupInput, lookup)]) + ) + call = asyncio.create_task( + plugin_server.call( + "lookup", session_id="calling", arguments={"note": "private-argument"} + ) + ) + try: + async with asyncio.timeout(5): + await entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": "lookup"}] + assert not quiet.events + finally: + release.set() + result = await call + assert result == ( + "Error running lookup: private-error" if failed else "private-result" + ) + recorder.assert_completed_tool("lookup") + assert not tools.events_sent + + async def test_custom_handler_cancellation_ends_activity(self) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + set_session_tools("cancelled", tools) + entered = asyncio.Event() + + async def lookup(args: LookupInput) -> str: + entered.set() + await asyncio.Event().wait() + return "unreachable" + + entry = build_custom_tool((LookupInput, lookup)) + task = asyncio.create_task( + entry.function( + ToolContext( + agent_id="agent", session_id="cancelled", customer_id="user" + ), + note="private-argument", + ) + ) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + recorder.assert_completed_tool("lookup") diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py new file mode 100644 index 000000000..f14f18e0e --- /dev/null +++ b/tests/platform/activity/conftest.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.activitysupport import install_activity_support +from tests.platform.activity.kit import DETAIL, ActivityPeer +from tests.runtime.helpers import rest_client_over + + +@pytest.fixture(autouse=True) +def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: + return install_activity_support(monkeypatch, learned=False) + + +@pytest.fixture +def activity_peer() -> ActivityPeer: + return ActivityPeer() + + +@pytest.fixture +def detail() -> AgentActivityDetail: + return AgentActivityDetail(**DETAIL) + + +@pytest.fixture +async def link(activity_peer: ActivityPeer) -> AsyncIterator[BandLink]: + """A link on the SDK's default (SaaS) ``rest_url``.""" + async with rest_client_over(activity_peer.respond) as rest: + link = BandLink(agent_id="agent", api_key="test-key") + link.rest = rest + yield link + + +@pytest.fixture +def second_link(link: BandLink) -> BandLink: + other = BandLink(agent_id="second-agent", api_key="test-key") + other.rest = link.rest + return other + + +@pytest.fixture +def learned(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: + """A process that already saw a supporting ``X-API-Version``.""" + return install_activity_support(monkeypatch, learned=True) diff --git a/tests/platform/activity/kit.py b/tests/platform/activity/kit.py new file mode 100644 index 000000000..9eae53c17 --- /dev/null +++ b/tests/platform/activity/kit.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +import json +from collections import deque +from typing import Any + +import httpx + +from band.platform.activity import API_VERSION_HEADER +from tests.activitysupport import SUPPORTED_VERSION + +DETAIL = {"kind": "tool_call", "name": "Bash"} + + +class ActivityPeer: + """The activity endpoint, advertising ``version`` on every response.""" + + def __init__(self) -> None: + self.version: str | None = SUPPORTED_VERSION + self.reports: list[dict[str, Any]] = [] + self.requests: list[httpx.Request] = [] + self.responses: deque[httpx.Response] = deque() + self.stall_post_number: int | None = None + + async def respond(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + self.requests.append(request) + self.reports.append(body) + if len(self.reports) == self.stall_post_number: + await asyncio.Event().wait() + response = ( + self.responses.popleft() + if self.responses + else httpx.Response(200, json={"data": {"working": body["working"]}}) + ) + if self.version is not None: + response.headers[API_VERSION_HEADER] = self.version + return response diff --git a/tests/platform/activity/test_control_link.py b/tests/platform/activity/test_control_link.py new file mode 100644 index 000000000..008781d86 --- /dev/null +++ b/tests/platform/activity/test_control_link.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.e2e.baseline.toolkit.control import ControlLink, ControlRuntime +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.usefixtures("learned") +class TestControlLinkActivity: + async def test_baseline_control_link_forwards_activity_detail( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + control_link = ControlLink(agent_id="agent", api_key="test-key") + control_link.rest = link.rest + control_link.control = ControlRuntime() + control_link.control_room_id = "room" + assert await control_link.report_activity("room", True, activity=detail) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] diff --git a/tests/platform/activity/test_deadlines.py b/tests/platform/activity/test_deadlines.py new file mode 100644 index 000000000..330c5e25c --- /dev/null +++ b/tests/platform/activity/test_deadlines.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.looptime +@pytest.mark.usefixtures("learned") +class TestActivityDeadline: + async def test_retry_cannot_extend_total_deadline_and_downgrade_survives_timeout( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + activity_peer.responses.append(httpx.Response(422)) + activity_peer.stall_post_number = 2 + started = asyncio.get_running_loop().time() + assert not await link.report_activity( + "room", True, activity=detail, timeout_seconds=2 + ) + assert asyncio.get_running_loop().time() - started <= 2.01 + assert activity_peer.reports == [ + {"working": True, "activity": DETAIL}, + {"working": True}, + ] + activity_peer.stall_post_number = None + assert await link.report_activity("next", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py new file mode 100644 index 000000000..28313ee62 --- /dev/null +++ b/tests/platform/activity/test_fallback.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import logging +from unittest.mock import Mock + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.usefixtures("learned") +class TestActivityFallback: + @pytest.mark.parametrize( + "body", [b'{"errors":{"activity":["unknown"]}}', b"not json", b""] + ) + async def test_422_retries_without_detail_and_latches_across_links( + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + body: bytes, + ) -> None: + activity_peer.responses.append(httpx.Response(422, content=body)) + assert await link.report_activity("first", True, activity=detail) + assert await second_link.report_activity("second", True, activity=detail) + assert activity_peer.reports == [ + {"working": True, "activity": DETAIL}, + {"working": True}, + {"working": True}, + ] + + @pytest.mark.parametrize( + ("working", "with_detail"), + [(True, False), (False, True)], + ids=["no-detail", "idle"], + ) + async def test_422_without_detail_neither_retries_nor_latches( + self, + link: BandLink, + activity_peer: ActivityPeer, + learned: ActivitySupport, + detail: AgentActivityDetail, + working: bool, + with_detail: bool, + ) -> None: + activity_peer.responses.append(httpx.Response(422)) + activity = detail if with_detail else None + assert not await link.report_activity("room", working, activity=activity) + assert len(activity_peer.reports) == 1 + assert learned.supported + + async def test_downgrade_logs_the_rejected_field_not_the_body( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.append( + httpx.Response( + 422, + json={ + "error": { + "code": "validation_error", + "details": {"/activity/name": ["echoed Bash --secret"]}, + "message": "Invalid request", + "request_id": "request-1", + } + }, + ) + ) + with caplog.at_level(logging.DEBUG, logger="band.platform"): + assert await link.report_activity("room", True, activity=detail) + [record] = [r for r in caplog.records if r.levelno == logging.INFO] + assert "/activity/name" in record.getMessage() + assert "secret" not in record.getMessage() + assert any( + r.levelno == logging.DEBUG and "secret" in r.getMessage() + for r in caplog.records + ) + + @pytest.mark.parametrize("status", [400, 403, 500]) + async def test_other_errors_do_not_retry_or_disable( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + status: int, + ) -> None: + activity_peer.responses.append( + httpx.Response(status, json={"error": "failure"}) + ) + assert not await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] * 2 + + async def test_fallback_echoed_false_keeps_stop_notification( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + activity_peer.responses.extend( + [ + httpx.Response(422), + httpx.Response(200, json={"data": {"working": False}}), + ] + ) + stopped = Mock() + assert await link.report_activity( + "room", True, activity=detail, on_stopped=stopped + ) + stopped.assert_called_once_with() + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_reporting.py b/tests/platform/activity/test_reporting.py new file mode 100644 index 000000000..bfc8bb51f --- /dev/null +++ b/tests/platform/activity/test_reporting.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import logging + +import httpx +import pytest + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +class TestActivityReporting: + @pytest.mark.parametrize("working", [True, False]) + async def test_reports_the_boolean_state_to_the_room( + self, link: BandLink, activity_peer: ActivityPeer, working: bool + ) -> None: + assert await link.report_activity("room-1", working) + assert activity_peer.reports == [{"working": working}] + assert activity_peer.requests[-1].url.path.endswith("/chats/room-1/activity") + + async def test_post_is_bounded_and_never_retried( + self, link: BandLink, activity_peer: ActivityPeer + ) -> None: + activity_peer.responses.append(httpx.Response(500)) + assert not await link.report_activity("room-1", True, timeout_seconds=2) + [request] = activity_peer.requests + assert request.extensions["timeout"]["read"] == 2 + + async def test_repeated_failures_warn_once_then_recover( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.extend([httpx.Response(500), httpx.Response(500)]) + with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): + assert not await link.report_activity("room-1", True) + assert not await link.report_activity("room-1", True) + assert await link.report_activity("room-1", True) + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + ] + assert "recovered" in caplog.records[-1].getMessage() diff --git a/tests/platform/activity/test_truncation.py b/tests/platform/activity/test_truncation.py new file mode 100644 index 000000000..93d83d290 --- /dev/null +++ b/tests/platform/activity/test_truncation.py @@ -0,0 +1,42 @@ +"""I-08: a tool name too long for the platform is truncated, never rejected.""" + +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + +LIMIT = 1024 + + +@pytest.mark.usefixtures("learned") +class TestNameTruncation: + async def test_over_long_name_is_clipped_before_the_post( + self, + link: BandLink, + activity_peer: ActivityPeer, + ) -> None: + long_name = "a" * (LIMIT + 16) + sent = await link.report_activity( + "room", + True, + activity=AgentActivityDetail(kind="tool_call", name=long_name), + ) + assert sent + assert activity_peer.reports == [ + { + "working": True, + "activity": {"kind": "tool_call", "name": "a" * LIMIT}, + } + ] + + async def test_exact_limit_name_is_untouched( + self, + link: BandLink, + activity_peer: ActivityPeer, + ) -> None: + detail = AgentActivityDetail(kind="tool_call", name="b" * LIMIT) + assert await link.report_activity("room", True, activity=detail) + assert activity_peer.reports[0]["activity"]["name"] == "b" * LIMIT diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py new file mode 100644 index 000000000..4274f23d9 --- /dev/null +++ b/tests/platform/activity/test_versions.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import logging + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.activitysupport import SUPPORTED_VERSION +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +class TestActivityVersion: + @pytest.mark.parametrize( + ("version", "supported"), + [ + ("1.24.99", False), + pytest.param(SUPPORTED_VERSION, True, id="minimum-version-on-saas-default"), + ("1.26.10", True), + ("2.0.0", True), + ("1.25.0-beta", False), + ("broken", False), + ("1.25", False), + ("2.0", False), + (None, False), + ], + ) + async def test_advertised_version_controls_later_detail( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + version: str | None, + supported: bool, + ) -> None: + activity_peer.version = version + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + later = ( + {"working": True, "activity": DETAIL} if supported else {"working": True} + ) + assert activity_peer.reports == [{"working": True}, later] + assert all( + request.url.path.endswith("/activity") for request in activity_peer.requests + ) + + async def test_learned_support_is_shared_across_links( + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + ) -> None: + assert await link.report_activity("first", True) + assert await second_link.report_activity("second", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True, "activity": DETAIL} + + @pytest.mark.parametrize("kind", ["reasoning", "memory", "permission"]) + async def test_non_tool_detail_omits_name_and_false_omits_detail( + self, link: BandLink, activity_peer: ActivityPeer, kind: str + ) -> None: + detail = AgentActivityDetail(kind=kind) + assert await link.report_activity("room", True) + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", False, activity=detail) + assert activity_peer.reports[1:] == [ + {"working": True, "activity": {"kind": kind}}, + {"working": False}, + ] + + async def test_unadvertised_version_is_logged_for_diagnosis( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.version = "1.24.0" + with caplog.at_level(logging.DEBUG, logger="band.platform.activity"): + assert await link.report_activity("room", True) + [record] = caplog.records + assert record.levelno == logging.DEBUG + assert "'1.24.0'" in record.getMessage() diff --git a/tests/platform/test_link.py b/tests/platform/test_link.py index d6401a71d..8a17eeb79 100644 --- a/tests/platform/test_link.py +++ b/tests/platform/test_link.py @@ -3,16 +3,9 @@ from __future__ import annotations import asyncio -import logging from unittest.mock import ANY, AsyncMock, MagicMock, create_autospec, patch import pytest -from band_rest import ( - AsyncRestClient, - NotFoundError, - UnauthorizedError, - UnprocessableEntityError, -) from band_rest.core.api_error import ApiError from band_rest.types.chat_message_metadata import ChatMessageMetadata from band_sdk_core import AgentTopicStatus, DeadReason, SessionState, chat_room_topic @@ -1346,135 +1339,3 @@ async def test_returns_empty_list_on_failure(self): messages = await link.get_stale_processing_messages("room-1") assert messages == [] - - -class TestReportActivity: - """Tests for BandLink.report_activity (boolean working-state reporting).""" - - @pytest.mark.asyncio - async def test_reports_working_true(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", True) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - call.assert_awaited_once() - assert call.call_args.kwargs["chat_id"] == "room-1" - assert call.call_args.kwargs["working"] is True - - @pytest.mark.asyncio - async def test_passes_per_post_timeout_and_no_retries(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - await link.report_activity("room-1", True, timeout_seconds=2) - - opts = link.rest.agent_api_activity.report_agent_chat_activity.call_args.kwargs[ - "request_options" - ] - assert opts["timeout_in_seconds"] == 2 - assert opts["max_retries"] == 0 - - @pytest.mark.asyncio - async def test_reports_working_false(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", False) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - assert call.call_args.kwargs["working"] is False - - @pytest.mark.asyncio - async def test_returns_false_on_not_found(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=NotFoundError(headers={}, body="no active execution") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unauthorized(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnauthorizedError(headers={}, body="bad key") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unprocessable_entity(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnprocessableEntityError(headers={}, body="bad uuid") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_network_error(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("network down") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - def test_real_client_exposes_activity_method(self): - """Guard: the real REST client must actually expose the activity method. - - The AsyncMock-based tests above auto-fabricate attributes, so they would - stay green even if `agent_api_activity.report_agent_chat_activity` - disappeared or was renamed in a band_rest bump. This test pins the - real wire contract: instantiate the real client and assert the method - exists and is callable. - """ - - client = AsyncRestClient(api_key="test-key", base_url="https://test.com") - method = getattr(client.agent_api_activity, "report_agent_chat_activity", None) - assert callable(method) - - @pytest.mark.asyncio - async def test_repeated_failures_warn_once_then_recover(self, caplog): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("down") - ) - - with caplog.at_level(logging.WARNING, logger="band.platform.message_lifecycle"): - assert await link.report_activity("room-1", True) is False - assert await link.report_activity("room-1", True) is False - - warnings = [r for r in caplog.records if r.levelno == logging.WARNING] - assert len(warnings) == 1 # debounced: only the first failure warns - - # Recovery logs once at INFO and re-arms the warning. - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): - caplog.clear() - assert await link.report_activity("room-1", True) is True - assert any("recovered" in r.message.lower() for r in caplog.records) diff --git a/tests/runtime/activity/test_namegate.py b/tests/runtime/activity/test_namegate.py new file mode 100644 index 000000000..19676192e --- /dev/null +++ b/tests/runtime/activity/test_namegate.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx + +from band.core.activity import sync_activity_tool_names +from band.core.types import Emit +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, + tool_call_activity, +) +from tests.conftest import make_message_event +from tests.runtime.activity.kit import ActivityPeer, activity_context + + +class DetailPeer(ActivityPeer): + """Capture the activity detail each report carries, alongside the base tuples.""" + + def __init__(self) -> None: + super().__init__() + self.details: list[dict[str, object] | None] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/activity"): + self.details.append(json.loads(request.content).get("activity")) + return super().answer(request) + + +def _reporting_handler( + *, tool: str | None = None, permission: bool = False, memory: bool = False +): + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if permission: + await set_permission_wait(tools, True) + await set_permission_wait(tools, False) + if memory: + await report_memory_activity(tools) + if tool is not None: + await report_tool_activity(tools, tool) + await end_tool_activity(tools) + + return handler + + +class TestActivityToolNameGate: + async def test_tool_names_report_while_tool_calls_are_emitted(self) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(tool="Bash"), + AsyncMock(), + ) as ctx: + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert kinds.count("tool_call") == 1 + named = [d for d in peer.details if d and d.get("kind") == "tool_call"] + assert named[0]["name"] == "Bash" + + async def test_tool_names_are_dropped_when_tool_calls_are_not_emitted( + self, + ) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(tool="Bash"), + AsyncMock(), + ) as ctx: + ctx.set_activity_tool_names(False) + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert "tool_call" not in kinds + assert "reasoning" in kinds + for detail in peer.details: + assert detail is None or not detail.get("name") + + async def test_permission_and_memory_survive_the_name_gate(self) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(permission=True, memory=True), + AsyncMock(), + ) as ctx: + ctx.set_activity_tool_names(False) + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert "memory" in kinds + assert "permission" in kinds + + async def test_flip_after_binding_hides_names_in_the_running_turn( + self, + ) -> None: + # The codex ordering: the observer is bound to the turn before the + # adapter's emit sync flips the flag. The gate must read the live + # choice at report time, not the choice at construction time. + peer = DetailPeer() + bound: list = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + bound.append(ctx.activity_observer(ctx.current_scope)) + ctx.set_activity_tool_names(False) + await bound[0].update(tool_call_activity("Bash")) + + async with activity_context(peer, handler, AsyncMock()) as ctx: + await ctx._process_event(make_message_event(msg_id="trigger")) + tool_frames = [d for d in peer.details if d and d.get("kind") == "tool_call"] + assert not tool_frames + assert {"kind": "reasoning"} in peer.details + + async def test_sync_resolves_names_from_the_emit_choice(self) -> None: + async with activity_context( + ActivityPeer(), _reporting_handler(), AsyncMock() + ) as ctx: + tools = SimpleNamespace(_ctx=ctx) + sync_activity_tool_names(tools, frozenset({Emit.TOOL_CALLS})) + assert ctx._activity_tool_names is True + sync_activity_tool_names(tools, frozenset({Emit.THOUGHTS})) + assert ctx._activity_tool_names is False + sync_activity_tool_names(SimpleNamespace(), frozenset()) + assert ctx._activity_tool_names is False diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py index 16bcf6bfc..4b2e227bd 100644 --- a/tests/runtime/activityownership/conftest.py +++ b/tests/runtime/activityownership/conftest.py @@ -14,25 +14,25 @@ from tests.runtime.helpers import ( AGENT_ID, ROOM_ID, - LifecyclePlatform, rest_client_over, ) @pytest.fixture -async def activity_platform() -> AsyncIterator[LifecyclePlatform]: - yield ActivityPlatform() +def activity_platform() -> ActivityPlatform: + return ActivityPlatform() @pytest.fixture async def execution( - activity_platform: LifecyclePlatform, + activity_platform: ActivityPlatform, ) -> AsyncIterator[Callable[..., ExecutionContext]]: contexts: list[ExecutionContext] = [] - async with rest_client_over(activity_platform.answer) as rest: + async with rest_client_over(activity_platform.respond) as rest: def create( handler: Callable[[ExecutionContext, PlatformEvent], Awaitable[None]], + hub_room_id: str | None = None, **settings: object, ) -> ExecutionContext: link = BandLink(agent_id=AGENT_ID, api_key="test-key") @@ -45,6 +45,7 @@ def create( link, handler, agent_id=AGENT_ID, + hub_room_id=hub_room_id, on_platform_stop=runtime._on_platform_stop, config=SessionConfig(enable_context_hydration=False, **settings), ) @@ -53,4 +54,5 @@ def create( yield create for ctx in contexts: + activity_platform.activity_release.set() await ctx.stop() diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index d43a5b921..3661bc2a4 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -1,13 +1,80 @@ from __future__ import annotations +import asyncio +import json +from itertools import groupby +from typing import Any, Literal + import httpx from tests.runtime.helpers import ALICE, LifecyclePlatform, page class ActivityPlatform(LifecyclePlatform): + def __init__(self) -> None: + super().__init__() + self.activity_bodies: list[dict[str, Any]] = [] + self.hold_next_activity = False + self.activity_entered = asyncio.Event() + self.activity_release = asyncio.Event() + self.activity_cancelled = asyncio.Event() + self.reject_detail = False + self.event_bodies: list[dict[str, Any]] = [] + + @property + def working_transitions(self) -> list[bool]: + return [ + working + for working, _ in groupby(body["working"] for body in self.activity_bodies) + ] + + async def respond(self, request: httpx.Request) -> httpx.Response: + response = self.answer(request) + if self.hold_next_activity and request.url.path.endswith("/activity"): + self.hold_next_activity = False + self.activity_entered.set() + try: + await self.activity_release.wait() + except asyncio.CancelledError: + self.activity_cancelled.set() + raise + return response + + def assert_hides(self, *private: str) -> None: + serialized = json.dumps(self.activity_bodies) + for value in private: + assert value not in serialized + + def assert_no_tool_narration(self) -> None: + assert not any( + body.get("message_type") in {"tool_call", "tool_result"} + for body in self.event_bodies + ) + def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] + if tail == "events" and request.method == "POST": + self.event_bodies.append(json.loads(request.content)) + if not self.stopped: + # A running room accepts narration; falling through to the + # lifecycle base would answer 403, which the platform client + # reads as a stopped execution and tears the turn machinery + # down around the first reported tool call. + return httpx.Response( + 201, + json={ + "data": { + "id": "e-1", + "message_type": "event", + "success": True, + } + }, + ) + if tail == "activity": + body = json.loads(request.content) + self.activity_bodies.append(body) + if self.reject_detail and "activity" in body: + return httpx.Response(422, json={"error": "unsupported activity"}) if tail == "participants": return page([{**ALICE, "role": "member", "status": "active"}]) if request.method == "POST" and tail == "messages": @@ -15,3 +82,16 @@ def answer(self, request: httpx.Request) -> httpx.Response: 201, json={"data": {"id": "m-1", "success": True, "recipients": []}} ) return super().answer(request) + + +class ObservedIdle(asyncio.Event): + """The idle event a graceful stop waits on, announcing that wait.""" + + def __init__(self) -> None: + super().__init__() + self.set() + self.awaited = asyncio.Event() + + async def wait(self) -> Literal[True]: + self.awaited.set() + return await super().wait() diff --git a/tests/runtime/activityownership/test_band_tools.py b/tests/runtime/activityownership/test_band_tools.py new file mode 100644 index 000000000..7dc23486d --- /dev/null +++ b/tests/runtime/activityownership/test_band_tools.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from collections.abc import Callable + +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.types import BandTool +from band.runtime.working_state import end_tool_activity, report_tool_activity +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestBandToolActivity: + async def test_direct_registered_method_and_model_event_are_observed_but_telemetry_is_quiet( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await tools.get_participants() + await report_tool_activity(tools, "Bash") + await tools.send_event("private telemetry", "thought") + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } + await end_tool_activity(tools) + await tools.execute_tool_call( + BandTool.SEND_EVENT, + {"content": "model telemetry", "message_type": "thought"}, + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + { + "working": True, + "activity": {"kind": "tool_call", "name": BandTool.GET_PARTICIPANTS}, + }, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": {"kind": "reasoning"}}, + { + "working": True, + "activity": {"kind": "tool_call", "name": BandTool.SEND_EVENT}, + }, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] diff --git a/tests/runtime/activityownership/test_cancelled_tool.py b/tests/runtime/activityownership/test_cancelled_tool.py new file mode 100644 index 000000000..b45cb393e --- /dev/null +++ b/tests/runtime/activityownership/test_cancelled_tool.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +from mcp import types +from mcp.shared.memory import create_connected_server_and_client_session + +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.integrations.mcp.engine import ( + EmbeddedResolver, + EngineSpec, + build_engine, + build_tool_registration, + extend_with_chat_id, +) +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestCancelledToolActivity: + async def test_mcp_request_cancellation_during_initial_report_restores_reasoning_for_live_provider( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + ready, release = asyncio.Event(), asyncio.Event() + captured: list[AgentTools] = [] + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + captured.append(tools) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + ready.set() + + ctx = execution(handler) + await ctx.start() + await ctx.on_event(make_message_event()) + await ready.wait() + await ctx._wait_for_idle(timeout=2) + request_ids: list[int | str] = [] + + def resolve(room_id: str) -> AgentTools: + request_ids.append(server.get_context().request_context.request_id) + return captured[0] + + definition = TOOL_DEFINITIONS[BandTool.GET_PARTICIPANTS] + registration = build_tool_registration( + definition, + extend_with_chat_id(definition.input_model, None), + resolver=EmbeddedResolver(get_tools=resolve), + strip_chat_id=True, + ) + server = build_engine( + EngineSpec(name="activity-cancellation", tools=(registration,)) + ) + try: + async with create_connected_server_and_client_session(server) as session: + activity_platform.hold_next_activity = True + request = asyncio.create_task( + session.call_tool( + BandTool.GET_PARTICIPANTS, {"chat_id": ctx.room_id} + ) + ) + try: + await activity_platform.activity_entered.wait() + await session.send_notification( + types.ClientNotification( + types.CancelledNotification( + params=types.CancelledNotificationParams( + requestId=request_ids[-1] + ) + ) + ) + ) + await activity_platform.activity_cancelled.wait() + finally: + request.cancel() + await asyncio.gather(request, return_exceptions=True) + activity_platform.activity_release.set() + mark = len(activity_platform.activity_bodies) + await asyncio.sleep(ctx.config.working_keep_alive_seconds + 0.1) + assert not workers[0].task.done() + assert len(activity_platform.activity_bodies) > mark + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "reasoning"}, + } + finally: + release.set() + await workers[0].task diff --git a/tests/runtime/activityownership/test_claude_native.py b/tests/runtime/activityownership/test_claude_native.py new file mode 100644 index 000000000..4718cda0e --- /dev/null +++ b/tests/runtime/activityownership/test_claude_native.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.converters.claude_sdk import ClaudeSDKSessionState +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.registry import MCP_TOOL_PREFIX +from tests.adapters.claude_sdk.fakecli import scripted_claude +from tests.adapters.opencode.helpers import make_platform_message +from tests.baseline.decisions import ModelDecision +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ROOM_ID + + +class TestClaudeNativeWireActivity: + @pytest.mark.httpx_mock(should_mock=lambda request: request.url.host != "127.0.0.1") + async def test_real_context_observes_native_tool_without_tool_narration( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + with scripted_claude() as cli: + cli.script( + [ + ModelDecision.call("Bash", command="private-argument"), + ModelDecision.call( + f"{MCP_TOOL_PREFIX}band_send_message", + content="private-result", + mentions=["@alice"], + ), + ] + ) + adapter = ClaudeSDKAdapter(emit=()) + await adapter.on_started("Test", "Test") + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + await adapter.on_message( + make_platform_message( + room_id=ROOM_ID, sender_id="user-456", sender_name="Alice" + ), + AgentTools.from_context(ctx), + ClaudeSDKSessionState(), + None, + None, + is_session_bootstrap=True, + room_id=ROOM_ID, + ) + + ctx = execution(handler) + try: + await ctx._process_event(make_message_event()) + assert { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } in activity_platform.activity_bodies + assert activity_platform.activity_bodies[-1] == {"working": False} + activity_platform.assert_no_tool_narration() + activity_platform.assert_hides("private-argument", "private-result") + finally: + await adapter.cleanup_all() diff --git a/tests/runtime/activityownership/test_detached_detail_feature.py b/tests/runtime/activityownership/test_detached_detail_feature.py new file mode 100644 index 000000000..4d7481aef --- /dev/null +++ b/tests/runtime/activityownership/test_detached_detail_feature.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from band.adapters.opencode import OpencodeAdapter +from band.core.types import Emit +from band.integrations.opencode.types import OpencodeSessionState +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import report_tool_activity +from tests.adapters.opencode.helpers import ( + FakeOpencodeClient, + event_permission, + event_session_idle, + event_tool_part, + make_platform_message, +) +from tests.conftest import make_message_event +from tests.mcpbackends import backends_created_by +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ROOM_ID + + +class TestDetachedActivityFeature: + @pytest.mark.looptime + @pytest.mark.parametrize( + "variant", ["normal", "silent", "downgrade", "cap", "disabled", "hub"] + ) + async def test_parked_approval_owns_detail_until_native_completion( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + httpx_mock: HTTPXMock, + variant: str, + ) -> None: + activity_platform.reject_detail = variant == "downgrade" + resumed, release = asyncio.Event(), asyncio.Event() + captured: list[AgentTools] = [] + + async def executing() -> None: + resumed.set() + await release.wait() + + def tool(status: str) -> dict[str, Any]: + return event_tool_part( + "sess-1", + "native", + tool="bash", + call_id="call", + status=status, + input_data={"command": "private-argument"}, + output="private-result", + ) + + client = FakeOpencodeClient( + prompt_event_sequences=[ + [tool("pending"), event_permission("sess-1", "ask")] + ], + reply_permission_events={ + "ask": [ + tool("running"), + executing, + tool("completed"), + event_session_idle("sess-1"), + ] + }, + ) + adapter = OpencodeAdapter( + client_factory=lambda _: client, + emit=() if variant == "silent" else (Emit.TOOL_CALLS,), + ) + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + captured.append(tools) + await adapter.on_message( + make_platform_message(room_id=ROOM_ID, sender_id="user-456"), + tools, + OpencodeSessionState(), + None, + None, + is_session_bootstrap=True, + room_id=ROOM_ID, + ) + + settings = {} + if variant == "cap": + settings["max_working_state_seconds"] = 5.0 + if variant == "disabled": + settings["enable_working_state"] = False + if variant == "hub": + settings["hub_room_id"] = ROOM_ID + with backends_created_by(): + await adapter.on_started("Test", "Test") + ctx = execution(handler, **settings) + try: + await ctx._process_event(make_message_event(msg_id="original")) + owner = captured[0]._scope + assert owner is not None + workers = tuple(owner.workers) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(9.1) + parked = list(activity_platform.activity_bodies) + match variant: + case "disabled" | "hub": + assert not parked + assert not httpx_mock.get_requests() + case "downgrade": + assert parked[0].get("activity") == {"kind": "reasoning"} + assert len(parked[1:]) >= 3 + assert all(body == {"working": True} for body in parked[1:]) + case "cap": + assert parked[-1] == { + "working": True, + "activity": {"kind": "permission"}, + } + assert len(parked) == 4 + case _: + assert ( + parked[-3:] + == [{"working": True, "activity": {"kind": "permission"}}] + * 3 + ) + assert activity_platform.marked("processed") == ["original"] + assert await adapter.try_handle_control( + make_platform_message( + room_id=ROOM_ID, content="approve ask", sender_id="user-456" + ), + captured[-1], + ) + await resumed.wait() + if variant in {"normal", "silent"}: + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "bash"}, + } + if variant == "cap": + assert activity_platform.activity_bodies == parked + release.set() + await asyncio.gather(*(worker.task for worker in workers)) + await ctx._join_activity_completion() + if variant in {"disabled", "hub"}: + assert not activity_platform.activity_bodies + assert not httpx_mock.get_requests() + else: + assert activity_platform.activity_bodies[-1] == {"working": False} + assert ( + activity_platform.activity_bodies.count({"working": False}) == 1 + ) + assert not client.aborted_sessions + activity_platform.assert_hides("private-argument", "private-result") + before = list(activity_platform.activity_bodies) + await report_tool_activity(captured[0], "late-old-callback") + assert activity_platform.activity_bodies == before + if variant == "silent": + activity_platform.assert_no_tool_narration() + finally: + release.set() + await adapter.on_cleanup(ROOM_ID) diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py new file mode 100644 index 000000000..564a0298c --- /dev/null +++ b/tests/runtime/activityownership/test_detail.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest + +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, +) +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform, ObservedIdle +from tests.runtime.helpers import ROOM_ID + + +class TestOwnedDetail: + async def test_real_tools_observer_reports_canonical_wire_detail( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await end_tool_activity(tools) + await report_memory_activity(tools) + await set_permission_wait(tools, False) + await end_tool_activity(tools) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": {"kind": "permission"}}, + {"working": True, "activity": {"kind": "memory"}}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + + @pytest.mark.looptime + async def test_settled_owner_remains_authoritative_and_transient_tools_are_quiet( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + release = asyncio.Event() + captured: list[AgentTools] = [] + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if captured: + await report_tool_activity(tools, "unrelated") + return + captured.append(tools) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + await set_permission_wait(tools, True) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(6.1) + assert ( + activity_platform.activity_bodies[-2:] + == [{"working": True, "activity": {"kind": "permission"}}] * 2 + ) + await report_tool_activity(captured[0], "Bash") + await set_permission_wait(captured[0], False) + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } + release.set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert activity_platform.activity_bodies[-1] == {"working": False} + before = list(activity_platform.activity_bodies) + await end_tool_activity(captured[0]) + assert activity_platform.activity_bodies == before + + async def test_settled_turn_cannot_restart_reporting( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + await ctx._join_activity_completion() + before = list(activity_platform.activity_bodies) + await report_tool_activity(captured[0], "Bash") + assert before[-1] == {"working": False} + assert activity_platform.activity_bodies == before + + async def test_graceful_stop_drops_observations_from_the_draining_turn( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + handler_started = asyncio.Event() + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + handler_started.set() + await release.wait() + + ctx = execution(handler) + idle = ctx._idle_event = ObservedIdle() + await ctx.start() + await ctx.on_event(make_message_event()) + await handler_started.wait() + stopping = asyncio.create_task(ctx.stop(timeout=30)) + await idle.awaited.wait() + await report_tool_activity(captured[0], "Bash") + release.set() + assert await stopping + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + + async def test_queued_observation_is_rechecked_after_stop_and_replacement( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + handler_started = asyncio.Event() + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + handler_started.set() + await release.wait() + + ctx = execution(handler) + turn = asyncio.create_task(ctx._process_event(make_message_event(msg_id="old"))) + await handler_started.wait() + activity_platform.hold_next_activity = True + change = asyncio.create_task(report_tool_activity(captured[0], "Bash")) + await activity_platform.activity_entered.wait() + stale = asyncio.create_task(report_memory_activity(captured[0])) + ctx.interrupt() + activity_platform.activity_release.set() + await asyncio.gather(turn, change, stale) + release.set() + await ctx._process_event(make_message_event(msg_id="new")) + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + assert not any( + body.get("activity", {}).get("kind") == "memory" + for body in activity_platform.activity_bodies + ) + + @pytest.mark.looptime + async def test_waiting_delivery_takes_over_reporting_once_the_owner_finishes( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + worker_release = asyncio.Event() + waiting_tools: list[AgentTools] = [] + waiting_started = asyncio.Event() + waiting_release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if workers: + waiting_tools.append(tools) + waiting_started.set() + await waiting_release.wait() + return + workers.append( + bind_turn_task(tools, asyncio.create_task(worker_release.wait())) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="owner")) + waiting = asyncio.create_task( + ctx._process_event(make_message_event(msg_id="waiting")) + ) + await waiting_started.wait() + worker_release.set() + await workers[0].task + await ctx._join_activity_completion() + await report_tool_activity(waiting_tools[0], "Bash") + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + ] + waiting_release.set() + await waiting + + @pytest.mark.parametrize("excluded", ["disabled", "hub"]) + async def test_excluded_reporting_ignores_observations( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + excluded: str, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await report_memory_activity(tools) + await end_tool_activity(tools) + + settings = ( + {"enable_working_state": False} + if excluded == "disabled" + else {"hub_room_id": ROOM_ID} + ) + ctx = execution(handler, **settings) + await ctx._process_event(make_message_event()) + assert not activity_platform.activity_bodies diff --git a/tests/runtime/activityownership/test_housekeeping.py b/tests/runtime/activityownership/test_housekeeping.py new file mode 100644 index 000000000..c0a36a247 --- /dev/null +++ b/tests/runtime/activityownership/test_housekeeping.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest +from pytest_httpx import HTTPXMock + +from band.core.turn_lifecycle import bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from tests.conftest import make_participant_added_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestHousekeepingActivity: + @pytest.mark.parametrize("detached", [False, True]) + async def test_registered_tools_and_workers_cannot_start_housekeeping_reports( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + httpx_mock: HTTPXMock, + detached: bool, + ) -> None: + handled, release = asyncio.Event(), asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await tools.get_participants() + if detached: + bind_turn_task(tools, asyncio.create_task(release.wait())) + handled.set() + + ctx = execution(handler) + await ctx.start() + try: + await ctx.on_event(make_participant_added_event()) + await handled.wait() + await ctx._wait_for_idle(timeout=2) + assert not activity_platform.activity_bodies + assert not httpx_mock.get_requests() + finally: + release.set() diff --git a/tests/runtime/activityownership/test_opencode.py b/tests/runtime/activityownership/test_opencode.py index e96a45859..9980f03df 100644 --- a/tests/runtime/activityownership/test_opencode.py +++ b/tests/runtime/activityownership/test_opencode.py @@ -19,7 +19,7 @@ ) from tests.conftest import make_message_event from tests.mcpbackends import backends_created_by -from tests.runtime.helpers import IDLE, KEEP_ALIVE, ROOM_ID, LifecyclePlatform +from tests.runtime.helpers import ROOM_ID, LifecyclePlatform class TestOpenCodeLifetime: @@ -59,7 +59,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) try: await ctx._process_event(make_message_event(msg_id="original")) - assert activity_platform.activity_reports == [KEEP_ALIVE] + assert activity_platform.working_transitions == [True] assert activity_platform.marked("processed") == ["original"] assert ctx.interrupt(kind=mode) notify = asyncio.create_task(adapter.on_interrupt(ROOM_ID, mode)) @@ -71,7 +71,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await ctx._join_activity_completion() assert client.aborted_sessions == ["sess-1"] assert cleaned.is_set() - assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.working_transitions == [True, False] assert activity_platform.marked("processed") == ["original"] finally: release.set() diff --git a/tests/runtime/conftest.py b/tests/runtime/conftest.py index 46f8a06ed..f7457f7ed 100644 --- a/tests/runtime/conftest.py +++ b/tests/runtime/conftest.py @@ -34,6 +34,7 @@ from band.core.types import PlatformConnection, PlatformMessage from band.runtime.presence import RoomPresence +from tests.activitysupport import install_activity_support from tests.conftest import make_participant_mock from tests.runtime.helpers import AGENT_ID @@ -259,3 +260,9 @@ def link(mock_rest_client: MagicMock) -> MagicMock: link.get_stale_processing_messages = AsyncMock(return_value=[]) link.report_activity = AsyncMock(return_value=True) return link + + +@pytest.fixture(autouse=True) +def runtime_activity_support(monkeypatch: pytest.MonkeyPatch) -> None: + """Start every runtime test on a platform that already advertised detail.""" + install_activity_support(monkeypatch, learned=True) diff --git a/tests/runtime/execution/test_crashrecoverysync.py b/tests/runtime/execution/test_crashrecoverysync.py index c2d6bc739..b779bf812 100644 --- a/tests/runtime/execution/test_crashrecoverysync.py +++ b/tests/runtime/execution/test_crashrecoverysync.py @@ -2,12 +2,14 @@ import asyncio from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, call import pytest from band_sdk_core import ClaimRegistry from band.client.streaming import MessageMetadata +from band.runtime import resync from band.runtime.execution import ( ExecutionContext, ResyncOutcome, @@ -1035,9 +1037,14 @@ async def test_refused_claims_never_spend_the_retry_budget( mock_handler.assert_awaited_once() assert not ctx._retry_tracker.is_permanently_failed(msg.id) + @pytest.mark.parametrize("initial_recovery_delay", [0.0, 10.0]) async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( - self, mock_link_with_next, mock_handler - ): + self, + mock_link_with_next, + mock_handler, + monkeypatch: pytest.MonkeyPatch, + initial_recovery_delay: float, + ) -> None: """Startup sync should not switch to WebSocket after an unclaimable backlog message.""" backlog_msg = PlatformMessage( @@ -1063,6 +1070,12 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( ), ) + monkeypatch.setattr( + resync, + "random", + SimpleNamespace(uniform=lambda lower, upper: initial_recovery_delay), + ) + await ctx.on_event( make_message_event(room_id="room-123", msg_id="msg-newer-ws") ) @@ -1071,8 +1084,10 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( lambda: mock_link_with_next.mark_processing.await_count >= 1 ) - mock_link_with_next.mark_processing.assert_awaited_once_with( - "room-123", "msg-older-claim-fails" + assert ( + mock_link_with_next.mark_processing.await_args_list + == [call("room-123", "msg-older-claim-fails")] + * mock_link_with_next.mark_processing.await_count ) mock_handler.assert_not_called() assert ctx._sync_complete is False diff --git a/tests/runtime/tooltests/test_agent_tools_context_sync_back.py b/tests/runtime/tooltests/test_agent_tools_context_sync_back.py index fa6acd8ee..7d317c3b2 100644 --- a/tests/runtime/tooltests/test_agent_tools_context_sync_back.py +++ b/tests/runtime/tooltests/test_agent_tools_context_sync_back.py @@ -20,64 +20,6 @@ class TestAgentToolsContextSyncBack: On the next turn, from_context() would copy the stale list again. """ - @pytest.mark.asyncio - async def test_add_participant_syncs_back_to_ctx(self, mock_rest_client): - """add_participant() must call ctx.add_participant() with full dict.""" - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [] - mock_ctx.hub_room_id = None - mock_ctx.add_participant = MagicMock() - - # Override list_agent_chat_participants to return empty (avoid "already_in_room") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[])) - ) - - tools = AgentTools.from_context(mock_ctx) - assert tools._ctx is mock_ctx - - await tools.add_participant("Agent Two") - - mock_ctx.add_participant.assert_called_once() - added = mock_ctx.add_participant.call_args.args[0] - assert added["id"] == "agent-2" - assert added["name"] == "Agent Two" - assert added["type"] == "Agent" - assert added["handle"] == "agent-two" - assert added["description"] == "Another agent" - - @pytest.mark.asyncio - async def test_remove_participant_syncs_back_to_ctx(self, mock_rest_client): - """remove_participant() must call ctx.remove_participant() with correct ID.""" - participant = { - "id": "user-1", - "name": "User One", - "type": "User", - "handle": "user-one", - } - - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [participant] - mock_ctx.hub_room_id = None - mock_ctx.remove_participant = MagicMock() - - # Return same participant from REST so snapshot matches - p_mock = make_participant_mock("user-1", "User One", "User", handle="user-one") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[p_mock])) - ) - - tools = AgentTools.from_context(mock_ctx) - await tools.remove_participant("User One") - - mock_ctx.remove_participant.assert_called_once_with("user-1") - @pytest.mark.asyncio async def test_add_participant_persists_across_recreated_tools( self, mock_rest_client @@ -103,8 +45,17 @@ async def test_add_participant_persists_across_recreated_tools( tools1 = AgentTools.from_context(ctx) await tools1.add_participant("Agent Two") + added = ctx.participants[0] assert len(ctx.participants) == 1 - assert ctx.participants[0]["id"] == "agent-2" + assert { + key: added[key] for key in ("id", "name", "type", "handle", "description") + } == { + "id": "agent-2", + "name": "Agent Two", + "type": "Agent", + "handle": "agent-two", + "description": "Another agent", + } # Turn 2: recreate tools — participant must still be there tools2 = AgentTools.from_context(ctx) diff --git a/tests/runtime/workingstate/test_detail.py b/tests/runtime/workingstate/test_detail.py new file mode 100644 index 000000000..6cecd58ec --- /dev/null +++ b/tests/runtime/workingstate/test_detail.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from band_rest import AgentActivityDetail + +from band.runtime.working_state import WorkingStateReporter + + +@pytest.mark.looptime +class TestActivityDetail: + async def test_changes_repeat_current_detail_and_keep_boolean_callback( + self, + ) -> None: + reports: list[tuple[bool, dict[str, Any] | None]] = [] + + async def report(working: bool) -> None: + reports.append( + ( + working, + reporter.activity.model_dump(exclude_unset=True) + if working + else None, + ) + ) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="tool_call", name="Bash", arguments="private" + ), + ) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="reasoning", name="private", output="private" + ), + ) + finally: + await reporter.stop() + assert reports == [ + (True, {"kind": "reasoning"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "reasoning"}), + (False, None), + ] + + async def test_manual_wait_overrides_latest_progress_without_restoring_stale_state( + self, + ) -> None: + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Read"), + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="reasoning") + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="memory") + ) + await reporter.update(current=lambda: True, waiting=False) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call", "permission", "memory"] + + async def test_cap_prevents_detail_from_reviving_the_indicator(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report, max_working_state_seconds=2) + try: + await reporter.start() + await asyncio.sleep(2.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3) + assert reports == [True] + finally: + await reporter.stop() + assert reports == [True, False] + + async def test_observation_queued_behind_a_report_is_rechecked_under_the_lock( + self, + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + if reporter.activity.kind == "tool_call": + entered.set() + await release.wait() + + reporter = WorkingStateReporter(report) + owned = True + try: + await reporter.start() + change = asyncio.create_task( + reporter.update( + current=lambda: owned, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + ) + await entered.wait() + stale = asyncio.create_task( + reporter.update( + current=lambda: owned, activity=AgentActivityDetail(kind="memory") + ) + ) + owned = False + release.set() + await asyncio.gather(change, stale) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call"] + + async def test_empty_tool_name_reports_unknown(self) -> None: + reports: list[dict[str, Any]] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.model_dump(exclude_unset=True)) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name=""), + ) + finally: + await reporter.stop() + assert reports[-1] == {"kind": "tool_call", "name": "unknown"} + + async def test_observation_outside_a_cycle_reports_nothing(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + assert reports == [] + + async def test_permission_wait_does_not_carry_into_the_next_cycle(self) -> None: + reports: list[str | None] = [] + + async def report(working: bool) -> None: + reports.append(reporter.activity.kind if working else None) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update(current=lambda: True, waiting=True) + await reporter.stop() + await reporter.start() + finally: + await reporter.stop() + assert reports == ["reasoning", "permission", None, "reasoning", None] diff --git a/tests/toolkit/activity/__init__.py b/tests/toolkit/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/toolkit/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/toolkit/activity/test_subscription.py b/tests/toolkit/activity/test_subscription.py new file mode 100644 index 000000000..2a1d73fa9 --- /dev/null +++ b/tests/toolkit/activity/test_subscription.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest +from phoenix_channels_python_client.exceptions import PHXTopicError + +from band.client.streaming import WebSocketClient +from band.testing import JoinOutcome, fake_phoenix_server +from tests.e2e.baseline.toolkit.activity import ActivityEvent, activity_topic +from tests.e2e.baseline.toolkit.capture import reply_capture + + +class TestActivitySubscription: + async def test_joins_before_trigger_dispatches_and_leaves(self) -> None: + async with ( + fake_phoenix_server() as server, + WebSocketClient(ws_url=server.url, api_key="test", agent_id=None) as ws, + ): + async with reply_capture( + ws, "room", observe_activity=True, deadline_s=1 + ) as capture: + topic = activity_topic("room") + assert topic in server.joined_topics + await server.push( + topic, + ActivityEvent.CHANGED, + { + "agent_id": "agent", + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + }, + ) + await capture.wait_for_tool_activity("agent", {"Bash"}) + mark = capture.activities.snapshot() + await server.push( + topic, + ActivityEvent.STOPPED, + {"agent_id": "agent", "working": False}, + ) + await capture.wait_for_activity_stop("agent", since=mark) + assert topic not in ws.joined_topics() + assert "chat_room:room" not in ws.joined_topics() + + async def test_rejected_activity_join_fails_and_releases_ordinary_capture( + self, + ) -> None: + topic = activity_topic("room") + async with ( + fake_phoenix_server( + join_outcomes={topic: [JoinOutcome.REJECTED]} + ) as server, + WebSocketClient(ws_url=server.url, api_key="test", agent_id=None) as ws, + ): + with pytest.raises(PHXTopicError): + async with reply_capture(ws, "room", observe_activity=True): + pytest.fail( + "Rejected activity subscription must not run a scenario" + ) + assert "chat_room:room" not in ws.joined_topics()